Index: Python/pythonrun.c =================================================================== --- Python/pythonrun.c (revision 58574) +++ Python/pythonrun.c (working copy) @@ -719,7 +719,7 @@ } /* Set sys.stdin */ - if (!(std = PyFile_FromFileEx(stdin, "", "r", fclose, -1, + if (!(std = PyFile_FromFd(STDIN_FILENO, "", "r", -1, NULL, "\n"))) { goto error; } @@ -728,7 +728,7 @@ Py_DECREF(std); /* Set sys.stdout */ - if (!(std = PyFile_FromFileEx(stdout, "", "w", fclose, -1, + if (!(std = PyFile_FromFd(STDOUT_FILENO, "", "w", -1, NULL, "\n"))) { goto error; } @@ -737,7 +737,7 @@ Py_DECREF(std); /* Set sys.stderr */ - if (!(std = PyFile_FromFileEx(stderr, "", "w", fclose, -1, + if (!(std = PyFile_FromFd(STDERR_FILENO, "", "w", -1, NULL, "\n"))) { goto error; } Index: Python/import.c =================================================================== --- Python/import.c (revision 58574) +++ Python/import.c (working copy) @@ -92,7 +92,7 @@ extern struct _inittab _PyImport_Inittab[]; /* Method from Parser/tokenizer.c */ -extern char * PyTokenizer_FindEncoding(FILE *fp); +extern char * PyTokenizer_FindEncoding(int); struct _inittab *PyImport_Inittab = _PyImport_Inittab; @@ -2561,6 +2561,7 @@ struct filedescr *fdp; char pathname[MAXPATHLEN+1]; FILE *fp = NULL; + int fd = -1; char *found_encoding = NULL; char *encoding = NULL; @@ -2571,17 +2572,24 @@ if (fdp == NULL) return NULL; if (fp != NULL) { + fd = fileno(fp); + if (fd != -1) + fd = dup(fd); + fclose(fp); + fp = NULL; + } + if (fd != -1) { if (strchr(fdp->mode, 'b') == NULL) { /* PyTokenizer_FindEncoding() returns PyMem_MALLOC'ed memory. */ - found_encoding = PyTokenizer_FindEncoding(fp); + found_encoding = PyTokenizer_FindEncoding(fd); encoding = (found_encoding != NULL) ? found_encoding : (char*)PyUnicode_GetDefaultEncoding(); } - fob = PyFile_FromFileEx(fp, pathname, fdp->mode, fclose, -1, + fob = PyFile_FromFd(fd, pathname, fdp->mode, -1, (char*)encoding, NULL); if (fob == NULL) { - fclose(fp); + close(fd); PyMem_FREE(found_encoding); return NULL; } Index: Include/fileobject.h =================================================================== --- Include/fileobject.h (revision 58574) +++ Include/fileobject.h (working copy) @@ -8,10 +8,7 @@ #define PY_STDIOTEXTMODE "b" -PyAPI_FUNC(PyObject *) PyFile_FromFile(FILE *, char *, char *, int (*)(FILE*)); -PyAPI_FUNC(PyObject *) PyFile_FromFileEx(FILE *, char *, char *, - int (*)(FILE *), int, char *, - char *); +PyAPI_FUNC(PyObject *) PyFile_FromFd(int, char *, char *, int, char *, char *); PyAPI_FUNC(PyObject *) PyFile_GetLine(PyObject *, int); PyAPI_FUNC(int) PyFile_WriteObject(PyObject *, PyObject *, int); PyAPI_FUNC(int) PyFile_WriteString(const char *, PyObject *); Index: Objects/fileobject.c =================================================================== --- Objects/fileobject.c (revision 58574) +++ Objects/fileobject.c (working copy) @@ -26,21 +26,15 @@ /* External C interface */ PyObject * -PyFile_FromFile(FILE *fp, char *name, char *mode, int (*close)(FILE *)) +PyFile_FromFd(int fd, char *name, char *mode, int buffering, char *encoding, + char *newline) { - return PyFile_FromFileEx(fp, name, mode, close, -1, NULL, NULL); -} - -PyObject * -PyFile_FromFileEx(FILE *fp, char *name, char *mode, int (*close)(FILE *), - int buffering, char *encoding, char *newline) -{ PyObject *io, *stream, *nameobj=NULL; io = PyImport_ImportModule("io"); if (io == NULL) return NULL; - stream = PyObject_CallMethod(io, "open", "isiss", fileno(fp), mode, + stream = PyObject_CallMethod(io, "open", "isiss", fd, mode, buffering, encoding, newline); Py_DECREF(io); if (stream == NULL) Index: Objects/bytesobject.c =================================================================== --- Objects/bytesobject.c (revision 58574) +++ Objects/bytesobject.c (working copy) @@ -1214,7 +1214,7 @@ Py_ssize_t len = PyBytes_GET_SIZE(self); const char* str; Py_buffer vsubstr; - int rv; + int rv = -1; str = PyBytes_AS_STRING(self); Index: Misc/NEWS =================================================================== --- Misc/NEWS (revision 58574) +++ Misc/NEWS (working copy) @@ -8,6 +8,18 @@ *Unreleased* +Core and Builtins +----------------- + +- Replaced `PyFile_FromFile()` with `PyFile_FromFd(fd, name. mode, buffer, + encoding, newline)` + +- Fixed `imp.find_module()` to obey the -*- coding: -*- header. + +- Changed `__file__` and `co_filename` to unicode. The path names are decoded + with `Py_FileSystemDefaultEncoding` and a new API method + `PyUnicode_DecodeFSDefault(char*)` was added. + Extension Modules ----------------- Index: Parser/tokenizer.c =================================================================== --- Parser/tokenizer.c (revision 58574) +++ Parser/tokenizer.c (working copy) @@ -1612,13 +1612,19 @@ when no longer needed. */ char * -PyTokenizer_FindEncoding(FILE *fp) { +PyTokenizer_FindEncoding(int fd) { struct tok_state *tok; + FILE *fp; char *p_start=NULL, *p_end=NULL, *encoding=NULL; + fp = fdopen(dup(fd), "r"); + if (fp == NULL) + return NULL; + if ((tok = PyTokenizer_FromFile(fp, NULL, NULL, NULL)) == NULL) { /* lseek() usage is on purpose; see note later in code. */ - lseek(fileno(fp), 0, 0); + fclose(fp); + lseek(fd, 0, 0); return NULL; } while(((tok->lineno < 2) && (tok->done == E_OK))) { @@ -1628,7 +1634,8 @@ /* lseek() must be used instead of fseek()/rewind() as those fail on OS X 10.4 to properly seek back to the beginning when reading from the file descriptor instead of the file pointer. */ - lseek(fileno(fp), 0, 0); + fclose(fp); + lseek(fd, 0, 0); if (tok->encoding) { encoding = (char *)PyMem_MALLOC(strlen(tok->encoding)); Index: Parser/tokenizer.h =================================================================== --- Parser/tokenizer.h (revision 58574) +++ Parser/tokenizer.h (working copy) @@ -67,7 +67,7 @@ extern int PyTokenizer_Get(struct tok_state *, char **, char **); extern char * PyTokenizer_RestoreEncoding(struct tok_state* tok, int len, int *offset); -extern char * PyTokenizer_FindEncoding(FILE *fp); +extern char * PyTokenizer_FindEncoding(int); #ifdef __cplusplus } Index: Doc/c-api/concrete.rst =================================================================== --- Doc/c-api/concrete.rst (revision 58575) +++ Doc/c-api/concrete.rst (working copy) @@ -2410,33 +2410,25 @@ :ctype:`PyFileObject`. -.. cfunction:: PyObject* PyFile_FromString(char *filename, char *mode) +.. cfunction:: PyFile_FromFd(int fd, char *name, char *mode, int buffering, char *encoding, char *newline) - .. index:: single: fopen() + Create a new :ctype:`PyFileObject` from the file descriptor of an already + opened file *fd*. The arguments *name*, *encoding* and *newline* can be + *NULL* as well as buffering can be *-1* to use the defaults. Return *NULL* on + failure. - On success, return a new file object that is opened on the file given by - *filename*, with a file mode given by *mode*, where *mode* has the same - semantics as the standard C routine :cfunc:`fopen`. On failure, return *NULL*. + .. warning:: + Take care when you are mixing streams and descriptors! For more + information, see `GNU C Library + `_. -.. cfunction:: PyObject* PyFile_FromFile(FILE *fp, char *name, char *mode, int (*close)(FILE*)) - Create a new :ctype:`PyFileObject` from the already-open standard C file - pointer, *fp*. The function *close* will be called when the file should be - closed. Return *NULL* on failure. +.. cfunction:: int PyObject_AsFileDescriptor(PyObject *p) -.. cfunction:: PyFile_FromFileEx(FILE *fp, char *name, char *mode, int (*close)(FILE *), int buffering, char *encoding, char *newline) + Return the file descriptor associated with *p* as an :ctype:`int`. - Create a new :ctype:`PyFileObject` from the already-open standard C file - pointer, *fp*. The functions works similar to *PyFile_FromFile* but takes - optional arguments for *buffering*, *encoding* and *newline*. Use -1 resp. - *NULL* for default values. -.. cfunction:: FILE* PyFile_AsFile(PyObject *p) - - Return the file object associated with *p* as a :ctype:`FILE\*`. - - .. cfunction:: PyObject* PyFile_GetLine(PyObject *p, int n) .. index:: single: EOFError (built-in exception) Index: Doc/reference/introduction.rst =================================================================== --- Doc/reference/introduction.rst (revision 58575) +++ Doc/reference/introduction.rst (working copy) @@ -60,7 +60,7 @@ This implementation actually uses the CPython implementation, but is a managed .NET application and makes .NET libraries available. This was created by Brian Lloyd. For more information, see the `Python for .NET home page - `_. + `_. IronPython An alternate Python for .NET. Unlike Python.NET, this is a complete Python Index: Lib/test/test_imp.py =================================================================== --- Lib/test/test_imp.py (revision 58574) +++ Lib/test/test_imp.py (working copy) @@ -44,6 +44,18 @@ fd = imp.find_module("heapq")[0] self.assertEqual(fd.encoding, "iso-8859-1") + def test_issue1267(self): + fd, filename, info = imp.find_module("pydoc") + self.assertEqual(fd.encoding, "iso-8859-1") + self.assertEqual(fd.tell(), 0) + self.assertEqual(fd.readline(), '#!/usr/bin/env python\n') + + fd, filename, info = imp.find_module("tokenize") + self.assertEqual(fd.encoding, "utf-8") + self.assertEqual(fd.tell(), 0) + self.assertEqual(fd.readline(), '"""Tokenization help for Python programs.\n') + + def test_main(): test_support.run_unittest( LockTests, Index: Modules/posixmodule.c =================================================================== --- Modules/posixmodule.c (revision 58574) +++ Modules/posixmodule.c (working copy) @@ -5386,11 +5386,18 @@ posix_tmpfile(PyObject *self, PyObject *noargs) { FILE *fp; + int fd; fp = tmpfile(); if (fp == NULL) return posix_error(); - return PyFile_FromFile(fp, "", "w+b", fclose); + fd = fileno(fp); + if (fd != -1) + fd = dup(fd); + fclose(fp); + if (fd == -1) + return posix_error(); + return PyFile_FromFd(fd, "", "w+b", -1, NULL, NULL); } #endif