Index: Lib/cgi.py =================================================================== --- Lib/cgi.py (revision 87747) +++ Lib/cgi.py (working copy) @@ -31,7 +31,7 @@ # Imports # ======= -from io import StringIO +from io import StringIO, BytesIO import sys import os import urllib.parse @@ -352,9 +352,10 @@ value: the value as a *string*; for file uploads, this transparently reads the file every time you request the value + and returns *bytes* - file: the file(-like) object from which you can read the data; - None if the data is stored a simple string + file: the file(-like) object from which you can read the data *in + binary mode* ; None if the data is stored a simple string type: the content-type, or None if not specified @@ -375,9 +376,9 @@ directory and unlinking them as soon as they have been opened. """ - def __init__(self, fp=None, headers=None, outerboundary="", - environ=os.environ, keep_blank_values=0, strict_parsing=0): + environ=os.environ, keep_blank_values=0, strict_parsing=0, + limit=None): """Constructor. Read multipart/* until last part. Arguments, all optional: @@ -436,6 +437,9 @@ self.fp = fp or sys.stdin self.headers = headers self.outerboundary = outerboundary + + self.bytes_read = 0 + self.limit = limit # Process content-disposition header cdisp, pdict = "", {} @@ -482,6 +486,8 @@ if maxlen and clen > maxlen: raise ValueError('Maximum content length exceeded') self.length = clen + if self.limit is None and clen: + self.limit = clen self.list = self.file = None self.done = 0 @@ -531,7 +537,7 @@ """Dictionary style get() method, including 'value' lookup.""" if key in self: value = self[key] - if type(value) is type([]): + if isinstance(value,list): return [x.value for x in value] else: return value.value @@ -542,7 +548,7 @@ """ Return the first value received.""" if key in self: value = self[key] - if type(value) is type([]): + if isinstance(value,list): return value[0].value else: return value.value @@ -553,7 +559,7 @@ """ Return list of received values.""" if key in self: value = self[key] - if type(value) is type([]): + if isinstance(value,list): return [x.value for x in value] else: return [value.value] @@ -584,10 +590,10 @@ qs = self.fp.read(self.length) if self.qs_on_post: qs += '&' + self.qs_on_post - self.list = list = [] + self.list = [] for key, value in urllib.parse.parse_qsl(qs, self.keep_blank_values, self.strict_parsing): - list.append(MiniFieldStorage(key, value)) + self.list.append(MiniFieldStorage(key, value)) self.skip_lines() FieldStorageClass = None @@ -600,23 +606,36 @@ self.list = [] if self.qs_on_post: for key, value in urllib.parse.parse_qsl(self.qs_on_post, - self.keep_blank_values, self.strict_parsing): + self.keep_blank_values, self.strict_parsing): self.list.append(MiniFieldStorage(key, value)) FieldStorageClass = None klass = self.FieldStorageClass or self.__class__ - parser = email.parser.FeedParser() - # Create bogus content-type header for proper multipart parsing - parser.feed('Content-Type: %s; boundary=%s\r\n\r\n' % (self.type, ib)) - parser.feed(self.fp.read()) - full_msg = parser.close() - # Get subparts - msgs = full_msg.get_payload() - for msg in msgs: - fp = StringIO(msg.get_payload()) - part = klass(fp, msg, ib, environ, keep_blank_values, - strict_parsing) + # data must be read as bytes, not strings, so we use the buffer attribute + first_line = self.fp.buffer.readline() + self.bytes_read += len(first_line) + # first line holds boundary ; ignore it, or check that + # "--"+ib == first_line.decode('ascii').strip() ? + while True: + parser = email.parser.FeedParser() + hdr_text = b"" + while True: + data = self.fp.buffer.readline() + hdr_text += data + if not data.strip(): + break + if not hdr_text: + break + # parser takes strings, not bytes + self.bytes_read += len(hdr_text) + parser.feed(hdr_text.decode(self.fp.encoding)) + headers = parser.close() + part = klass(self.fp, headers, ib, environ, keep_blank_values, + strict_parsing,self.limit-self.bytes_read) + self.bytes_read += part.bytes_read self.list.append(part) + if self.bytes_read >= self.length: + break self.skip_lines() def read_single(self): @@ -636,7 +655,8 @@ todo = self.length if todo >= 0: while todo > 0: - data = self.fp.read(min(todo, self.bufsize)) + data = self.fp.buffer.read(min(todo, self.bufsize)) + self.bytes_read += len(data) if not data: self.done = -1 break @@ -645,42 +665,57 @@ def read_lines(self): """Internal: read lines until EOF or outerboundary.""" - self.file = self.__file = StringIO() + if self.filename is not None: + self.file = self.__file = BytesIO() # store data as bytes for files + else: + self.file = self.__file = StringIO() # as strings for other fields if self.outerboundary: self.read_lines_to_outerboundary() else: self.read_lines_to_eof() def __write(self, line): + """line is always bytes, not string""" if self.__file is not None: if self.__file.tell() + len(line) > 1000: self.file = self.make_file() data = self.__file.getvalue() self.file.write(data) self.__file = None - self.file.write(line) - + if self.filename is not None: + self.file.write(line) # keep bytes + else: + self.file.write(line.decode(self.fp.encoding)) # decode to string + def read_lines_to_eof(self): """Internal: read lines until EOF.""" - while 1: - line = self.fp.readline(1<<16) + while True: + line = self.fp.buffer.readline(1<<16) # bytes + self.bytes_read += len(line) if not line: self.done = -1 break self.__write(line) def read_lines_to_outerboundary(self): - """Internal: read lines until outerboundary.""" - next = "--" + self.outerboundary - last = next + "--" - delim = "" + """Internal: read lines until outerboundary. + Data is read as bytes : boundaries and line ends must be converted + to bytes for comparisons""" + next = b"--" + self.outerboundary.encode(self.fp.encoding) + last = next + b"--" + delim = b"" last_line_lfend = True - while 1: - line = self.fp.readline(1<<16) + _read = 0 + while True: + if _read >= self.limit: + break + line = self.fp.buffer.readline(1<<16) # bytes + self.bytes_read += len(line) + _read += len(line) if not line: self.done = -1 break - if line[:2] == "--" and last_line_lfend: + if line[:2] == b"--" and last_line_lfend: strippedline = line.strip() if strippedline == next: break @@ -688,16 +723,16 @@ self.done = 1 break odelim = delim - if line[-2:] == "\r\n": - delim = "\r\n" + if line.endswith(b"\r\n"): + delim = b"\r\n" line = line[:-2] last_line_lfend = True - elif line[-1] == "\n": - delim = "\n" + elif line.endswith(b"\n"): + delim = b"\n" line = line[:-1] last_line_lfend = True else: - delim = "" + delim = b"" last_line_lfend = False self.__write(odelim + line) @@ -705,22 +740,23 @@ """Internal: skip lines until outer boundary if defined.""" if not self.outerboundary or self.done: return - next = "--" + self.outerboundary - last = next + "--" + next = b"--" + self.outerboundary.encode(self.fp.encoding) + last = next + b"--" last_line_lfend = True - while 1: - line = self.fp.readline(1<<16) + while True: + line = self.fp.buffer.readline(1<<16) + self.bytes_read += len(line) if not line: self.done = -1 break - if line[:2] == "--" and last_line_lfend: + if line.endswith(b"--") and last_line_lfend: strippedline = line.strip() if strippedline == next: break if strippedline == last: self.done = 1 break - last_line_lfend = line.endswith('\n') + last_line_lfend = line.endswith(b'\n') def make_file(self): """Overridable: return a readable & writable file. @@ -730,7 +766,8 @@ - seek(0) - data is read from it - The file is always opened in text mode. + The file is opened in binary mode for files, in text mode + for other fields This version opens a temporary file for reading and writing, and immediately deletes (unlinks) it. The trick (on Unix!) is @@ -746,7 +783,10 @@ """ import tempfile - return tempfile.TemporaryFile("w+", encoding="utf-8", newline="\n") + if self.filename is not None: + return tempfile.TemporaryFile("wb+") + else: + return tempfile.TemporaryFile("w+") # Test/debug code Index: Lib/test/test_cgi.py =================================================================== --- Lib/test/test_cgi.py (revision 87747) +++ Lib/test/test_cgi.py (working copy) @@ -4,7 +4,7 @@ import sys import tempfile import unittest -from io import StringIO +from io import StringIO, BytesIO, TextIOWrapper class HackedSysModule: # The regression test will have real values in sys.argv, which @@ -106,7 +106,7 @@ return [(p[0], p[1][0]) for p in list] def gen_result(data, environ): - fake_stdin = StringIO(data) + fake_stdin = TextIOWrapper(BytesIO(data.encode('ascii'))) fake_stdin.seek(0) form = cgi.FieldStorage(fp=fake_stdin, environ=environ) @@ -216,7 +216,8 @@ Add\x20 -----------------------------721837373350705526688164684-- """ - fs = cgi.FieldStorage(fp=StringIO(postdata), environ=env) + fp = TextIOWrapper(BytesIO(postdata.encode('ascii'))) + fs = cgi.FieldStorage(fp=fp, environ=env) self.assertEqual(len(fs.list), 4) expect = [{'name':'id', 'filename':None, 'value':'1234'}, {'name':'title', 'filename':None, 'value':''},