diff -r 0d1536ec44e8 Lib/logging/handlers.py --- a/Lib/logging/handlers.py Thu Dec 01 03:22:44 2011 +0100 +++ b/Lib/logging/handlers.py Thu Dec 01 17:57:51 2011 +0100 @@ -25,6 +25,7 @@ """ import logging, socket, os, pickle, struct, time, re +import gzip, shutil from codecs import BOM_UTF8 from stat import ST_DEV, ST_INO, ST_MTIME import queue @@ -46,6 +47,14 @@ _MIDNIGHT = 24 * 60 * 60 # number of seconds in a day +def _gzip(sfn, dfn): + """ + Compres file in gzip file format + """ + with open(sfn, 'rb') as sfo, gzip.open(dfn, 'wb') as dfo: + shutil.copyfileobj(sfo, dfo) + os.remove(sfn) + class BaseRotatingHandler(logging.FileHandler): """ Base class for handlers that rotate log files at a certain point. @@ -81,7 +90,7 @@ Handler for logging to a set of files, which switches from one file to the next when the current file reaches a certain size. """ - def __init__(self, filename, mode='a', maxBytes=0, backupCount=0, encoding=None, delay=0): + def __init__(self, filename, mode='a', maxBytes=0, backupCount=0, encoding=None, delay=0, gzip=0): """ Open the specified file and use it as the stream for logging. @@ -100,6 +109,13 @@ exist, then they are renamed to "app.log.2", "app.log.3" etc. respectively. + Compress old log files is disabled by default. If you set gzip to a + value between 1 and backupCount, log files with number extension from + gzip value to up will be compressed using gzip file format. For + example, with backupCount set to 4 and gzip set to 2, you would get + "app.log", "app.log.1", "app.log.2.gz", "app.log.3.gz" and + "app.log.4.gz" + If maxBytes is zero, rollover never occurs. """ # If rotation/rollover is wanted, it doesn't make sense to use another @@ -112,6 +128,7 @@ BaseRotatingHandler.__init__(self, filename, mode, encoding, delay) self.maxBytes = maxBytes self.backupCount = backupCount + self.gzip = gzip def doRollover(self): """ @@ -121,17 +138,44 @@ self.stream.close() self.stream = None if self.backupCount > 0: - for i in range(self.backupCount - 1, 0, -1): - sfn = "%s.%d" % (self.baseFilename, i) - dfn = "%s.%d" % (self.baseFilename, i + 1) + nogzip = self.backupCount + if self.gzip > 0 and self.gzip <= self.backupCount: + nogzip = self.gzip - 1 + + # log.n.gz -> log.n+1.gz + for i in range(self.backupCount - 1, self.gzip - 1, -1): + sfn = "%s.%d.gz" % (self.baseFilename, i) + dfn = "%s.%d.gz" % (self.baseFilename, i + 1) + if os.path.exists(sfn): + if os.path.exists(dfn): + os.remove(dfn) + os.rename(sfn, dfn) + if self.gzip == 1: + # log -> log.1.gz + sfn = self.baseFilename + else: + # log.n -> log.n+1.gz + sfn = "%s.%d" % (self.baseFilename, self.gzip - 1) + dfn = "%s.%d.gz" % (self.baseFilename, self.gzip) if os.path.exists(sfn): if os.path.exists(dfn): os.remove(dfn) - os.rename(sfn, dfn) - dfn = self.baseFilename + ".1" - if os.path.exists(dfn): - os.remove(dfn) - os.rename(self.baseFilename, dfn) + _gzip(sfn, dfn) + + if self.gzip != 1: + # log.n -> log.n+1 + for i in range(nogzip - 1, 0, -1): + sfn = "%s.%d" % (self.baseFilename, i) + dfn = "%s.%d" % (self.baseFilename, i + 1) + if os.path.exists(sfn): + if os.path.exists(dfn): + os.remove(dfn) + os.rename(sfn, dfn) + # log -> log.1 + dfn = self.baseFilename + ".1" + if os.path.exists(dfn): + os.remove(dfn) + os.rename(self.baseFilename, dfn) self.mode = 'w' self.stream = self._open() @@ -158,12 +202,16 @@ If backupCount is > 0, when rollover is done, no more than backupCount files are kept - the oldest ones are deleted. + + If gzip is > 0, older log files are gziped counting from gzip-th newer + file to the oldest. """ - def __init__(self, filename, when='h', interval=1, backupCount=0, encoding=None, delay=False, utc=False): + def __init__(self, filename, when='h', interval=1, backupCount=0, encoding=None, delay=False, utc=False, gzip=0): BaseRotatingHandler.__init__(self, filename, 'a', encoding, delay) self.when = when.upper() self.backupCount = backupCount self.utc = utc + self.gzip = gzip # Calculate the real rollover interval, which is just the number of # seconds between rollovers. Also set the filename suffix used when # a rollover occurs. Current 'when' events supported: @@ -179,19 +227,19 @@ if self.when == 'S': self.interval = 1 # one second self.suffix = "%Y-%m-%d_%H-%M-%S" - self.extMatch = r"^\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}$" + self.extMatch = r"^\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}(\.gz){0,1}$" elif self.when == 'M': self.interval = 60 # one minute self.suffix = "%Y-%m-%d_%H-%M" - self.extMatch = r"^\d{4}-\d{2}-\d{2}_\d{2}-\d{2}$" + self.extMatch = r"^\d{4}-\d{2}-\d{2}_\d{2}-\d{2}(\.gz){0,1}$" elif self.when == 'H': self.interval = 60 * 60 # one hour self.suffix = "%Y-%m-%d_%H" - self.extMatch = r"^\d{4}-\d{2}-\d{2}_\d{2}$" + self.extMatch = r"^\d{4}-\d{2}-\d{2}_\d{2}(\.gz){0,1}$" elif self.when == 'D' or self.when == 'MIDNIGHT': self.interval = 60 * 60 * 24 # one day self.suffix = "%Y-%m-%d" - self.extMatch = r"^\d{4}-\d{2}-\d{2}$" + self.extMatch = r"^\d{4}-\d{2}-\d{2}(\.gz){0,1}$" elif self.when.startswith('W'): self.interval = 60 * 60 * 24 * 7 # one week if len(self.when) != 2: @@ -200,7 +248,7 @@ raise ValueError("Invalid day specified for weekly rollover: %s" % self.when) self.dayOfWeek = int(self.when[1]) self.suffix = "%Y-%m-%d" - self.extMatch = r"^\d{4}-\d{2}-\d{2}$" + self.extMatch = r"^\d{4}-\d{2}-\d{2}(\.gz){0,1}$" else: raise ValueError("Invalid rollover interval specified: %s" % self.when) @@ -283,11 +331,9 @@ return 1 return 0 - def getFilesToDelete(self): + def getFilesList(self): """ - Determine the files to delete when rolling over. - - More specific than the earlier method, which just used glob.glob(). + Return a list of log files. """ dirName, baseName = os.path.split(self.baseFilename) fileNames = os.listdir(dirName) @@ -300,12 +346,34 @@ if self.extMatch.match(suffix): result.append(os.path.join(dirName, fileName)) result.sort() + return result + + def getFilesToDelete(self): + """ + Determine the files to delete when rolling over. + + More specific than the earlier method, which just used glob.glob(). + """ + result = self.getFilesList() if len(result) < self.backupCount: result = [] else: result = result[:len(result) - self.backupCount] return result + def getFilesToGzip(self): + """ + Determine the files to compress using gzip. + """ + result = self.getFilesList() + result.reverse() + if len(result) < self.gzip: + result = [] + else: + result = result[self.gzip - 1:] + result = [f for f in result if not f.endswith(".gz")] + return result + def doRollover(self): """ do a rollover; in this case, a date/time stamp is appended to the filename @@ -330,6 +398,9 @@ if self.backupCount > 0: for s in self.getFilesToDelete(): os.remove(s) + if self.gzip > 0: + for s in self.getFilesToGzip(): + _gzip(s, s + ".gz") self.mode = 'w' self.stream = self._open() currentTime = int(time.time()) diff -r 0d1536ec44e8 Lib/test/test_logging.py --- a/Lib/test/test_logging.py Thu Dec 01 03:22:44 2011 +0100 +++ b/Lib/test/test_logging.py Thu Dec 01 17:57:51 2011 +0100 @@ -3588,13 +3588,15 @@ def test_rollover_filenames(self): rh = logging.handlers.RotatingFileHandler( - self.fn, backupCount=2, maxBytes=1) + self.fn, backupCount=3, maxBytes=1, gzip=3) rh.emit(self.next_rec()) self.assertLogFile(self.fn) rh.emit(self.next_rec()) self.assertLogFile(self.fn + ".1") rh.emit(self.next_rec()) self.assertLogFile(self.fn + ".2") + rh.emit(self.next_rec()) + self.assertLogFile(self.fn + ".3.gz") self.assertFalse(os.path.exists(self.fn + ".3")) rh.close() @@ -3602,12 +3604,15 @@ # other test methods added below def test_rollover(self): fh = logging.handlers.TimedRotatingFileHandler(self.fn, 'S', - backupCount=1) + backupCount=2, + gzip=2) r = logging.makeLogRecord({'msg': 'testing'}) fh.emit(r) self.assertLogFile(self.fn) time.sleep(1.01) # just a little over a second ... fh.emit(r) + time.sleep(1.01) + fh.emit(r) fh.close() # At this point, we should have a recent rotated file which we # can test for the existence of. However, in practice, on some @@ -3616,23 +3621,31 @@ # bit, and stop as soon as we see a rotated file. In theory this # could of course still fail, but the chances are lower. found = False + found_gz = False now = datetime.datetime.now() GO_BACK = 5 * 60 # seconds for secs in range(GO_BACK): prev = now - datetime.timedelta(seconds=secs) - fn = self.fn + prev.strftime(".%Y-%m-%d_%H-%M-%S") - found = os.path.exists(fn) + if not found: + fn = self.fn + prev.strftime(".%Y-%m-%d_%H-%M-%S") + found = os.path.exists(fn) + if found: + self.rmfiles.append(fn) + continue if found: - self.rmfiles.append(fn) - break + fn = self.fn + prev.strftime(".%Y-%m-%d_%H-%M-%S.gz") + found_gz = os.path.exists(fn) + if found_gz: + self.rmfiles.append(fn) + break msg = 'No rotated files found, went back %d seconds' % GO_BACK - if not found: + if not found or not found_gz: #print additional diagnostics dn, fn = os.path.split(self.fn) files = [f for f in os.listdir(dn) if f.startswith(fn)] print('Test time: %s' % now.strftime("%Y-%m-%d %H-%M-%S"), file=sys.stderr) print('The only matching files are: %s' % files, file=sys.stderr) - self.assertTrue(found, msg=msg) + self.assertTrue(found and found_gz, msg=msg) def test_invalid(self): assertRaises = self.assertRaises