[Python-checkins] cpython: Refactored logging rotating handlers for improved flexibility.
vinay.sajip
python-checkins at python.org
Wed Jan 4 13:02:42 CET 2012
http://hg.python.org/cpython/rev/57295c4d81ac
changeset: 74267:57295c4d81ac
user: Vinay Sajip <vinay_sajip at yahoo.co.uk>
date: Wed Jan 04 12:02:26 2012 +0000
summary:
Refactored logging rotating handlers for improved flexibility.
files:
Doc/howto/logging-cookbook.rst | 28 ++++++
Doc/library/logging.handlers.rst | 81 ++++++++++++++++++++
Lib/logging/handlers.py | 70 +++++++++++++---
Lib/test/test_logging.py | 53 ++++++++++++-
4 files changed, 215 insertions(+), 17 deletions(-)
diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst
--- a/Doc/howto/logging-cookbook.rst
+++ b/Doc/howto/logging-cookbook.rst
@@ -1102,3 +1102,31 @@
For more information about this configuration, you can see the `relevant
section <https://docs.djangoproject.com/en/1.3/topics/logging/#configuring-logging>`_
of the Django documentation.
+
+.. _cookbook-rotator-namer:
+
+Using a rotator and namer to customise log rotation processing
+--------------------------------------------------------------
+
+An example of how you can define a namer and rotator is given in the following
+snippet, which shows zlib-based compression of the log file::
+
+ def namer(name):
+ return name + ".gz"
+
+ def rotator(source, dest):
+ with open(source, "rb") as sf:
+ data = sf.read()
+ compressed = zlib.compress(data, 9)
+ with open(dest, "wb") as df:
+ df.write(compressed)
+ os.remove(source)
+
+ rh = logging.handlers.RotatingFileHandler(...)
+ rh.rotator = rotator
+ rh.namer = namer
+
+These are not “true” .gz files, as they are bare compressed data, with no
+“container” such as you’d find in an actual gzip file. This snippet is just
+for illustration purposes.
+
diff --git a/Doc/library/logging.handlers.rst b/Doc/library/logging.handlers.rst
--- a/Doc/library/logging.handlers.rst
+++ b/Doc/library/logging.handlers.rst
@@ -164,6 +164,87 @@
changed. If it has, the existing stream is flushed and closed and the
file opened again, before outputting the record to the file.
+.. _base-rotating-handler:
+
+BaseRotatingHandler
+^^^^^^^^^^^^^^^^^^^
+
+The :class:`BaseRotatingHandler` class, located in the :mod:`logging.handlers`
+module, is the base class for the rotating file handlers,
+:class:`RotatingFileHandler` and :class:`TimedRotatingFileHandler`. You should
+not need to instantiate this class, but it has attributes and methods you may
+need to override.
+
+.. class:: BaseRotatingHandler(filename, mode, encoding=None, delay=False)
+
+ The parameters are as for :class:`FileHandler`. The attributes are:
+
+ .. attribute:: namer
+
+ If this attribute is set to a callable, the :meth:`rotation_filename`
+ method delegates to this callable. The parameters passed to the callable
+ are those passed to :meth:`rotation_filename`.
+
+ .. note:: The namer function is called quite a few times during rollover,
+ so it should be as simple and as fast as possible. It should also
+ return the same output every time for a given input, otherwise the
+ rollover behaviour may not work as expected.
+
+ .. versionadded:: 3.3
+
+
+ .. attribute:: BaseRotatingHandler.rotator
+
+ If this attribute is set to a callable, the :meth:`rotate` method
+ delegates to this callable. The parameters passed to the callable are
+ those passed to :meth:`rotate`.
+
+ .. versionadded:: 3.3
+
+ .. method:: BaseRotatingHandler.rotation_filename(default_name)
+
+ Modify the filename of a log file when rotating.
+
+ This is provided so that a custom filename can be provided.
+
+ The default implementation calls the 'namer' attribute of the handler,
+ if it's callable, passing the default name to it. If the attribute isn't
+ callable (the default is `None`), the name is returned unchanged.
+
+ :param default_name: The default name for the log file.
+
+ .. versionadded:: 3.3
+
+
+ .. method:: BaseRotatingHandler.rotate(source, dest)
+
+ When rotating, rotate the current log.
+
+ The default implementation calls the 'rotator' attribute of the handler,
+ if it's callable, passing the source and dest arguments to it. If the
+ attribute isn't callable (the default is `None`), the source is simply
+ renamed to the destination.
+
+ :param source: The source filename. This is normally the base
+ filename, e.g. 'test.log'
+ :param dest: The destination filename. This is normally
+ what the source is rotated to, e.g. 'test.log.1'.
+
+ .. versionadded:: 3.3
+
+The reason the attributes exist is to save you having to subclass - you can use
+the same callables for instances of :class:`RotatingFileHandler` and
+:class:`TimedRotatingFileHandler`. If either the namer or rotator callable
+raises an exception, this will be handled in the same way as any other
+exception during an :meth:`emit` call, i.e. via the :meth:`handleError` method
+of the handler.
+
+If you need to make more significant changes to rotation processing, you can
+override the methods.
+
+For an example, see :ref:`cookbook-rotator-namer`.
+
+
.. _rotating-file-handler:
RotatingFileHandler
diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py
--- a/Lib/logging/handlers.py
+++ b/Lib/logging/handlers.py
@@ -52,13 +52,15 @@
Not meant to be instantiated directly. Instead, use RotatingFileHandler
or TimedRotatingFileHandler.
"""
- def __init__(self, filename, mode, encoding=None, delay=0):
+ def __init__(self, filename, mode, encoding=None, delay=False):
"""
Use the specified filename for streamed logging
"""
logging.FileHandler.__init__(self, filename, mode, encoding, delay)
self.mode = mode
self.encoding = encoding
+ self.namer = None
+ self.rotator = None
def emit(self, record):
"""
@@ -76,12 +78,50 @@
except:
self.handleError(record)
+ def rotation_filename(self, default_name):
+ """
+ Modify the filename of a log file when rotating.
+
+ This is provided so that a custom filename can be provided.
+
+ The default implementation calls the 'namer' attribute of the
+ handler, if it's callable, passing the default name to
+ it. If the attribute isn't callable (the default is None), the name
+ is returned unchanged.
+
+ :param default_name: The default name for the log file.
+ """
+ if not callable(self.namer):
+ result = default_name
+ else:
+ result = self.namer(default_name)
+ return result
+
+ def rotate(self, source, dest):
+ """
+ When rotating, rotate the current log.
+
+ The default implementation calls the 'rotator' attribute of the
+ handler, if it's callable, passing the source and dest arguments to
+ it. If the attribute isn't callable (the default is None), the source
+ is simply renamed to the destination.
+
+ :param source: The source filename. This is normally the base
+ filename, e.g. 'test.log'
+ :param dest: The destination filename. This is normally
+ what the source is rotated to, e.g. 'test.log.1'.
+ """
+ if not callable(self.rotator):
+ os.rename(source, dest)
+ else:
+ self.rotator(source, dest)
+
class RotatingFileHandler(BaseRotatingHandler):
"""
Handler for logging to a set of files, which switches from one file
to the next when the current file reaches a certain size.
"""
- def __init__(self, filename, mode='a', maxBytes=0, backupCount=0, encoding=None, delay=0):
+ def __init__(self, filename, mode='a', maxBytes=0, backupCount=0, encoding=None, delay=False):
"""
Open the specified file and use it as the stream for logging.
@@ -122,16 +162,17 @@
self.stream = None
if self.backupCount > 0:
for i in range(self.backupCount - 1, 0, -1):
- sfn = "%s.%d" % (self.baseFilename, i)
- dfn = "%s.%d" % (self.baseFilename, i + 1)
+ sfn = self.rotation_filename("%s.%d" % (self.baseFilename, i))
+ dfn = self.rotation_filename("%s.%d" % (self.baseFilename,
+ i + 1))
if os.path.exists(sfn):
if os.path.exists(dfn):
os.remove(dfn)
os.rename(sfn, dfn)
- dfn = self.baseFilename + ".1"
+ dfn = self.rotation_filename(self.baseFilename + ".1")
if os.path.exists(dfn):
os.remove(dfn)
- os.rename(self.baseFilename, dfn)
+ self.rotate(self.baseFilename, dfn)
self.mode = 'w'
self.stream = self._open()
@@ -179,19 +220,19 @@
if self.when == 'S':
self.interval = 1 # one second
self.suffix = "%Y-%m-%d_%H-%M-%S"
- self.extMatch = r"^\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}$"
+ self.extMatch = r"^\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}(\.\w+)?$"
elif self.when == 'M':
self.interval = 60 # one minute
self.suffix = "%Y-%m-%d_%H-%M"
- self.extMatch = r"^\d{4}-\d{2}-\d{2}_\d{2}-\d{2}$"
+ self.extMatch = r"^\d{4}-\d{2}-\d{2}_\d{2}-\d{2}(\.\w+)?$"
elif self.when == 'H':
self.interval = 60 * 60 # one hour
self.suffix = "%Y-%m-%d_%H"
- self.extMatch = r"^\d{4}-\d{2}-\d{2}_\d{2}$"
+ self.extMatch = r"^\d{4}-\d{2}-\d{2}_\d{2}(\.\w+)?$"
elif self.when == 'D' or self.when == 'MIDNIGHT':
self.interval = 60 * 60 * 24 # one day
self.suffix = "%Y-%m-%d"
- self.extMatch = r"^\d{4}-\d{2}-\d{2}$"
+ self.extMatch = r"^\d{4}-\d{2}-\d{2}(\.\w+)?$"
elif self.when.startswith('W'):
self.interval = 60 * 60 * 24 * 7 # one week
if len(self.when) != 2:
@@ -200,7 +241,7 @@
raise ValueError("Invalid day specified for weekly rollover: %s" % self.when)
self.dayOfWeek = int(self.when[1])
self.suffix = "%Y-%m-%d"
- self.extMatch = r"^\d{4}-\d{2}-\d{2}$"
+ self.extMatch = r"^\d{4}-\d{2}-\d{2}(\.\w+)?$"
else:
raise ValueError("Invalid rollover interval specified: %s" % self.when)
@@ -323,10 +364,11 @@
timeTuple = time.gmtime(t)
else:
timeTuple = time.localtime(t)
- dfn = self.baseFilename + "." + time.strftime(self.suffix, timeTuple)
+ dfn = self.rotation_filename(self.baseFilename + "." +
+ time.strftime(self.suffix, timeTuple))
if os.path.exists(dfn):
os.remove(dfn)
- os.rename(self.baseFilename, dfn)
+ self.rotate(self.baseFilename, dfn)
if self.backupCount > 0:
for s in self.getFilesToDelete():
os.remove(s)
@@ -367,7 +409,7 @@
This handler is based on a suggestion and patch by Chad J.
Schroeder.
"""
- def __init__(self, filename, mode='a', encoding=None, delay=0):
+ def __init__(self, filename, mode='a', encoding=None, delay=False):
logging.FileHandler.__init__(self, filename, mode, encoding, delay)
if not os.path.exists(self.baseFilename):
self.dev, self.ino = -1, -1
diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py
--- a/Lib/test/test_logging.py
+++ b/Lib/test/test_logging.py
@@ -46,6 +46,7 @@
import unittest
import warnings
import weakref
+import zlib
try:
import threading
# The following imports are needed only for tests which
@@ -3587,15 +3588,61 @@
rh.close()
def test_rollover_filenames(self):
+ def namer(name):
+ return name + ".test"
rh = logging.handlers.RotatingFileHandler(
self.fn, backupCount=2, maxBytes=1)
+ rh.namer = namer
rh.emit(self.next_rec())
self.assertLogFile(self.fn)
rh.emit(self.next_rec())
- self.assertLogFile(self.fn + ".1")
+ self.assertLogFile(namer(self.fn + ".1"))
rh.emit(self.next_rec())
- self.assertLogFile(self.fn + ".2")
- self.assertFalse(os.path.exists(self.fn + ".3"))
+ self.assertLogFile(namer(self.fn + ".2"))
+ self.assertFalse(os.path.exists(namer(self.fn + ".3")))
+ rh.close()
+
+ def test_rotator(self):
+ def namer(name):
+ return name + ".gz"
+
+ def rotator(source, dest):
+ with open(source, "rb") as sf:
+ data = sf.read()
+ compressed = zlib.compress(data, 9)
+ with open(dest, "wb") as df:
+ df.write(compressed)
+ os.remove(source)
+
+ rh = logging.handlers.RotatingFileHandler(
+ self.fn, backupCount=2, maxBytes=1)
+ rh.rotator = rotator
+ rh.namer = namer
+ m1 = self.next_rec()
+ rh.emit(m1)
+ self.assertLogFile(self.fn)
+ m2 = self.next_rec()
+ rh.emit(m2)
+ fn = namer(self.fn + ".1")
+ self.assertLogFile(fn)
+ with open(fn, "rb") as f:
+ compressed = f.read()
+ data = zlib.decompress(compressed)
+ self.assertEqual(data.decode("ascii"), m1.msg + "\n")
+ rh.emit(self.next_rec())
+ fn = namer(self.fn + ".2")
+ self.assertLogFile(fn)
+ with open(fn, "rb") as f:
+ compressed = f.read()
+ data = zlib.decompress(compressed)
+ self.assertEqual(data.decode("ascii"), m1.msg + "\n")
+ rh.emit(self.next_rec())
+ fn = namer(self.fn + ".2")
+ with open(fn, "rb") as f:
+ compressed = f.read()
+ data = zlib.decompress(compressed)
+ self.assertEqual(data.decode("ascii"), m2.msg + "\n")
+ self.assertFalse(os.path.exists(namer(self.fn + ".3")))
rh.close()
class TimedRotatingFileHandlerTest(BaseFileTest):
--
Repository URL: http://hg.python.org/cpython
More information about the Python-checkins
mailing list