[Python-checkins] r60719 - in python/trunk: Doc/library/pickletools.rst Lib/pickletools.py Misc/NEWS

[raymond.hettinger]

Author: raymond.hettinger
Date: Sun Feb 10 21:35:16 2008
New Revision: 60719

Modified:
   python/trunk/Doc/library/pickletools.rst
   python/trunk/Lib/pickletools.py
   python/trunk/Misc/NEWS
Log:
Complete an open todo on pickletools -- add a pickle optimizer.

Modified: python/trunk/Doc/library/pickletools.rst
==============================================================================
--- python/trunk/Doc/library/pickletools.rst	(original)
+++ python/trunk/Doc/library/pickletools.rst	Sun Feb 10 21:35:16 2008
@@ -35,3 +35,10 @@
    the opcode's argument; *pos* is the position at which this opcode is located.
    *pickle* can be a string or a file-like object.
 
+.. function:: optimize(picklestring)
+
+   Returns a new equivalent pickle string after eliminating unused ``PUT``
+   opcodes. The optimized pickle is shorter, takes less transmission time,
+   requires less storage space, and unpickles more efficiently.
+
+   .. versionadded:: 2.6

Modified: python/trunk/Lib/pickletools.py
==============================================================================
--- python/trunk/Lib/pickletools.py	(original)
+++ python/trunk/Lib/pickletools.py	Sun Feb 10 21:35:16 2008
@@ -10,9 +10,7 @@
    Print a symbolic disassembly of a pickle.
 '''
 
-__all__ = ['dis',
-           'genops',
-          ]
+__all__ = ['dis', 'genops', 'optimize']
 
 # Other ideas:
 #
@@ -1858,6 +1856,33 @@
             break
 
 ##############################################################################
+# A pickle optimizer.
+
+def optimize(p):
+    'Optimize a pickle string by removing unused PUT opcodes'
+    gets = set()            # set of args used by a GET opcode
+    puts = []               # (arg, startpos, stoppos) for the PUT opcodes
+    prevpos = None          # set to pos if previous opcode was a PUT
+    for opcode, arg, pos in genops(p):
+        if prevpos is not None:
+            puts.append((prevarg, prevpos, pos))
+            prevpos = None
+        if 'PUT' in opcode.name:
+            prevarg, prevpos = arg, pos
+        elif 'GET' in opcode.name:
+            gets.add(arg)
+
+    # Copy the pickle string except for PUTS without a corresponding GET
+    s = []
+    i = 0
+    for arg, start, stop in puts:
+        j = stop if (arg in gets) else start
+        s.append(p[i:j])
+        i = stop
+    s.append(p[i:])
+    return ''.join(s)
+
+##############################################################################
 # A symbolic pickle disassembler.
 
 def dis(pickle, out=None, memo=None, indentlevel=4):

Modified: python/trunk/Misc/NEWS
==============================================================================
--- python/trunk/Misc/NEWS	(original)
+++ python/trunk/Misc/NEWS	Sun Feb 10 21:35:16 2008
@@ -400,6 +400,9 @@
 Library
 -------
 
+- The pickletools module now provides an optimize() function
+  that eliminates unused PUT opcodes from a pickle string.
+
 - #2021: Allow tempfile.NamedTemporaryFile and SpooledTemporaryFile
   to be used in with statements by correctly supporting the context
   management protocol.