[Jython-checkins] jython: Update pickle tests to latest and repatch for Jython
jim.baker
jython-checkins at python.org
Sun Apr 19 17:15:13 CEST 2015
https://hg.python.org/jython/rev/72a33dee5f95
changeset: 7674:72a33dee5f95
user: Jim Baker <jim.baker at rackspace.com>
date: Sun Apr 19 09:05:18 2015 -0600
summary:
Update pickle tests to latest and repatch for Jython
All pickle tests should eventually pass in a future release of
Jython. Most notably the tests that are currently failing are for
cPickle: exact float representations, dynamic classes, and list
pickling.
Part of the fix for http://bugs.jython.org/issue2323
files:
Lib/test/pickletester.py | 69 +++++-
Lib/test/test_cpickle.py | 51 ++++-
Lib/test/test_xpickle.py | 288 +++++++++++++++++++++++++++
3 files changed, 391 insertions(+), 17 deletions(-)
diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py
--- a/Lib/test/pickletester.py
+++ b/Lib/test/pickletester.py
@@ -6,7 +6,8 @@
import pickletools
import copy_reg
-from test.test_support import TestFailed, have_unicode, TESTFN, is_jython
+from test.test_support import (TestFailed, have_unicode, TESTFN, _2G, _1M,
+ precisionbigmemtest, is_jython)
# Tests that try a number of pickle protocols should have a
# for proto in protocols:
@@ -124,6 +125,21 @@
class use_metaclass(object):
__metaclass__ = metaclass
+class pickling_metaclass(type):
+ def __eq__(self, other):
+ return (type(self) == type(other) and
+ self.reduce_args == other.reduce_args)
+
+ def __reduce__(self):
+ return (create_dynamic_class, self.reduce_args)
+
+ __hash__ = None
+
+def create_dynamic_class(name, bases):
+ result = pickling_metaclass(name, bases, dict())
+ result.reduce_args = (name, bases)
+ return result
+
# DATA0 .. DATA2 are the pickles we expect under the various protocols, for
# the object returned by create_data().
@@ -487,10 +503,10 @@
i = C()
i.attr = i
for proto in protocols:
- s = self.dumps(i, 2)
+ s = self.dumps(i, proto)
x = self.loads(s)
self.assertEqual(dir(x), dir(i))
- self.assertTrue(x.attr is x)
+ self.assertIs(x.attr, x)
def test_recursive_multi(self):
l = []
@@ -574,8 +590,6 @@
self.assertEqual(n, got)
# Try a monster. This is quadratic-time in protos 0 & 1, so don't
# bother with those.
- if is_jython:#see http://jython.org/bugs/1754225
- return
nbase = long("deadbeeffeedface", 16)
nbase += nbase << 1000000
for n in nbase, -nbase:
@@ -583,7 +597,6 @@
got = self.loads(p)
self.assertEqual(n, got)
- @unittest.skip("FIXME: not working.")
def test_float(self):
test_values = [0.0, 4.94e-324, 1e-310, 7e-308, 6.626e-34, 0.1, 0.5,
3.14, 263.44582062374053, 6.022e23, 1e30]
@@ -612,6 +625,14 @@
b = self.loads(s)
self.assertEqual(a.__class__, b.__class__)
+ def test_dynamic_class(self):
+ a = create_dynamic_class("my_dynamic_class", (object,))
+ copy_reg.pickle(pickling_metaclass, pickling_metaclass.__reduce__)
+ for proto in protocols:
+ s = self.dumps(a, proto)
+ b = self.loads(s)
+ self.assertEqual(a, b)
+
def test_structseq(self):
import time
import os
@@ -906,7 +927,6 @@
y = self.loads(s)
self.assertEqual(y._proto, proto)
- @unittest.skip("FIXME: max recursion")
def test_reduce_calls_base(self):
for proto in protocols:
x = REX_five()
@@ -953,7 +973,7 @@
"Failed protocol %d: %r != %r"
% (proto, obj, loaded))
- @unittest.skip("FIXME: not working.")
+ @unittest.skipIf(is_jython, "FIXME: not working on Jython")
def test_attribute_name_interning(self):
# Test that attribute names of pickled objects are interned when
# unpickling.
@@ -1082,7 +1102,6 @@
# Of course this needs to be changed when HIGHEST_PROTOCOL changes.
self.assertEqual(self.module.HIGHEST_PROTOCOL, 2)
- @unittest.skip("FIXME: not working.")
def test_callapi(self):
f = cStringIO.StringIO()
# With and without keyword arguments
@@ -1093,12 +1112,11 @@
self.module.Pickler(f, -1)
self.module.Pickler(f, protocol=-1)
- @unittest.skip("FIXME: not working.")
def test_incomplete_input(self):
s = StringIO.StringIO("X''.")
self.assertRaises(EOFError, self.module.load, s)
- @unittest.skip("FIXME: not working.")
+ @unittest.skipIf(is_jython, "FIXME: not working on Jython, do similar patch for http://bugs.python.org/issue7128")
def test_restricted(self):
# issue7128: cPickle failed in restricted mode
builtins = {self.module.__name__: self.module,
@@ -1108,7 +1126,6 @@
exec teststr in {'__builtins__': builtins}, d
d['f']()
- @unittest.skip("FIXME: not working.")
def test_bad_input(self):
# Test issue4298
s = '\x58\0\0\0\x54'
@@ -1266,3 +1283,31 @@
f.write(pickled2)
f.seek(0)
self.assertEqual(unpickler.load(), data2)
+
+class BigmemPickleTests(unittest.TestCase):
+
+ # Memory requirements: 1 byte per character for input strings, 1 byte
+ # for pickled data, 1 byte for unpickled strings, 1 byte for internal
+ # buffer and 1 byte of free space for resizing of internal buffer.
+
+ @precisionbigmemtest(size=_2G + 100*_1M, memuse=5)
+ def test_huge_strlist(self, size):
+ chunksize = 2**20
+ data = []
+ while size > chunksize:
+ data.append('x' * chunksize)
+ size -= chunksize
+ chunksize += 1
+ data.append('y' * size)
+
+ try:
+ for proto in protocols:
+ try:
+ pickled = self.dumps(data, proto)
+ res = self.loads(pickled)
+ self.assertEqual(res, data)
+ finally:
+ res = None
+ pickled = None
+ finally:
+ data = None
diff --git a/Lib/test/test_cpickle.py b/Lib/test/test_cpickle.py
--- a/Lib/test/test_cpickle.py
+++ b/Lib/test/test_cpickle.py
@@ -4,7 +4,27 @@
from test.pickletester import AbstractPickleTests, AbstractPickleModuleTests
from test import test_support
-class cPickleTests(AbstractPickleTests, AbstractPickleModuleTests):
+
+class ApproxFloat(unittest.TestCase):
+ # FIXME for Jython: remove this class - and its use from bases in
+ # subsequent test classes - when we can guarantee that floats that
+ # are pickled by cPickle are exact in the same way they are on
+ # CPython
+
+ def test_float(self):
+ from test.pickletester import protocols
+
+ test_values = [0.0, 4.94e-324, 1e-310, 7e-308, 6.626e-34, 0.1, 0.5,
+ 3.14, 263.44582062374053, 6.022e23, 1e30]
+ test_values = test_values + [-x for x in test_values]
+ for proto in protocols:
+ for value in test_values:
+ pickle = self.dumps(value, proto)
+ got = self.loads(pickle)
+ self.assertAlmostEqual(value, got)
+
+
+class cPickleTests(ApproxFloat, AbstractPickleTests, AbstractPickleModuleTests):
def setUp(self):
self.dumps = cPickle.dumps
@@ -13,7 +33,16 @@
error = cPickle.BadPickleGet
module = cPickle
-class cPicklePicklerTests(AbstractPickleTests):
+ @unittest.skipIf(test_support.is_jython, "FIXME: not working on Jython")
+ def test_callapi(self):
+ pass
+
+ @unittest.skipIf(test_support.is_jython, "FIXME: not working on Jython")
+ def test_dynamic_class(self):
+ pass
+
+
+class cPicklePicklerTests(ApproxFloat, AbstractPickleTests):
def dumps(self, arg, proto=0):
f = StringIO()
@@ -29,6 +58,11 @@
error = cPickle.BadPickleGet
+ @unittest.skipIf(test_support.is_jython, "FIXME: not working on Jython")
+ def test_dynamic_class(self):
+ pass
+
+
class cPickleListPicklerTests(AbstractPickleTests):
def dumps(self, arg, proto=0):
@@ -43,7 +77,7 @@
error = cPickle.BadPickleGet
-class cPickleFastPicklerTests(AbstractPickleTests):
+class cPickleFastPicklerTests(ApproxFloat, AbstractPickleTests):
def dumps(self, arg, proto=0):
f = StringIO()
@@ -91,6 +125,11 @@
b = self.loads(self.dumps(a))
self.assertEqual(a, b)
+ @unittest.skipIf(test_support.is_jython, "FIXME: not working on Jython")
+ def test_dynamic_class(self):
+ pass
+
+
def test_main():
tests = [
cPickleTests,
@@ -99,13 +138,15 @@
cPickleFastPicklerTests
]
if test_support.is_jython:
- # XXX: Jython doesn't support list based picklers
+ # FIXME Jython currently doesn't support list based picklers
tests.remove(cPickleListPicklerTests)
- # XXX: These don't cause exceptions on Jython
+ # FIXME these cause NullPointerException on Jython
del cPickleFastPicklerTests.test_recursive_list
del cPickleFastPicklerTests.test_recursive_inst
del cPickleFastPicklerTests.test_recursive_dict
del cPickleFastPicklerTests.test_recursive_multi
+
+
test_support.run_unittest(*tests)
if __name__ == "__main__":
diff --git a/Lib/test/test_xpickle.py b/Lib/test/test_xpickle.py
new file mode 100644
--- /dev/null
+++ b/Lib/test/test_xpickle.py
@@ -0,0 +1,288 @@
+# test_pickle dumps and loads pickles via pickle.py.
+# test_cpickle does the same, but via the cPickle module.
+# This test covers the other two cases, making pickles with one module and
+# loading them via the other. It also tests backwards compatibility with
+# previous version of Python by bouncing pickled objects through Python 2.4
+# and Python 2.5 running this file.
+
+import cPickle
+import os
+import os.path
+import pickle
+import subprocess
+import sys
+import types
+import unittest
+
+from test import test_support
+
+# Most distro-supplied Pythons don't include the tests
+# or test support files, and some don't include a way to get these back even if
+# you're will to install extra packages (like Ubuntu). Doing things like this
+# "provides" a pickletester module for older versions of Python that may be
+# installed without it. Note that one other design for this involves messing
+# with sys.path, which is less precise.
+mod_path = os.path.abspath(os.path.join(os.path.dirname(__file__),
+ "pickletester.py"))
+pickletester = types.ModuleType("test.pickletester")
+exec compile(open(mod_path).read(), mod_path, 'exec') in pickletester.__dict__
+AbstractPickleTests = pickletester.AbstractPickleTests
+if pickletester.__name__ in sys.modules:
+ raise RuntimeError("Did not expect to find test.pickletester loaded")
+sys.modules[pickletester.__name__] = pickletester
+
+
+class ApproxFloat(unittest.TestCase):
+ # FIXME for Jython: remove this class - and its use from bases in
+ # subsequent test classes - when we can guarantee that floats that
+ # are pickled by cPickle are exact in the same way they are on
+ # CPython
+
+ def test_float(self):
+ from test.pickletester import protocols
+
+ test_values = [0.0, 4.94e-324, 1e-310, 7e-308, 6.626e-34, 0.1, 0.5,
+ 3.14, 263.44582062374053, 6.022e23, 1e30]
+ test_values = test_values + [-x for x in test_values]
+ for proto in protocols:
+ for value in test_values:
+ pickle = self.dumps(value, proto)
+ got = self.loads(pickle)
+ self.assertAlmostEqual(value, got)
+
+
+class DumpCPickle_LoadPickle(ApproxFloat, AbstractPickleTests):
+
+ error = KeyError
+
+ def dumps(self, arg, proto=0, fast=False):
+ # Ignore fast
+ return cPickle.dumps(arg, proto)
+
+ def loads(self, buf):
+ # Ignore fast
+ return pickle.loads(buf)
+
+ @unittest.skipIf(test_support.is_jython, "FIXME: not working on Jython")
+ def test_dynamic_class(self):
+ pass
+
+
+class DumpPickle_LoadCPickle(AbstractPickleTests):
+
+ error = cPickle.BadPickleGet
+
+ def dumps(self, arg, proto=0, fast=False):
+ # Ignore fast
+ return pickle.dumps(arg, proto)
+
+ def loads(self, buf):
+ # Ignore fast
+ return cPickle.loads(buf)
+
+def have_python_version(name):
+ """Check whether the given name is a valid Python binary and has
+ test.test_support.
+
+ This respects your PATH.
+
+ Args:
+ name: short string name of a Python binary such as "python2.4".
+
+ Returns:
+ True if the name is valid, False otherwise.
+ """
+ return os.system(name + " -c 'import test.test_support'") == 0
+
+
+class AbstractCompatTests(AbstractPickleTests):
+
+ module = None
+ python = None
+ error = None
+
+ def setUp(self):
+ self.assertTrue(self.python)
+ self.assertTrue(self.module)
+ self.assertTrue(self.error)
+
+ def send_to_worker(self, python, obj, proto):
+ """Bounce a pickled object through another version of Python.
+
+ This will pickle the object, send it to a child process where it will be
+ unpickled, then repickled and sent back to the parent process.
+
+ Args:
+ python: the name of the Python binary to start.
+ obj: object to pickle.
+ proto: pickle protocol number to use.
+
+ Returns:
+ The pickled data received from the child process.
+ """
+ # Prevent the subprocess from picking up invalid .pyc files.
+ target = __file__
+ if target[-1] in ("c", "o"):
+ target = target[:-1]
+
+ data = self.module.dumps((proto, obj), proto)
+ worker = subprocess.Popen([python, target, "worker"],
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ stdout, stderr = worker.communicate(data)
+ if worker.returncode != 0:
+ raise RuntimeError(stderr)
+ return stdout
+
+ def dumps(self, arg, proto=0, fast=False):
+ return self.send_to_worker(self.python, arg, proto)
+
+ def loads(self, input):
+ return self.module.loads(input)
+
+ # These tests are disabled because they require some special setup
+ # on the worker that's hard to keep in sync.
+ def test_global_ext1(self):
+ pass
+
+ def test_global_ext2(self):
+ pass
+
+ def test_global_ext4(self):
+ pass
+
+ # This is a cut-down version of pickletester's test_float. Backwards
+ # compatibility for the values in for_bin_protos was explicitly broken in
+ # r68903 to fix a bug.
+ def test_float(self):
+ for_bin_protos = [4.94e-324, 1e-310]
+ neg_for_bin_protos = [-x for x in for_bin_protos]
+ test_values = [0.0, 7e-308, 6.626e-34, 0.1, 0.5,
+ 3.14, 263.44582062374053, 6.022e23, 1e30]
+ test_proto0_values = test_values + [-x for x in test_values]
+ test_values = test_proto0_values + for_bin_protos + neg_for_bin_protos
+
+ for value in test_proto0_values:
+ pickle = self.dumps(value, 0)
+ got = self.loads(pickle)
+ self.assertEqual(value, got)
+
+ for proto in pickletester.protocols[1:]:
+ for value in test_values:
+ pickle = self.dumps(value, proto)
+ got = self.loads(pickle)
+ self.assertEqual(value, got)
+
+ # Backwards compatibility was explicitly broken in r67934 to fix a bug.
+ def test_unicode_high_plane(self):
+ pass
+
+ # This tests a fix that's in 2.7 only
+ def test_dynamic_class(self):
+ pass
+
+ if test_support.have_unicode:
+ # This is a cut-down version of pickletester's test_unicode. Backwards
+ # compatibility was explicitly broken in r67934 to fix a bug.
+ def test_unicode(self):
+ endcases = [u'', u'<\\u>', u'<\\\u1234>', u'<\n>', u'<\\>']
+ for proto in pickletester.protocols:
+ for u in endcases:
+ p = self.dumps(u, proto)
+ u2 = self.loads(p)
+ self.assertEqual(u2, u)
+
+
+def run_compat_test(python_name):
+ return (test_support.is_resource_enabled("xpickle") and
+ have_python_version(python_name))
+
+
+# Test backwards compatibility with Python 2.4.
+if not run_compat_test("python2.4"):
+ class CPicklePython24Compat(unittest.TestCase):
+ pass
+else:
+ class CPicklePython24Compat(AbstractCompatTests):
+
+ module = cPickle
+ python = "python2.4"
+ error = cPickle.BadPickleGet
+
+ # Disable these tests for Python 2.4. Making them pass would require
+ # nontrivially monkeypatching the pickletester module in the worker.
+ def test_reduce_calls_base(self):
+ pass
+
+ def test_reduce_ex_calls_base(self):
+ pass
+
+class PicklePython24Compat(CPicklePython24Compat):
+
+ module = pickle
+ error = KeyError
+
+
+# Test backwards compatibility with Python 2.5.
+if not run_compat_test("python2.5"):
+ class CPicklePython25Compat(unittest.TestCase):
+ pass
+else:
+ class CPicklePython25Compat(AbstractCompatTests):
+
+ module = cPickle
+ python = "python2.5"
+ error = cPickle.BadPickleGet
+
+class PicklePython25Compat(CPicklePython25Compat):
+
+ module = pickle
+ error = KeyError
+
+
+# Test backwards compatibility with Python 2.6.
+if not run_compat_test("python2.6"):
+ class CPicklePython26Compat(unittest.TestCase):
+ pass
+else:
+ class CPicklePython26Compat(AbstractCompatTests):
+
+ module = cPickle
+ python = "python2.6"
+ error = cPickle.BadPickleGet
+
+class PicklePython26Compat(CPicklePython26Compat):
+
+ module = pickle
+ error = KeyError
+
+
+def worker_main(in_stream, out_stream):
+ message = cPickle.load(in_stream)
+ protocol, obj = message
+ cPickle.dump(obj, out_stream, protocol)
+
+
+def test_main():
+ if not test_support.is_resource_enabled("xpickle"):
+ print >>sys.stderr, "test_xpickle -- skipping backwards compat tests."
+ print >>sys.stderr, "Use 'regrtest.py -u xpickle' to run them."
+ sys.stderr.flush()
+
+ test_support.run_unittest(
+ DumpCPickle_LoadPickle,
+ DumpPickle_LoadCPickle,
+ CPicklePython24Compat,
+ CPicklePython25Compat,
+ CPicklePython26Compat,
+ PicklePython24Compat,
+ PicklePython25Compat,
+ PicklePython26Compat,
+ )
+
+if __name__ == "__main__":
+ if "worker" in sys.argv:
+ worker_main(sys.stdin, sys.stdout)
+ else:
+ test_main()
--
Repository URL: https://hg.python.org/jython
More information about the Jython-checkins
mailing list