[pypy-commit] pypy py3k-qualname: merge py3.3

pjenvey noreply at buildbot.pypy.org
Fri Aug 8 02:20:53 CEST 2014


Author: Philip Jenvey <pjenvey at underboss.org>
Branch: py3k-qualname
Changeset: r72715:525ebcd3c24a
Date: 2014-08-05 17:36 -0700
http://bitbucket.org/pypy/pypy/changeset/525ebcd3c24a/

Log:	merge py3.3

diff too long, truncating to 2000 out of 4104 lines

diff --git a/lib-python/3/distutils/command/build_ext.py b/lib-python/3/distutils/command/build_ext.py
--- a/lib-python/3/distutils/command/build_ext.py
+++ b/lib-python/3/distutils/command/build_ext.py
@@ -4,7 +4,8 @@
 modules (currently limited to C extensions, should accommodate C++
 extensions ASAP)."""
 
-import sys, os, re, imp
+import sys, os, re
+import importlib.machinery
 from distutils.core import Command
 from distutils.errors import *
 from distutils.sysconfig import customize_compiler, get_python_version
@@ -36,9 +37,8 @@
     show_compilers()
 
 def _get_c_extension_suffix():
-    for ext, mod, typ in imp.get_suffixes():
-        if typ == imp.C_EXTENSION:
-            return ext
+    suffixes = importlib.machinery.EXTENSION_SUFFIXES
+    return suffixes[0] if suffixes else None
 
 
 class build_ext(Command):
diff --git a/lib-python/3/test/test_audioop.py b/lib-python/3/test/test_audioop.py
--- a/lib-python/3/test/test_audioop.py
+++ b/lib-python/3/test/test_audioop.py
@@ -1,6 +1,7 @@
 import audioop
 import sys
 import unittest
+from test.support import run_unittest, impl_detail
 
 def pack(width, data):
     return b''.join(v.to_bytes(width, sys.byteorder, signed=True) for v in data)
@@ -170,6 +171,7 @@
         self.assertEqual(audioop.lin2lin(datas[4], 4, 2),
             packs[2](0, 0x1234, 0x4567, -0x4568, 0x7fff, -0x8000, -1))
 
+    @impl_detail(pypy=False)
     def test_adpcm2lin(self):
         self.assertEqual(audioop.adpcm2lin(b'\x07\x7f\x7f', 1, None),
                          (b'\x00\x00\x00\xff\x00\xff', (-179, 40)))
@@ -184,6 +186,7 @@
             self.assertEqual(audioop.adpcm2lin(b'\0' * 5, w, None),
                              (b'\0' * w * 10, (0, 0)))
 
+    @impl_detail(pypy=False)
     def test_lin2adpcm(self):
         self.assertEqual(audioop.lin2adpcm(datas[1], 1, None),
                          (b'\x07\x7f\x7f', (-221, 39)))
@@ -197,6 +200,7 @@
             self.assertEqual(audioop.lin2adpcm(b'\0' * w * 10, w, None),
                              (b'\0' * 5, (0, 0)))
 
+    @impl_detail(pypy=False)
     def test_lin2alaw(self):
         self.assertEqual(audioop.lin2alaw(datas[1], 1),
                          b'\xd5\x87\xa4\x24\xaa\x2a\x5a')
@@ -205,6 +209,7 @@
         self.assertEqual(audioop.lin2alaw(datas[4], 4),
                          b'\xd5\x87\xa4\x24\xaa\x2a\x55')
 
+    @impl_detail(pypy=False)
     def test_alaw2lin(self):
         encoded = b'\x00\x03\x24\x2a\x51\x54\x55\x58\x6b\x71\x7f'\
                   b'\x80\x83\xa4\xaa\xd1\xd4\xd5\xd8\xeb\xf1\xff'
@@ -219,6 +224,7 @@
             decoded = audioop.alaw2lin(encoded, w)
             self.assertEqual(audioop.lin2alaw(decoded, w), encoded)
 
+    @impl_detail(pypy=False)
     def test_lin2ulaw(self):
         self.assertEqual(audioop.lin2ulaw(datas[1], 1),
                          b'\xff\xad\x8e\x0e\x80\x00\x67')
@@ -227,6 +233,7 @@
         self.assertEqual(audioop.lin2ulaw(datas[4], 4),
                          b'\xff\xad\x8e\x0e\x80\x00\x7e')
 
+    @impl_detail(pypy=False)
     def test_ulaw2lin(self):
         encoded = b'\x00\x0e\x28\x3f\x57\x6a\x76\x7c\x7e\x7f'\
                   b'\x80\x8e\xa8\xbf\xd7\xea\xf6\xfc\xfe\xff'
@@ -341,6 +348,7 @@
         self.assertRaises(audioop.error,
             audioop.findmax, bytes(range(256)), -2392392)
 
+    @impl_detail(pypy=False)
     def test_issue7673(self):
         state = None
         for data, size in INVALID_DATA:
@@ -365,6 +373,7 @@
             self.assertRaises(audioop.error, audioop.lin2alaw, data, size)
             self.assertRaises(audioop.error, audioop.lin2adpcm, data, size, state)
 
+    @impl_detail(pypy=False)
     def test_wrongsize(self):
         data = b'abcdefgh'
         state = None
diff --git a/lib-python/3/test/test_builtin.py b/lib-python/3/test/test_builtin.py
--- a/lib-python/3/test/test_builtin.py
+++ b/lib-python/3/test/test_builtin.py
@@ -15,7 +15,8 @@
 import unittest
 import warnings
 from operator import neg
-from test.support import TESTFN, unlink,  run_unittest, check_warnings
+from test.support import (
+    TESTFN, unlink,  run_unittest, check_warnings, check_impl_detail)
 try:
     import pty, signal
 except ImportError:
@@ -558,18 +559,21 @@
         self.assertEqual((g, l), ({'a': 1}, {'b': 2}))
 
     def test_exec_globals(self):
-        code = compile("print('Hello World!')", "", "exec")
-        # no builtin function
-        self.assertRaisesRegex(NameError, "name 'print' is not defined",
-                               exec, code, {'__builtins__': {}})
-        # __builtins__ must be a mapping type
-        self.assertRaises(TypeError,
-                          exec, code, {'__builtins__': 123})
+        if check_impl_detail():
+            # strict __builtins__ compliance (CPython)
+            code = compile("print('Hello World!')", "", "exec")
+            # no builtin function
+            self.assertRaisesRegex(NameError, "name 'print' is not defined",
+                                   exec, code, {'__builtins__': {}})
+            # __builtins__ must be a mapping type
+            self.assertRaises(TypeError,
+                              exec, code, {'__builtins__': 123})
 
-        # no __build_class__ function
-        code = compile("class A: pass", "", "exec")
-        self.assertRaisesRegex(NameError, "__build_class__ not found",
-                               exec, code, {'__builtins__': {}})
+            # no __build_class__ function
+            code = compile("class A: pass", "", "exec")
+            if True:
+                self.assertRaisesRegex(NameError, "__build_class__ not found",
+                                       exec, code, {'__builtins__': {}})
 
         class frozendict_error(Exception):
             pass
@@ -579,7 +583,7 @@
                 raise frozendict_error("frozendict is readonly")
 
         # read-only builtins
-        frozen_builtins = frozendict(__builtins__)
+        frozen_builtins = frozendict(builtins.__dict__)
         code = compile("__builtins__['superglobal']=2; print(superglobal)", "test", "exec")
         self.assertRaises(frozendict_error,
                           exec, code, {'__builtins__': frozen_builtins})
diff --git a/lib-python/3/test/test_capi.py b/lib-python/3/test/test_capi.py
--- a/lib-python/3/test/test_capi.py
+++ b/lib-python/3/test/test_capi.py
@@ -110,6 +110,8 @@
         self.assertRaises(TypeError, _posixsubprocess.fork_exec,
                           Z(),[b'1'],3,[1, 2],5,6,7,8,9,10,11,12,13,14,15,16,17)
 
+ at unittest.skipIf(support.check_impl_detail(pypy=True),
+                 'Py_AddPendingCall not currently supported.')
 @unittest.skipUnless(threading, 'Threading required for this test.')
 class TestPendingCalls(unittest.TestCase):
 
@@ -327,6 +329,8 @@
         self.assertRaises(ValueError, _testcapi.parse_tuple_and_keywords,
                           (), {}, b'', [42])
 
+ at unittest.skipIf(support.check_impl_detail(pypy=True),
+                 'Not currently supported under PyPy')
 @unittest.skipUnless(threading, 'Threading required for this test.')
 class TestThreadState(unittest.TestCase):
 
diff --git a/lib-python/3/test/test_concurrent_futures.py b/lib-python/3/test/test_concurrent_futures.py
--- a/lib-python/3/test/test_concurrent_futures.py
+++ b/lib-python/3/test/test_concurrent_futures.py
@@ -295,14 +295,19 @@
         event = threading.Event()
         def future_func():
             event.wait()
-        oldswitchinterval = sys.getswitchinterval()
-        sys.setswitchinterval(1e-6)
+        newgil = hasattr(sys, 'getswitchinterval')
+        if newgil:
+            geti, seti = sys.getswitchinterval, sys.setswitchinterval
+        else:
+            geti, seti = sys.getcheckinterval, sys.setcheckinterval
+        oldinterval = geti()
+        seti(1e-6 if newgil else 1)
         try:
             fs = {self.executor.submit(future_func) for i in range(100)}
             event.set()
             futures.wait(fs, return_when=futures.ALL_COMPLETED)
         finally:
-            sys.setswitchinterval(oldswitchinterval)
+            seti(oldinterval)
 
 
 class ProcessPoolWaitTests(ProcessPoolMixin, WaitTests, unittest.TestCase):
diff --git a/lib-python/3/test/test_descr.py b/lib-python/3/test/test_descr.py
--- a/lib-python/3/test/test_descr.py
+++ b/lib-python/3/test/test_descr.py
@@ -1196,7 +1196,7 @@
         self.assertEqual(Counted.counter, 0)
 
         # Test lookup leaks [SF bug 572567]
-        if hasattr(gc, 'get_objects'):
+        if hasattr(gc, 'get_objects') and support.check_impl_detail(pypy=False):
             class G(object):
                 def __eq__(self, other):
                     return False
@@ -3035,15 +3035,24 @@
         class R(J):
             __slots__ = ["__dict__", "__weakref__"]
 
-        for cls, cls2 in ((G, H), (G, I), (I, H), (Q, R), (R, Q)):
+        if support.check_impl_detail(pypy=False):
+            lst = ((G, H), (G, I), (I, H), (Q, R), (R, Q))
+        else:
+            # Not supported in pypy: changing the __class__ of an object
+            # to another __class__ that just happens to have the same slots.
+            # If needed, we can add the feature, but what we'll likely do
+            # then is to allow mostly any __class__ assignment, even if the
+            # classes have different __slots__, because we it's easier.
+            lst = ((Q, R), (R, Q))
+        for cls, cls2 in lst:
             x = cls()
             x.a = 1
             x.__class__ = cls2
-            self.assertIs(x.__class__, cls2,
+            self.assertTrue(x.__class__ is cls2,
                    "assigning %r as __class__ for %r silently failed" % (cls2, x))
             self.assertEqual(x.a, 1)
             x.__class__ = cls
-            self.assertIs(x.__class__, cls,
+            self.assertTrue(x.__class__ is cls,
                    "assigning %r as __class__ for %r silently failed" % (cls, x))
             self.assertEqual(x.a, 1)
         for cls in G, J, K, L, M, N, P, R, list, Int:
@@ -3055,7 +3064,8 @@
         # Issue5283: when __class__ changes in __del__, the wrong
         # type gets DECREF'd.
         class O(object):
-            pass
+            def __del__(self):
+                pass
         class A(object):
             def __del__(self):
                 self.__class__ = O
@@ -3118,7 +3128,8 @@
             except TypeError:
                 pass
             else:
-                self.fail("%r's __dict__ can be modified" % cls)
+                if support.check_impl_detail(pypy=False):
+                    self.fail("%r's __dict__ can be modified" % cls)
 
         # Modules also disallow __dict__ assignment
         class Module1(types.ModuleType, Base):
diff --git a/lib-python/3/test/test_exceptions.py b/lib-python/3/test/test_exceptions.py
--- a/lib-python/3/test/test_exceptions.py
+++ b/lib-python/3/test/test_exceptions.py
@@ -512,6 +512,7 @@
         except MyException as e:
             pass
         obj = None
+        gc_collect()
         obj = wr()
         self.assertTrue(obj is None, "%s" % obj)
 
@@ -523,6 +524,7 @@
         except MyException:
             pass
         obj = None
+        gc_collect()
         obj = wr()
         self.assertTrue(obj is None, "%s" % obj)
 
@@ -534,6 +536,7 @@
         except:
             pass
         obj = None
+        gc_collect()
         obj = wr()
         self.assertTrue(obj is None, "%s" % obj)
 
@@ -546,6 +549,7 @@
             except:
                 break
         obj = None
+        gc_collect() # XXX it seems it's not enough
         obj = wr()
         self.assertTrue(obj is None, "%s" % obj)
 
@@ -564,6 +568,7 @@
             # must clear the latter manually for our test to succeed.
             e.__context__ = None
             obj = None
+            gc_collect()
             obj = wr()
             # guarantee no ref cycles on CPython (don't gc_collect)
             if check_impl_detail(cpython=False):
@@ -708,6 +713,7 @@
         next(g)
         testfunc(g)
         g = obj = None
+        gc_collect()
         obj = wr()
         self.assertIs(obj, None)
 
@@ -761,6 +767,7 @@
             raise Exception(MyObject())
         except:
             pass
+        gc_collect()
         self.assertEqual(e, (None, None, None))
 
     def testUnicodeChangeAttributes(self):
@@ -911,6 +918,7 @@
             self.assertNotEqual(wr(), None)
         else:
             self.fail("MemoryError not raised")
+        gc_collect()
         self.assertEqual(wr(), None)
 
     @no_tracing
@@ -931,6 +939,7 @@
             self.assertNotEqual(wr(), None)
         else:
             self.fail("RuntimeError not raised")
+        gc_collect()
         self.assertEqual(wr(), None)
 
     def test_errno_ENOTDIR(self):
diff --git a/lib-python/3/test/test_fileio.py b/lib-python/3/test/test_fileio.py
--- a/lib-python/3/test/test_fileio.py
+++ b/lib-python/3/test/test_fileio.py
@@ -10,6 +10,7 @@
 from functools import wraps
 
 from test.support import TESTFN, check_warnings, run_unittest, make_bad_fd, cpython_only
+from test.support import gc_collect
 from collections import UserList
 
 from _io import FileIO as _FileIO
@@ -32,6 +33,7 @@
         self.assertEqual(self.f.tell(), p.tell())
         self.f.close()
         self.f = None
+        gc_collect()
         self.assertRaises(ReferenceError, getattr, p, 'tell')
 
     def testSeekTell(self):
diff --git a/lib-python/3/test/test_functools.py b/lib-python/3/test/test_functools.py
--- a/lib-python/3/test/test_functools.py
+++ b/lib-python/3/test/test_functools.py
@@ -45,6 +45,8 @@
         self.assertEqual(p.args, (1, 2))
         self.assertEqual(p.keywords, dict(a=10, b=20))
         # attributes should not be writable
+        if not support.check_impl_detail():
+            return
         self.assertRaises(AttributeError, setattr, p, 'func', map)
         self.assertRaises(AttributeError, setattr, p, 'args', (1, 2))
         self.assertRaises(AttributeError, setattr, p, 'keywords', dict(a=1, b=2))
@@ -136,6 +138,7 @@
         p = proxy(f)
         self.assertEqual(f.func, p.func)
         f = None
+        support.gc_collect()
         self.assertRaises(ReferenceError, getattr, p, 'func')
 
     def test_with_bound_and_unbound_methods(self):
@@ -192,9 +195,13 @@
                 raise IndexError
 
         f = self.thetype(object)
-        self.assertRaisesRegex(SystemError,
-                "new style getargs format but argument is not a tuple",
-                f.__setstate__, BadSequence())
+        if support.check_impl_detail(pypy=True):
+            # CPython fails, pypy does not :-)
+            f.__setstate__(BadSequence())
+        else:
+            self.assertRaisesRegex(SystemError,
+                    "new style getargs format but argument is not a tuple",
+                    f.__setstate__, BadSequence())
 
 class PartialSubclass(functools.partial):
     pass
@@ -223,7 +230,7 @@
                       updated=functools.WRAPPER_UPDATES):
         # Check attributes were assigned
         for name in assigned:
-            self.assertTrue(getattr(wrapper, name) is getattr(wrapped, name))
+            self.assertTrue(getattr(wrapper, name) == getattr(wrapped, name))
         # Check attributes were updated
         for name in updated:
             wrapper_attr = getattr(wrapper, name)
diff --git a/lib-python/3/test/test_int.py b/lib-python/3/test/test_int.py
--- a/lib-python/3/test/test_int.py
+++ b/lib-python/3/test/test_int.py
@@ -307,9 +307,10 @@
                 try:
                     int(TruncReturnsNonIntegral())
                 except TypeError as e:
-                    self.assertEqual(str(e),
-                                      "__trunc__ returned non-Integral"
-                                      " (type NonIntegral)")
+                    if support.check_impl_detail(pypy=False):
+                        self.assertEqual(str(e),
+                                          "__trunc__ returned non-Integral"
+                                          " (type NonIntegral)")
                 else:
                     self.fail("Failed to raise TypeError with %s" %
                               ((base, trunc_result_base),))
diff --git a/lib-python/3/test/test_marshal.py b/lib-python/3/test/test_marshal.py
--- a/lib-python/3/test/test_marshal.py
+++ b/lib-python/3/test/test_marshal.py
@@ -203,6 +203,7 @@
         s = b'c' + (b'X' * 4*4) + b'{' * 2**20
         self.assertRaises(ValueError, marshal.loads, s)
 
+    @support.impl_detail('specific recursion check')
     def test_recursion_limit(self):
         # Create a deeply nested structure.
         head = last = []
@@ -291,6 +292,10 @@
 
 LARGE_SIZE = 2**31
 pointer_size = 8 if sys.maxsize > 0xFFFFFFFF else 4
+if support.check_impl_detail(pypy=False):
+    sizeof_large_size = sys.getsizeof(LARGE_SIZE-1)
+else:
+    sizeof_large_size = 32  # Some value for PyPy
 
 class NullWriter:
     def write(self, s):
@@ -318,13 +323,13 @@
         self.check_unmarshallable([None] * size)
 
     @support.bigmemtest(size=LARGE_SIZE,
-            memuse=pointer_size*12 + sys.getsizeof(LARGE_SIZE-1),
+            memuse=pointer_size*12 + sizeof_large_size,
             dry_run=False)
     def test_set(self, size):
         self.check_unmarshallable(set(range(size)))
 
     @support.bigmemtest(size=LARGE_SIZE,
-            memuse=pointer_size*12 + sys.getsizeof(LARGE_SIZE-1),
+            memuse=pointer_size*12 + sizeof_large_size,
             dry_run=False)
     def test_frozenset(self, size):
         self.check_unmarshallable(frozenset(range(size)))
diff --git a/lib-python/3/test/test_peepholer.py b/lib-python/3/test/test_peepholer.py
--- a/lib-python/3/test/test_peepholer.py
+++ b/lib-python/3/test/test_peepholer.py
@@ -81,10 +81,13 @@
             self.assertIn(elem, asm)
 
     def test_pack_unpack(self):
+        # On PyPy, "a, b = ..." is even more optimized, by removing
+        # the ROT_TWO.  But the ROT_TWO is not removed if assigning
+        # to more complex expressions, so check that.
         for line, elem in (
             ('a, = a,', 'LOAD_CONST',),
-            ('a, b = a, b', 'ROT_TWO',),
-            ('a, b, c = a, b, c', 'ROT_THREE',),
+            ('a[1], b = a, b', 'ROT_TWO',),
+            ('a, b[2], c = a, b, c', 'ROT_THREE',),
             ):
             asm = dis_single(line)
             self.assertIn(elem, asm)
@@ -92,6 +95,8 @@
             self.assertNotIn('UNPACK_TUPLE', asm)
 
     def test_folding_of_tuples_of_constants(self):
+        # On CPython, "a,b,c=1,2,3" turns into "a,b,c=<constant (1,2,3)>"
+        # but on PyPy, it turns into "a=1;b=2;c=3".
         for line, elem in (
             ('a = 1,2,3', '((1, 2, 3))'),
             ('("a","b","c")', "(('a', 'b', 'c'))"),
@@ -100,7 +105,8 @@
             ('((1, 2), 3, 4)', '(((1, 2), 3, 4))'),
             ):
             asm = dis_single(line)
-            self.assertIn(elem, asm)
+            self.assert_(elem in asm or (
+                line == 'a,b,c = 1,2,3' and 'UNPACK_TUPLE' not in asm))
             self.assertNotIn('BUILD_TUPLE', asm)
 
         # Long tuples should be folded too.
diff --git a/lib-python/3/test/test_subprocess.py b/lib-python/3/test/test_subprocess.py
--- a/lib-python/3/test/test_subprocess.py
+++ b/lib-python/3/test/test_subprocess.py
@@ -1314,6 +1314,7 @@
                         stdin=subprocess.PIPE, stdout=subprocess.PIPE,
                         stderr=subprocess.PIPE, preexec_fn=raise_it)
 
+    @support.impl_detail("PyPy's _posixsubprocess doesn't have to disable gc")
     def test_preexec_gc_module_failure(self):
         # This tests the code that disables garbage collection if the child
         # process will execute any Python.
@@ -1964,6 +1965,7 @@
         ident = id(p)
         pid = p.pid
         del p
+        support.gc_collect()
         # check that p is in the active processes list
         self.assertIn(ident, [id(o) for o in subprocess._active])
 
@@ -1983,6 +1985,7 @@
         ident = id(p)
         pid = p.pid
         del p
+        support.gc_collect()
         os.kill(pid, signal.SIGKILL)
         # check that p is in the active processes list
         self.assertIn(ident, [id(o) for o in subprocess._active])
diff --git a/lib-python/3/test/test_sys.py b/lib-python/3/test/test_sys.py
--- a/lib-python/3/test/test_sys.py
+++ b/lib-python/3/test/test_sys.py
@@ -405,8 +405,10 @@
         self.assertEqual(len(sys.float_info), 11)
         self.assertEqual(sys.float_info.radix, 2)
         self.assertEqual(len(sys.int_info), 2)
-        self.assertTrue(sys.int_info.bits_per_digit % 5 == 0)
-        self.assertTrue(sys.int_info.sizeof_digit >= 1)
+        if test.support.check_impl_detail(cpython=True):
+            self.assertTrue(sys.int_info.bits_per_digit % 5 == 0)
+        else:
+            self.assertTrue(sys.int_info.sizeof_digit >= 1)
         self.assertEqual(type(sys.int_info.bits_per_digit), int)
         self.assertEqual(type(sys.int_info.sizeof_digit), int)
         self.assertIsInstance(sys.hexversion, int)
@@ -503,6 +505,7 @@
         self.assertTrue(repr(sys.flags))
         self.assertEqual(len(sys.flags), len(attrs))
 
+    @test.support.impl_detail("sys._clear_type_cache", pypy=False)
     def test_clear_type_cache(self):
         sys._clear_type_cache()
 
diff --git a/lib-python/3/test/test_weakref.py b/lib-python/3/test/test_weakref.py
--- a/lib-python/3/test/test_weakref.py
+++ b/lib-python/3/test/test_weakref.py
@@ -8,6 +8,7 @@
 import copy
 
 from test import support
+from test.support import gc_collect
 
 # Used in ReferencesTestCase.test_ref_created_during_del() .
 ref_from_del = None
@@ -88,6 +89,7 @@
         ref1 = weakref.ref(o, self.callback)
         ref2 = weakref.ref(o, self.callback)
         del o
+        gc_collect()
         self.assertIsNone(ref1(), "expected reference to be invalidated")
         self.assertIsNone(ref2(), "expected reference to be invalidated")
         self.assertEqual(self.cbcalled, 2,
@@ -117,13 +119,16 @@
         ref1 = weakref.proxy(o, self.callback)
         ref2 = weakref.proxy(o, self.callback)
         del o
+        gc_collect()
 
         def check(proxy):
             proxy.bar
 
         self.assertRaises(ReferenceError, check, ref1)
         self.assertRaises(ReferenceError, check, ref2)
-        self.assertRaises(ReferenceError, bool, weakref.proxy(C()))
+        ref3 = weakref.proxy(C())
+        gc_collect()
+        self.assertRaises(ReferenceError, bool, ref3)
         self.assertEqual(self.cbcalled, 2)
 
     def check_basic_ref(self, factory):
@@ -140,6 +145,7 @@
         o = factory()
         ref = weakref.ref(o, self.callback)
         del o
+        gc_collect()
         self.assertEqual(self.cbcalled, 1,
                      "callback did not properly set 'cbcalled'")
         self.assertIsNone(ref(),
@@ -164,6 +170,7 @@
         self.assertEqual(weakref.getweakrefcount(o), 2,
                      "wrong weak ref count for object")
         del proxy
+        gc_collect()
         self.assertEqual(weakref.getweakrefcount(o), 1,
                      "wrong weak ref count for object after deleting proxy")
 
@@ -338,6 +345,7 @@
                      "got wrong number of weak reference objects")
 
         del ref1, ref2, proxy1, proxy2
+        gc_collect()
         self.assertEqual(weakref.getweakrefcount(o), 0,
                      "weak reference objects not unlinked from"
                      " referent when discarded.")
@@ -351,6 +359,7 @@
         ref1 = weakref.ref(o, self.callback)
         ref2 = weakref.ref(o, self.callback)
         del ref1
+        gc_collect()
         self.assertEqual(weakref.getweakrefs(o), [ref2],
                      "list of refs does not match")
 
@@ -358,10 +367,12 @@
         ref1 = weakref.ref(o, self.callback)
         ref2 = weakref.ref(o, self.callback)
         del ref2
+        gc_collect()
         self.assertEqual(weakref.getweakrefs(o), [ref1],
                      "list of refs does not match")
 
         del ref1
+        gc_collect()
         self.assertEqual(weakref.getweakrefs(o), [],
                      "list of refs not cleared")
 
@@ -647,9 +658,11 @@
         gc.collect()
         self.assertEqual(alist, [])
 
+    @support.impl_detail(pypy=False)
     def test_gc_during_ref_creation(self):
         self.check_gc_during_creation(weakref.ref)
 
+    @support.impl_detail(pypy=False)
     def test_gc_during_proxy_creation(self):
         self.check_gc_during_creation(weakref.proxy)
 
@@ -811,6 +824,7 @@
         self.assertTrue(mr.called)
         self.assertEqual(mr.value, 24)
         del o
+        gc_collect()
         self.assertIsNone(mr())
         self.assertTrue(mr.called)
 
@@ -917,6 +931,7 @@
         n1 = len(dct)
         del it
         gc.collect()
+        gc.collect()
         n2 = len(dct)
         # one item may be kept alive inside the iterator
         self.assertIn(n1, (0, 1))
@@ -928,6 +943,7 @@
     def test_weak_valued_len_cycles(self):
         self.check_len_cycles(weakref.WeakValueDictionary, lambda k: (1, k))
 
+    @support.impl_detail(pypy=False)
     def check_len_race(self, dict_type, cons):
         # Extended sanity checks for len() in the face of cyclic collection
         self.addCleanup(gc.set_threshold, *gc.get_threshold())
@@ -976,15 +992,18 @@
         del items1, items2
         self.assertEqual(len(dict), self.COUNT)
         del objects[0]
+        gc_collect()
         self.assertEqual(len(dict), self.COUNT - 1,
                      "deleting object did not cause dictionary update")
         del objects, o
+        gc_collect()
         self.assertEqual(len(dict), 0,
                      "deleting the values did not clear the dictionary")
         # regression on SF bug #447152:
         dict = weakref.WeakValueDictionary()
         self.assertRaises(KeyError, dict.__getitem__, 1)
         dict[2] = C()
+        gc_collect()
         self.assertRaises(KeyError, dict.__getitem__, 2)
 
     def test_weak_keys(self):
@@ -1005,9 +1024,11 @@
         del items1, items2
         self.assertEqual(len(dict), self.COUNT)
         del objects[0]
+        gc_collect()
         self.assertEqual(len(dict), (self.COUNT - 1),
                      "deleting object did not cause dictionary update")
         del objects, o
+        gc_collect()
         self.assertEqual(len(dict), 0,
                      "deleting the keys did not clear the dictionary")
         o = Object(42)
@@ -1368,6 +1389,7 @@
         for o in objs:
             count += 1
             del d[o]
+        gc_collect()
         self.assertEqual(len(d), 0)
         self.assertEqual(count, 2)
 
@@ -1389,6 +1411,7 @@
 
 libreftest = """ Doctest for examples in the library reference: weakref.rst
 
+>>> from test.support import gc_collect
 >>> import weakref
 >>> class Dict(dict):
 ...     pass
@@ -1408,6 +1431,7 @@
 >>> o is o2
 True
 >>> del o, o2
+>>> gc_collect()
 >>> print(r())
 None
 
@@ -1460,6 +1484,7 @@
 >>> id2obj(a_id) is a
 True
 >>> del a
+>>> gc_collect()
 >>> try:
 ...     id2obj(a_id)
 ... except KeyError:
diff --git a/lib-python/3/test/test_weakset.py b/lib-python/3/test/test_weakset.py
--- a/lib-python/3/test/test_weakset.py
+++ b/lib-python/3/test/test_weakset.py
@@ -416,11 +416,13 @@
         n1 = len(s)
         del it
         gc.collect()
+        gc.collect()
         n2 = len(s)
         # one item may be kept alive inside the iterator
         self.assertIn(n1, (0, 1))
         self.assertEqual(n2, 0)
 
+    @support.impl_detail("PyPy has no cyclic collection", pypy=False)
     def test_len_race(self):
         # Extended sanity checks for len() in the face of cyclic collection
         self.addCleanup(gc.set_threshold, *gc.get_threshold())
diff --git a/lib_pypy/_curses.py b/lib_pypy/_curses.py
--- a/lib_pypy/_curses.py
+++ b/lib_pypy/_curses.py
@@ -309,11 +309,9 @@
 #endif
 
 int _m_ispad(WINDOW *win) {
-#if defined WINDOW_HAS_FLAGS
+    // <curses.h> may not have _flags (and possibly _ISPAD),
+    // but for now let's assume that <ncurses.h> always has it
     return (win->_flags & _ISPAD);
-#else
-    return 0;
-#endif
 }
 
 void _m_getsyx(int *yx) {
diff --git a/lib_pypy/_pypy_testcapi.py b/lib_pypy/_pypy_testcapi.py
--- a/lib_pypy/_pypy_testcapi.py
+++ b/lib_pypy/_pypy_testcapi.py
@@ -1,5 +1,6 @@
 import os, sys, imp
 import tempfile, binascii
+import importlib.machinery
 
 
 def get_hashed_dir(cfile):
@@ -28,9 +29,8 @@
 
 
 def _get_c_extension_suffix():
-    for ext, mod, typ in imp.get_suffixes():
-        if typ == imp.C_EXTENSION:
-            return ext
+    suffixes = importlib.machinery.EXTENSION_SUFFIXES
+    return suffixes[0] if suffixes else None
 
 
 def compile_shared(csource, modulename, output_dir=None):
diff --git a/lib_pypy/cffi/verifier.py b/lib_pypy/cffi/verifier.py
--- a/lib_pypy/cffi/verifier.py
+++ b/lib_pypy/cffi/verifier.py
@@ -1,7 +1,17 @@
-import sys, os, binascii, imp, shutil
+import sys, os, binascii, shutil
 from . import __version__
 from . import ffiplatform
 
+if sys.version_info >= (3, 3):
+    import importlib.machinery
+    def extension_suffixes():
+        return importlib.machinery.EXTENSION_SUFFIXES[:]
+else:
+    import imp
+    def extension_suffixes():
+        return [suffix for suffix, _, type in imp.get_suffixes()
+                if type == imp.C_EXTENSION]
+
 
 class Verifier(object):
 
@@ -222,11 +232,7 @@
             pass
 
 def _get_so_suffixes():
-    suffixes = []
-    for suffix, mode, type in imp.get_suffixes():
-        if type == imp.C_EXTENSION:
-            suffixes.append(suffix)
-
+    suffixes = extension_suffixes()
     if not suffixes:
         # bah, no C_EXTENSION available.  Occurs on pypy without cpyext
         if sys.platform == 'win32':
diff --git a/lib_pypy/pyrepl/module_lister.py b/lib_pypy/pyrepl/module_lister.py
--- a/lib_pypy/pyrepl/module_lister.py
+++ b/lib_pypy/pyrepl/module_lister.py
@@ -40,8 +40,8 @@
     return sorted(set(l))
 
 def _make_module_list():
-    import imp
-    suffs = [x[0] for x in imp.get_suffixes() if x[0] != '.pyc']
+    import importlib.machinery
+    suffs = [x for x in importlib.machinery.all_suffixes() if x != '.pyc']
     suffs.sort(reverse=True)
     _packages[''] = list(sys.builtin_module_names)
     for dir in sys.path:
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -37,7 +37,7 @@
     "binascii", "_multiprocessing", '_warnings', "_collections",
     "_multibytecodec", "_continuation", "_cffi_backend",
     "_csv", "_pypyjson", "_posixsubprocess", # "cppyy", "micronumpy"
-    "faulthandler",
+    "faulthandler", "_lzma",
 ])
 
 translation_modules = default_modules.copy()
@@ -106,6 +106,7 @@
     "_hashlib"  : ["pypy.module._ssl.interp_ssl"],
     "_minimal_curses": ["pypy.module._minimal_curses.fficurses"],
     "_continuation": ["rpython.rlib.rstacklet"],
+    "_lzma"     : ["pypy.module._lzma.interp_lzma"],
     }
 
 def get_module_validator(modname):
diff --git a/pypy/doc/coding-guide.rst b/pypy/doc/coding-guide.rst
--- a/pypy/doc/coding-guide.rst
+++ b/pypy/doc/coding-guide.rst
@@ -740,7 +740,7 @@
 
 Adding an entry under pypy/module (e.g. mymodule) entails automatic
 creation of a new config option (such as --withmod-mymodule and
---withoutmod-mymodule (the later being the default)) for py.py and
+--withoutmod-mymodule (the latter being the default)) for py.py and
 translate.py.
 
 Testing modules in ``lib_pypy/``
@@ -931,7 +931,7 @@
             assert self.result == 2 ** 6
 
 which executes the code string function with the given arguments at app level.
-Note the use of ``w_result`` in ``setup_class`` but self.result in the test 
+Note the use of ``w_result`` in ``setup_class`` but self.result in the test.
 Here is how to define an app level class  in ``setup_class`` that can be used
 in subsequent tests::
 
diff --git a/pypy/doc/config/objspace.usemodules._lzma.txt b/pypy/doc/config/objspace.usemodules._lzma.txt
new file mode 100644
--- /dev/null
+++ b/pypy/doc/config/objspace.usemodules._lzma.txt
@@ -0,0 +1,2 @@
+Use the '_lzma' module. 
+This module is expected to be working and is included by default.
diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst
--- a/pypy/doc/cpython_differences.rst
+++ b/pypy/doc/cpython_differences.rst
@@ -328,7 +328,7 @@
 * directly calling the internal magic methods of a few built-in types
   with invalid arguments may have a slightly different result.  For
   example, ``[].__add__(None)`` and ``(2).__add__(None)`` both return
-  ``NotImplemented`` on PyPy; on CPython, only the later does, and the
+  ``NotImplemented`` on PyPy; on CPython, only the latter does, and the
   former raises ``TypeError``.  (Of course, ``[]+None`` and ``2+None``
   both raise ``TypeError`` everywhere.)  This difference is an
   implementation detail that shows up because of internal C-level slots
diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py
--- a/pypy/interpreter/astcompiler/assemble.py
+++ b/pypy/interpreter/astcompiler/assemble.py
@@ -7,7 +7,7 @@
 from pypy.tool import stdlib_opcode as ops
 
 from pypy.interpreter.error import OperationError
-from rpython.rlib.objectmodel import we_are_translated
+from rpython.rlib.objectmodel import specialize, we_are_translated
 from rpython.rlib import rfloat
 
 
@@ -141,11 +141,12 @@
             i += 1
     return result
 
-def _list_to_dict(l, offset=0):
+ at specialize.argtype(0)
+def _iter_to_dict(iterable, offset=0):
     result = {}
     index = offset
-    for i in range(len(l)):
-        result[l[i]] = index
+    for item in iterable:
+        result[item] = index
         index += 1
     return result
 
@@ -161,10 +162,10 @@
         self.first_block = self.new_block()
         self.use_block(self.first_block)
         self.names = {}
-        self.var_names = _list_to_dict(scope.varnames)
+        self.var_names = _iter_to_dict(scope.varnames)
         self.cell_vars = _make_index_dict_filter(scope.symbols,
                                                  symtable.SCOPE_CELL)
-        self.free_vars = _list_to_dict(scope.free_vars, len(self.cell_vars))
+        self.free_vars = _iter_to_dict(scope.free_vars, len(self.cell_vars))
         self.w_consts = space.newdict()
         self.argcount = 0
         self.kwonlyargcount = 0
diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py
--- a/pypy/interpreter/astcompiler/codegen.py
+++ b/pypy/interpreter/astcompiler/codegen.py
@@ -244,7 +244,7 @@
         self.emit_op_arg(op, self.add_name(container, identifier))
 
     def possible_docstring(self, node):
-        if isinstance(node, ast.Expr):
+        if isinstance(node, ast.Expr) and self.compile_info.optimize < 2:
             expr_value = node.value
             if isinstance(expr_value, ast.Str):
                 return expr_value
diff --git a/pypy/interpreter/astcompiler/symtable.py b/pypy/interpreter/astcompiler/symtable.py
--- a/pypy/interpreter/astcompiler/symtable.py
+++ b/pypy/interpreter/astcompiler/symtable.py
@@ -37,7 +37,7 @@
         self.roles = {}
         self.varnames = []
         self.children = []
-        self.free_vars = []
+        self.free_vars = {}
         self.temp_name_counter = 1
         self.has_free = False
         self.child_has_free = False
@@ -136,7 +136,9 @@
                 err = "no binding for nonlocal '%s' found" % (name,)
                 raise SyntaxError(err, self.lineno, self.col_offset)
             self.symbols[name] = SCOPE_FREE
+            self.free_vars[name] = None
             free[name] = None
+            self.has_free = True
         elif flags & SYM_BOUND:
             self.symbols[name] = SCOPE_LOCAL
             local[name] = None
@@ -146,7 +148,7 @@
                 pass
         elif bound and name in bound:
             self.symbols[name] = SCOPE_FREE
-            self.free_vars.append(name)
+            self.free_vars[name] = None
             free[name] = None
             self.has_free = True
         elif name in globs:
@@ -203,7 +205,7 @@
             except KeyError:
                 if name in bound:
                     self.symbols[name] = SCOPE_FREE
-                    self.free_vars.append(name)
+                    self.free_vars[name] = None
             else:
                 if role_here & (SYM_BOUND | SYM_GLOBAL) and \
                         self._hide_bound_from_nested_scopes:
@@ -212,7 +214,7 @@
                     # scope.  We add the name to the class scope's list of free
                     # vars, so it will be passed through by the interpreter, but
                     # we leave the scope alone, so it can be local on its own.
-                    self.free_vars.append(name)
+                    self.free_vars[name] = None
         self._check_optimization()
         free.update(new_free)
 
@@ -244,18 +246,12 @@
         return Scope.note_symbol(self, identifier, role)
 
     def note_yield(self, yield_node):
-        if self.return_with_value:
-            raise SyntaxError("'return' with argument inside generator",
-                              self.ret.lineno, self.ret.col_offset)
         self.is_generator = True
         if self._in_try_body_depth > 0:
             self.has_yield_inside_try = True
 
     def note_return(self, ret):
         if ret.value:
-            if self.is_generator:
-                raise SyntaxError("'return' with argument inside generator",
-                                  ret.lineno, ret.col_offset)
             self.return_with_value = True
             self.ret = ret
 
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -808,6 +808,28 @@
             return y"""
         yield self.st, test, "f()", 4
 
+    def test_nonlocal_from_arg(self):
+        test = """if 1:
+        def test1(x):
+            def test2():
+                nonlocal x
+                def test3():
+                    return x
+                return test3()
+            return test2()"""
+        yield self.st, test, "test1(2)", 2
+
+    def test_class_nonlocal_from_arg(self):
+        test = """if 1:
+        def f(x):
+            class c:
+                nonlocal x
+                x += 1
+                def get(self):
+                    return x
+            return c().get()"""
+        yield self.st, test, "f(3)", 4
+
     def test_lots_of_loops(self):
         source = "for x in y: pass\n" * 1000
         compile_with_astcompiler(source, 'exec', self.space)
diff --git a/pypy/interpreter/astcompiler/test/test_symtable.py b/pypy/interpreter/astcompiler/test/test_symtable.py
--- a/pypy/interpreter/astcompiler/test/test_symtable.py
+++ b/pypy/interpreter/astcompiler/test/test_symtable.py
@@ -361,8 +361,7 @@
             assert exc.msg == "'yield' outside function"
         for input in ("yield\n    return x", "return x\n    yield"):
             input = "def f():\n    " + input
-            exc = py.test.raises(SyntaxError, self.func_scope, input).value
-            assert exc.msg == "'return' with argument inside generator"
+            scp = self.func_scope(input)
         scp = self.func_scope("def f():\n    return\n    yield x")
 
     def test_yield_inside_try(self):
diff --git a/pypy/interpreter/astcompiler/validate.py b/pypy/interpreter/astcompiler/validate.py
--- a/pypy/interpreter/astcompiler/validate.py
+++ b/pypy/interpreter/astcompiler/validate.py
@@ -14,6 +14,9 @@
     def __init__(self, message):
         self.message = message
 
+    def __str__(self):
+        return self.message
+
 
 def expr_context_name(ctx):
     if not 1 <= ctx <= len(ast.expr_context_to_class):
diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -99,7 +99,7 @@
             # if the frame is now marked as finished, it was RETURNed from
             if frame.frame_finished_execution:
                 self.frame = None
-                raise OperationError(space.w_StopIteration, space.w_None)
+                raise OperationError(space.w_StopIteration, w_result)
             else:
                 return w_result     # YIELDed
         finally:
diff --git a/pypy/interpreter/module.py b/pypy/interpreter/module.py
--- a/pypy/interpreter/module.py
+++ b/pypy/interpreter/module.py
@@ -3,7 +3,7 @@
 """
 
 from pypy.interpreter.baseobjspace import W_Root
-from pypy.interpreter.error import OperationError
+from pypy.interpreter.error import OperationError, oefmt
 from rpython.rlib.objectmodel import we_are_translated
 
 
@@ -123,3 +123,10 @@
         except OperationError:
             __file__ = u'?'
         return space.wrap(u"<module %s from %s>" % (name, __file__))
+
+    def descr_module__dir__(self, space):
+        w_dict = space.getattr(self, space.wrap('__dict__'))
+        if not space.isinstance_w(w_dict, space.w_dict):
+            raise oefmt(space.w_TypeError, "%N.__dict__ is not a dictionary",
+                        self)
+        return space.call_function(space.w_list, w_dict)
diff --git a/pypy/interpreter/nestedscope.py b/pypy/interpreter/nestedscope.py
--- a/pypy/interpreter/nestedscope.py
+++ b/pypy/interpreter/nestedscope.py
@@ -75,8 +75,9 @@
         if self.w_value is None:
             content = "empty"
         else:
-            content = "%s object at 0x%x" % (space.type(self.w_value).name, uid(self.w_value))
-        s = "<cell at 0x%x: %s>" % (uid(self), content)
+            content = "%s object at 0x%s" % (space.type(self.w_value).name,
+                                             self.w_value.getaddrstring(space))
+        s = "<cell at 0x%s: %s>" % (self.getaddrstring(space), content)
         return space.wrap(s.decode('utf-8'))
 
     def descr__cell_contents(self, space):
diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py
--- a/pypy/interpreter/pycode.py
+++ b/pypy/interpreter/pycode.py
@@ -4,7 +4,7 @@
 The bytecode interpreter itself is implemented by the PyFrame class.
 """
 
-import dis, imp, struct, types, new, sys
+import imp, struct, types, new, sys
 
 from pypy.interpreter import eval
 from pypy.interpreter.signature import Signature
@@ -13,6 +13,7 @@
 from pypy.interpreter.astcompiler.consts import (
     CO_OPTIMIZED, CO_NEWLOCALS, CO_VARARGS, CO_VARKEYWORDS, CO_NESTED,
     CO_GENERATOR, CO_KILL_DOCSTRING, CO_YIELD_INSIDE_TRY)
+from pypy.tool import dis3
 from pypy.tool.stdlib_opcode import opcodedesc, HAVE_ARGUMENT
 from rpython.rlib.rarithmetic import intmask
 from rpython.rlib.objectmodel import compute_hash, we_are_translated
@@ -60,7 +61,7 @@
     else:
         varargname = None
     if code.co_flags & CO_VARKEYWORDS:
-        kwargname = code.co_varnames[argcount]
+        kwargname = code.co_varnames[argcount+kwonlyargcount]
         argcount += 1
     else:
         kwargname = None
@@ -245,33 +246,6 @@
             if isinstance(w_co, PyCode):
                 w_co.remove_docstrings(space)
 
-    def _to_code(self):
-        """For debugging only."""
-        consts = [None] * len(self.co_consts_w)
-        num = 0
-        for w in self.co_consts_w:
-            if isinstance(w, PyCode):
-                consts[num] = w._to_code()
-            else:
-                consts[num] = self.space.unwrap(w)
-            num += 1
-        assert self.co_kwonlyargcount == 0, 'kwonlyargcount is py3k only, cannot turn this code object into a Python2 one'
-        return new.code(self.co_argcount,
-                        #self.co_kwonlyargcount, # this does not exists in python2
-                        self.co_nlocals,
-                        self.co_stacksize,
-                        self.co_flags,
-                        self.co_code,
-                        tuple(consts),
-                        tuple(self.co_names),
-                        tuple(self.co_varnames),
-                        self.co_filename,
-                        self.co_name,
-                        self.co_firstlineno,
-                        self.co_lnotab,
-                        tuple(self.co_freevars),
-                        tuple(self.co_cellvars))
-
     def exec_host_bytecode(self, w_globals, w_locals):
         if sys.version_info < (2, 7):
             raise Exception("PyPy no longer supports Python 2.6 or lower")
@@ -280,11 +254,11 @@
         return frame.run()
 
     def dump(self):
-        """A dis.dis() dump of the code object."""
-        print 'WARNING: dumping a py3k bytecode using python2 opmap, the result might be inaccurate or wrong'
-        print
-        co = self._to_code()
-        dis.dis(co)
+        """NOT_RPYTHON: A dis.dis() dump of the code object."""
+        if not hasattr(self, 'co_consts'):
+            self.co_consts = [w if isinstance(w, PyCode) else self.space.unwrap(w)
+                              for w in self.co_consts_w]
+        dis3.dis(self)
 
     def fget_co_consts(self, space):
         return space.newtuple(self.co_consts_w)
diff --git a/pypy/interpreter/pycompiler.py b/pypy/interpreter/pycompiler.py
--- a/pypy/interpreter/pycompiler.py
+++ b/pypy/interpreter/pycompiler.py
@@ -96,7 +96,7 @@
 
     XXX: This class should override the baseclass implementation of
          compile_command() in order to optimize it, especially in case
-         of incomplete inputs (e.g. we shouldn't re-compile from sracth
+         of incomplete inputs (e.g. we shouldn't re-compile from scratch
          the whole source after having only added a new '\n')
     """
     def __init__(self, space, override_version=None):
diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1006,13 +1006,14 @@
             else:
                 w_retval = space.call_method(w_gen, "send", w_value)
         except OperationError as e:
-            if not e.match(self.space, self.space.w_StopIteration):
+            if not e.match(space, space.w_StopIteration):
                 raise
             self.popvalue()  # Remove iter from stack
+            e.normalize_exception(space)
             try:
                 w_value = space.getattr(e.get_w_value(space), space.wrap("value"))
             except OperationError as e:
-                if not e.match(self.space, self.space.w_AttributeError):
+                if not e.match(space, space.w_AttributeError):
                     raise
                 w_value = space.w_None
             self.pushvalue(w_value)
diff --git a/pypy/interpreter/pytraceback.py b/pypy/interpreter/pytraceback.py
--- a/pypy/interpreter/pytraceback.py
+++ b/pypy/interpreter/pytraceback.py
@@ -50,6 +50,10 @@
         self.lasti = space.int_w(w_lasti)
         self.next = space.interp_w(PyTraceback, w_next, can_be_None=True)
 
+    def descr__dir__(self, space):
+        return space.newlist([space.wrap(n) for n in
+            ['tb_frame', 'tb_next', 'tb_lasti', 'tb_lineno']])
+
 
 def record_application_traceback(space, operror, frame, last_instruction):
     if frame.pycode.hidden_applevel:
diff --git a/pypy/interpreter/test/test_compiler.py b/pypy/interpreter/test/test_compiler.py
--- a/pypy/interpreter/test/test_compiler.py
+++ b/pypy/interpreter/test/test_compiler.py
@@ -268,10 +268,7 @@
 
     def test_return_in_generator(self):
         code = 'def f():\n return None\n yield 19\n'
-        e = py.test.raises(OperationError, self.compiler.compile, code, '', 'single', 0)
-        ex = e.value
-        ex.normalize_exception(self.space)
-        assert ex.match(self.space, self.space.w_SyntaxError)
+        self.compiler.compile(code, '', 'single', 0)
 
     def test_yield_in_finally(self):
         code ='def f():\n try:\n  yield 19\n finally:\n  pass\n'
@@ -717,6 +714,27 @@
         else:
             py.test.fail("Did not raise")
 
+    def test_signature_kwargname(self):
+        from pypy.interpreter.pycode import cpython_code_signature
+        from pypy.interpreter.signature import Signature
+
+        def find_func(code):
+            for w_const in code.co_consts_w:
+                if isinstance(w_const, PyCode):
+                    return w_const
+
+        snippet = 'def f(a, b, m=1, n=2, **kwargs): pass'
+        containing_co = self.compiler.compile(snippet, '<string>', 'single', 0)
+        co = find_func(containing_co)
+        sig = cpython_code_signature(co)
+        assert sig == Signature(['a', 'b', 'm', 'n'], None, 'kwargs', [])
+
+        snippet = 'def f(a, b, *, m=1, n=2, **kwargs): pass'
+        containing_co = self.compiler.compile(snippet, '<string>', 'single', 0)
+        co = find_func(containing_co)
+        sig = cpython_code_signature(co)
+        assert sig == Signature(['a', 'b'], None, 'kwargs', ['m', 'n'])
+
 
 class AppTestCompiler:
 
diff --git a/pypy/interpreter/test/test_generator.py b/pypy/interpreter/test/test_generator.py
--- a/pypy/interpreter/test/test_generator.py
+++ b/pypy/interpreter/test/test_generator.py
@@ -301,6 +301,39 @@
             raise StopIteration
         assert tuple(f()) == (1,)
 
+    def test_yield_return(self):
+        """
+        def f():
+            yield 1
+            return 2
+        g = f()
+        assert next(g) == 1
+        try:
+            next(g)
+        except StopIteration as e:
+            assert e.value == 2
+        else:
+            assert False, 'Expected StopIteration'
+            """
+
+    def test_yield_from_return(self):
+        """
+        def f1():
+            result = yield from f2()
+            return result
+        def f2():
+            yield 1
+            return 2
+        g = f1()
+        assert next(g) == 1
+        try:
+            next(g)
+        except StopIteration as e:
+            assert e.value == 2
+        else:
+            assert False, 'Expected StopIteration'
+            """
+
 
 def test_should_not_inline(space):
     from pypy.interpreter.generator import should_not_inline
diff --git a/pypy/interpreter/test/test_module.py b/pypy/interpreter/test/test_module.py
--- a/pypy/interpreter/test/test_module.py
+++ b/pypy/interpreter/test/test_module.py
@@ -68,6 +68,11 @@
         m = type(_pypy_interact).__new__(type(_pypy_interact))
         assert repr(m).startswith("<module '?'")
 
+    def test_dir(self):
+        import sys
+        items = sys.__dir__()
+        assert sorted(items) == dir(sys)
+
     def test_package(self):
         import sys
         import os
diff --git a/pypy/interpreter/test/test_pycode.py b/pypy/interpreter/test/test_pycode.py
new file mode 100644
--- /dev/null
+++ b/pypy/interpreter/test/test_pycode.py
@@ -0,0 +1,19 @@
+import sys, StringIO
+
+def test_dump(space):
+    """test that pycode.dump kind of works with py3 opcodes"""
+    compiler = space.createcompiler()
+    code = compiler.compile('lambda *, y=7: None', 'filename', 'exec', 0)
+    output = None
+    stdout = sys.stdout
+    try:
+        sys.stdout = StringIO.StringIO()
+        code.dump()
+        output = sys.stdout.getvalue()
+        sys.stdout.close()
+    finally:
+        sys.stdout = stdout
+    print '>>>\n' + output + '\n<<<'
+    assert ' 1 (7)' in output
+    assert ' 3 (None)' in output
+    assert ' 16 RETURN_VALUE ' in output
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -779,6 +779,7 @@
     __new__ = interp2app(Module.descr_module__new__.im_func),
     __init__ = interp2app(Module.descr_module__init__),
     __repr__ = interp2app(Module.descr_module__repr__),
+    __dir__ = interp2app(Module.descr_module__dir__),
     __reduce__ = interp2app(Module.descr__reduce__),
     __dict__ = GetSetProperty(descr_get_dict, cls=Module), # module dictionaries are readonly attributes
     __doc__ = 'module(name[, doc])\n\nCreate a module object.\nThe name must be a string; the optional doc argument can have any type.'
@@ -912,6 +913,7 @@
 PyTraceback.typedef = TypeDef("traceback",
     __reduce__ = interp2app(PyTraceback.descr__reduce__),
     __setstate__ = interp2app(PyTraceback.descr__setstate__),
+    __dir__ = interp2app(PyTraceback.descr__dir__),
     tb_frame = interp_attrproperty('frame', cls=PyTraceback),
     tb_lasti = interp_attrproperty('lasti', cls=PyTraceback),
     tb_lineno = GetSetProperty(PyTraceback.descr_tb_lineno),
diff --git a/pypy/module/__builtin__/app_inspect.py b/pypy/module/__builtin__/app_inspect.py
--- a/pypy/module/__builtin__/app_inspect.py
+++ b/pypy/module/__builtin__/app_inspect.py
@@ -7,8 +7,8 @@
 
 from __pypy__ import lookup_special
 
-def _caller_locals(): 
-    return sys._getframe(0).f_locals 
+def _caller_locals():
+    return sys._getframe(0).f_locals
 
 def vars(*obj):
     """Return a dictionary of all the attributes currently bound in obj.  If
@@ -18,11 +18,10 @@
         return _caller_locals()
     elif len(obj) != 1:
         raise TypeError("vars() takes at most 1 argument.")
-    else:
-        try:
-            return obj[0].__dict__
-        except AttributeError:
-            raise TypeError("vars() argument must have __dict__ attribute")
+    try:
+        return obj[0].__dict__
+    except AttributeError:
+        raise TypeError("vars() argument must have __dict__ attribute")
 
 def dir(*args):
     """dir([object]) -> list of strings
@@ -38,76 +37,16 @@
         attributes of its class's base classes.
     """
     if len(args) > 1:
-        raise TypeError("dir expected at most 1 arguments, got %d"
-                        % len(args))
+        raise TypeError("dir expected at most 1 arguments, got %d" % len(args))
     if len(args) == 0:
-        local_names = list(_caller_locals().keys()) # 2 stackframes away
-        local_names.sort()
-        return local_names
-
-    import types
+        return sorted(_caller_locals().keys()) # 2 stackframes away
 
     obj = args[0]
-
-    dir_meth = lookup_special(obj, "__dir__")
+    dir_meth = lookup_special(obj, '__dir__')
     if dir_meth is not None:
-        result = dir_meth()
-        if not isinstance(result, list):
-            result = list(result)  # Will throw TypeError if not iterable
-        result.sort()
-        return result
-    elif isinstance(obj, types.ModuleType):
-        try:
-            result = list(obj.__dict__)
-            result.sort()
-            return result
-        except AttributeError:
-            return []
-
-    elif isinstance(obj, type):
-        #Don't look at __class__, as metaclass methods would be confusing.
-        result = list(_classdir(obj).keys())
-        result.sort()
-        return result
-
-    else: #(regular item)
-        Dict = {}
-        try:
-            if isinstance(obj.__dict__, dict):
-                Dict.update(obj.__dict__)
-        except AttributeError:
-            pass
-        try:
-            Dict.update(_classdir(obj.__class__))
-        except AttributeError:
-            pass
-        result = list(Dict.keys())
-        result.sort()
-        return result
-
-def _classdir(klass):
-    """Return a dict of the accessible attributes of class/type klass.
-
-    This includes all attributes of klass and all of the
-    base classes recursively.
-
-    The values of this dict have no meaning - only the keys have
-    meaning.  
-    """
-    Dict = {}
-    try:
-        Dict.update(klass.__dict__)
-    except AttributeError: pass 
-    try:
-        # XXX - Use of .__mro__ would be suggested, if the existance
-        #   of that attribute could be guarranted.
-        bases = klass.__bases__
-    except AttributeError: pass
-    else:
-        try:
-            #Note that since we are only interested in the keys,
-            #  the order we merge classes is unimportant
-            for base in bases:
-                Dict.update(_classdir(base))
-        except TypeError: pass
-    return Dict
+        # obscure: lookup_special won't bind None.__dir__!
+        result = dir_meth(obj) if obj is None else dir_meth()
+        # Will throw TypeError if not iterable
+        return sorted(result)
+    # we should never reach here since object.__dir__ exists
+    return []
diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py
--- a/pypy/module/__builtin__/functional.py
+++ b/pypy/module/__builtin__/functional.py
@@ -670,8 +670,8 @@
     def descr_reduce(self, space):
         w_map = space.getattr(space.getbuiltinmodule('builtins'),
                 space.wrap('map'))
-        args = [self.w_fun] + self.iterators_w
-        return space.newtuple([w_map, space.newtuple(args)])
+        args_w = [self.w_fun] + self.iterators_w
+        return space.newtuple([w_map, space.newtuple(args_w)])
 
 
 def W_Map___new__(space, w_subtype, w_fun, args_w):
@@ -721,9 +721,9 @@
     def descr_reduce(self, space):
         w_filter = space.getattr(space.getbuiltinmodule('builtins'),
                 space.wrap('filter'))
-        args = [space.w_None if self.no_predicate else self.w_predicate,
-                self.iterable]
-        return space.newtuple([w_filter, space.newtuple(args)])
+        args_w = [space.w_None if self.no_predicate else self.w_predicate,
+                  self.iterable]
+        return space.newtuple([w_filter, space.newtuple(args_w)])
 
 
 def W_Filter___new__(space, w_subtype, w_predicate, w_iterable):
diff --git a/pypy/module/__builtin__/test/test_compile.py b/pypy/module/__builtin__/test/test_compile.py
--- a/pypy/module/__builtin__/test/test_compile.py
+++ b/pypy/module/__builtin__/test/test_compile.py
@@ -1,8 +1,7 @@
 class AppTestCompile:
 
-    # TODO: This test still fails for now because the docstrings are not
-    #       removed with optimize=2.
-    def untest_compile(self):
+    def test_compile(self):
+        """Clone of the part of the original test that was failing."""
         import ast
 
         codestr = '''def f():
@@ -37,7 +36,7 @@
                 assert rv == (debugval, docstring)
 
     def test_assert_remove(self):
-        """Test just removal of the asserts with optimize=1."""
+        """Test removal of the asserts with optimize=1."""
         import ast
 
         code = """def f():
@@ -50,9 +49,73 @@
             exec(compiled, ns)
             ns['f']()
 
+    def test_docstring_remove(self):
+        """Test removal of docstrings with optimize=2."""
+        import ast
+        import marshal
 
-# TODO: Remove docstrings with optimize=2.
+        code = """
+'module_doc'
+
+def f():
+    'func_doc'
+
+class C:
+    'class_doc'
+"""
+        tree = ast.parse(code)
+        for to_compile in [code, tree]:
+            compiled = compile(to_compile, "<test>", "exec", optimize=2)
+
+            ns = {}
+            exec(compiled, ns)
+            assert '__doc__' not in ns
+            assert ns['f'].__doc__ is None
+            assert ns['C'].__doc__ is None
+
+            # Check that the docstrings are gone from the bytecode and not just
+            # inaccessible.
+            marshalled = str(marshal.dumps(compiled))
+            assert 'module_doc' not in marshalled
+            assert 'func_doc' not in marshalled
+            assert 'class_doc' not in marshalled
+
+
+class TestOptimizeO:
+    """Test interaction of -O flag and optimize parameter of compile."""
+
+    def setup_method(self, method):
+        space = self.space
+        self._sys_debug = space.sys.debug
+        # imitate -O
+        space.sys.debug = False
+
+    def teardown_method(self, method):
+        self.space.sys.debug = self._sys_debug
+
+    def test_O_optmize_0(self):
+        """Test that assert is not ignored if -O flag is set but optimize=0."""
+        space = self.space
+        w_res = space.appexec([], """():
+            assert False  # check that our -O imitation hack works
+            try:
+                exec(compile('assert False', '', 'exec', optimize=0))
+            except AssertionError:
+                return True
+            else:
+                return False
+        """)
+        assert space.unwrap(w_res)
+
+    def test_O_optimize__1(self):
+        """Test that assert is ignored with -O and optimize=-1."""
+        space = self.space
+        space.appexec([], """():
+            exec(compile('assert False', '', 'exec', optimize=-1))
+        """)
+
+
 # TODO: Check the value of __debug__ inside of the compiled block!
 #       According to the documentation, it should follow the optimize flag.
-# TODO: It would also be good to test that with the assert is not removed and
-#       is executed when -O flag is set but optimize=0.
+#       However, cpython3.5.0a0 behaves the same way as PyPy (__debug__ follows
+#       -O, -OO flags of the interpreter).
diff --git a/pypy/module/__builtin__/test/test_dir.py b/pypy/module/__builtin__/test/test_dir.py
--- a/pypy/module/__builtin__/test/test_dir.py
+++ b/pypy/module/__builtin__/test/test_dir.py
@@ -24,3 +24,82 @@
             def __dir__(self):
                 return 42
         raises(TypeError, dir, Foo())
+
+    def test_dir_traceback(self):
+        """Test dir() of traceback."""
+        try:
+            raise IndexError
+        except Exception as e:
+            tb_dir = dir(e.__traceback__)
+            assert tb_dir == ['tb_frame', 'tb_lasti', 'tb_lineno', 'tb_next']
+
+    def test_dir_object_inheritance(self):
+        """Dir should behave the same regardless of inheriting from object."""
+        class A:
+            pass
+
+        class B(object):
+            pass
+        assert dir(A) == dir(B)
+
+    def test_dir_sanity(self):
+        """Test that dir returns reasonable items."""
+        class A(object):
+            a = 1
+
+        class B(A):
+            y = 2
+
+        b = B()
+        b.z = 1
+
+        names = dir(b)
+        for name in 'ayz':
+            assert name in names
+
+        assert '__doc__' in names
+        assert '__module__' in names
+        assert '__dict__' in names
+        assert '__dir__' in names
+        assert '__weakref__' in names
+        assert '__class__' in names
+        assert '__format__' in names
+        # Not an exhaustive list, but will be enough if dir is very broken.
+
+    def test_dir_module(self):
+        import sys
+        assert dir(sys) == list(sorted(sys.__dict__))
+
+    def test_dir_list(self):
+        """Check that dir([]) has methods from list and from object."""
+        names = dir([])
+
+        dct = {}
+        dct.update(list.__dict__)
+        dct.update(object.__dict__)
+
+        assert names == sorted(dct)
+
+    def test_dir_builtins(self):
+        """Test that builtin objects have sane __dir__()."""
+        import sys
+
+        for builtin in [sys, object(), [], {}, {1}, "", 1, (), sys,
+                map(ord, "abc"), filter(None, "abc"), zip([1, 2], [3, 4]),
+                compile('1', '', 'exec')]:
+            assert sorted(builtin.__dir__()) == dir(builtin)
+
+    def test_dir_type(self):
+        """Test .__dir__() and dir(...) behavior on types.
+
+        * t.__dir__() throws a TypeError,
+        * dir(t) == sorted(t().__dir__())
+
+        This is the behavior that I observe with cpython3.3.2.
+        """
+        for t in [int, list, tuple, set, str]:
+            raises(TypeError, t.__dir__)
+            assert dir(t) == sorted(t().__dir__())
+
+    def test_dir_none(self):
+        assert dir(None) == sorted(None.__dir__())
diff --git a/pypy/module/__builtin__/test/test_format.py b/pypy/module/__builtin__/test/test_format.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/__builtin__/test/test_format.py
@@ -0,0 +1,38 @@
+class AppTestFormat:
+
+    def test_format(self):
+        """Test deprecation warnings from format(object(), 'nonempty')"""
+
+        import warnings
+
+        def test_deprecated(obj, fmt_str, should_raise_warning):
+            with warnings.catch_warnings(record=True) as w:
+                warnings.simplefilter("always", DeprecationWarning)
+                format(obj, fmt_str)
+            if should_raise_warning:
+                assert len(w) == 1
+                assert isinstance(w[0].message, DeprecationWarning)
+                assert 'object.__format__ with a non-empty format string '\
+                        in str(w[0].message)
+            else:
+                assert len(w) == 0
+
+        fmt_strs = ['', 's']
+
+        class A:
+            def __format__(self, fmt_str):
+                return format('', fmt_str)
+
+        for fmt_str in fmt_strs:
+            test_deprecated(A(), fmt_str, False)
+
+        class B:
+            pass
+
+        class C(object):
+            pass
+
+        for cls in [object, B, C]:
+            for fmt_str in fmt_strs:
+                print(cls, fmt_str)
+                test_deprecated(cls(), fmt_str, len(fmt_str) != 0)
diff --git a/pypy/module/_io/interp_textio.py b/pypy/module/_io/interp_textio.py
--- a/pypy/module/_io/interp_textio.py
+++ b/pypy/module/_io/interp_textio.py
@@ -287,7 +287,8 @@
     try:
         w_locale = space.call_method(space.builtin, '__import__',
                                      space.wrap('locale'))
-        w_encoding = space.call_method(w_locale, 'getpreferredencoding')
+        w_encoding = space.call_method(w_locale, 'getpreferredencoding',
+                                       space.w_False)
     except OperationError as e:
         # getpreferredencoding() may also raise ImportError
         if not e.match(space, space.w_ImportError):
diff --git a/pypy/module/_lzma/__init__.py b/pypy/module/_lzma/__init__.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_lzma/__init__.py
@@ -0,0 +1,20 @@
+from pypy.interpreter.mixedmodule import MixedModule
+
+class Module(MixedModule):
+    # The private part of the lzma module.
+
+    applevel_name = '_lzma'
+
+    interpleveldefs = {
+        'LZMACompressor': 'interp_lzma.W_LZMACompressor',
+        'LZMADecompressor': 'interp_lzma.W_LZMADecompressor',
+        '_encode_filter_properties': 'interp_lzma.encode_filter_properties',
+        '_decode_filter_properties': 'interp_lzma.decode_filter_properties',
+        'FORMAT_AUTO': 'space.wrap(interp_lzma.FORMAT_AUTO)',
+        'FORMAT_XZ': 'space.wrap(interp_lzma.FORMAT_XZ)',
+        'FORMAT_ALONE': 'space.wrap(interp_lzma.FORMAT_ALONE)',
+        'FORMAT_RAW': 'space.wrap(interp_lzma.FORMAT_RAW)',
+    }
+
+    appleveldefs = {
+    }
diff --git a/pypy/module/_lzma/interp_lzma.py b/pypy/module/_lzma/interp_lzma.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/_lzma/interp_lzma.py
@@ -0,0 +1,360 @@
+from pypy.interpreter.baseobjspace import W_Root
+from pypy.interpreter.typedef import (
+    TypeDef, interp_attrproperty_bytes, interp_attrproperty)
+from pypy.interpreter.error import oefmt
+from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
+from pypy.module.thread.os_lock import Lock
+from rpython.rlib.objectmodel import specialize
+from rpython.rlib.rarithmetic import LONGLONG_MASK, r_ulonglong
+from rpython.rtyper.tool import rffi_platform as platform
+from rpython.rtyper.lltypesystem import rffi
+from rpython.rtyper.lltypesystem import lltype
+from rpython.translator.tool.cbuild import ExternalCompilationInfo
+
+
+FORMAT_AUTO, FORMAT_XZ, FORMAT_ALONE, FORMAT_RAW = range(4)
+R_LONGLONG_MASK = r_ulonglong(LONGLONG_MASK)
+
+
+eci = ExternalCompilationInfo(
+    includes = ['lzma.h'],
+    libraries = ['lzma'],
+    )
+eci = platform.configure_external_library(
+    'lzma', eci,
+    [dict(prefix='lzma-')])
+if not eci:
+    raise ImportError("Could not find lzma library")
+
+
+class CConfig:
+    _compilation_info_ = eci
+    calling_conv = 'c'
+
+    BUFSIZ = platform.ConstantInteger("BUFSIZ")
+
+    lzma_stream = platform.Struct(
+        'lzma_stream',
+        [('next_in', rffi.CCHARP),
+         ('avail_in', rffi.UINT),
+         ('total_in', rffi.UINT),
+         ('next_out', rffi.CCHARP),
+         ('avail_out', rffi.UINT),
+         ('total_out', rffi.UINT),
+         ])
+
+    lzma_options_lzma = platform.Struct(
+        'lzma_options_lzma',
+        [])
+
+constant_names = '''
+    LZMA_RUN LZMA_FINISH
+    LZMA_OK LZMA_GET_CHECK LZMA_NO_CHECK LZMA_STREAM_END
+    LZMA_PRESET_DEFAULT
+    LZMA_CHECK_ID_MAX
+    LZMA_TELL_ANY_CHECK LZMA_TELL_NO_CHECK
+    '''.split()
+for name in constant_names:
+    setattr(CConfig, name, platform.ConstantInteger(name))
+
+class cConfig(object):
+    pass
+for k, v in platform.configure(CConfig).items():
+    setattr(cConfig, k, v)
+
+for name in constant_names:
+    globals()[name] = getattr(cConfig, name)
+lzma_stream = lltype.Ptr(cConfig.lzma_stream)
+lzma_options_lzma = lltype.Ptr(cConfig.lzma_options_lzma)
+BUFSIZ = cConfig.BUFSIZ
+LZMA_CHECK_UNKNOWN = LZMA_CHECK_ID_MAX + 1
+
+def external(name, args, result, **kwds):
+    return rffi.llexternal(name, args, result, compilation_info=
+                           CConfig._compilation_info_, **kwds)
+
+lzma_ret = rffi.INT
+lzma_action = rffi.INT
+lzma_bool = rffi.INT
+
+lzma_lzma_preset = external('lzma_lzma_preset', [lzma_options_lzma, rffi.UINT], lzma_bool)
+lzma_alone_encoder = external('lzma_alone_encoder', [lzma_stream, lzma_options_lzma], lzma_ret)
+lzma_end = external('lzma_end', [lzma_stream], lltype.Void, releasegil=False)
+
+lzma_auto_decoder = external('lzma_auto_decoder', [lzma_stream, rffi.LONG, rffi.INT], lzma_ret)
+lzma_get_check = external('lzma_get_check', [lzma_stream], rffi.INT)
+
+lzma_code = external('lzma_code', [lzma_stream, lzma_action], rffi.INT)
+
+
+ at specialize.arg(1)
+def raise_error(space, fmt, *args):
+    raise oefmt(space.w_RuntimeError, fmt, *args)
+
+
+def _catch_lzma_error(space, lzret):
+    if (lzret == LZMA_OK or lzret == LZMA_GET_CHECK or
+        lzret == LZMA_NO_CHECK or lzret == LZMA_STREAM_END):
+        return
+    raise raise_error(space, "Unrecognized error from liblzma: %d", lzret)
+
+
+if BUFSIZ < 8192:
+    SMALLCHUNK = 8192
+else:
+    SMALLCHUNK = BUFSIZ
+if rffi.sizeof(rffi.INT) > 4:
+    BIGCHUNK = 512 * 32
+else:
+    BIGCHUNK = 512 * 1024
+
+
+def _new_buffer_size(current_size):
+    # keep doubling until we reach BIGCHUNK; then the buffer size is no
+    # longer increased
+    if current_size < BIGCHUNK:
+        return current_size + current_size
+    return current_size
+
+
+class OutBuffer(object):
+    """Handler for the output buffer.  A bit custom code trying to
+    encapsulate the logic of setting up the fields of 'lzs' and
+    allocating raw memory as needed.
+    """
+    def __init__(self, lzs, initial_size=SMALLCHUNK):
+        # when the constructor is called, allocate a piece of memory
+        # of length 'piece_size' and make lzs ready to dump there.
+        self.temp = []
+        self.lzs = lzs
+        self._allocate_chunk(initial_size)
+
+    def _allocate_chunk(self, size):
+        self.raw_buf, self.gc_buf = rffi.alloc_buffer(size)
+        self.current_size = size
+        self.lzs.c_next_out = self.raw_buf
+        rffi.setintfield(self.lzs, 'c_avail_out', size)
+
+    def _get_chunk(self, chunksize):
+        assert 0 <= chunksize <= self.current_size
+        raw_buf = self.raw_buf
+        gc_buf = self.gc_buf
+        s = rffi.str_from_buffer(raw_buf, gc_buf, self.current_size, chunksize)
+        rffi.keep_buffer_alive_until_here(raw_buf, gc_buf)
+        self.current_size = 0
+        return s
+
+    def prepare_next_chunk(self):
+        size = self.current_size
+        self.temp.append(self._get_chunk(size))
+        self._allocate_chunk(_new_buffer_size(size))
+
+    def make_result_string(self):
+        count_unoccupied = rffi.getintfield(self.lzs, 'c_avail_out')
+        s = self._get_chunk(self.current_size - count_unoccupied)
+        if self.temp:
+            self.temp.append(s)
+            return ''.join(self.temp)
+        else:
+            return s
+
+    def free(self):
+        if self.current_size > 0:
+            rffi.keep_buffer_alive_until_here(self.raw_buf, self.gc_buf)
+
+    def __enter__(self):
+        return self
+    def __exit__(self, *args):
+        self.free()
+
+
+class W_LZMACompressor(W_Root):
+    def __init__(self, space, format):
+        self.format = format
+        self.lock = Lock(space)
+        self.flushed = False
+        self.lzs = lltype.malloc(lzma_stream.TO, flavor='raw', zero=True)
+
+    def __del__(self):
+        lzma_end(self.lzs)
+        lltype.free(self.lzs, flavor='raw')
+
+    def _init_alone(self, space, preset, w_filters):
+        if space.is_none(w_filters):
+            with lltype.scoped_alloc(lzma_options_lzma.TO) as options:
+                if lzma_lzma_preset(options, preset):
+                    raise_error(space, "Invalid compression preset: %d", preset)
+                lzret = lzma_alone_encoder(self.lzs, options)
+        else:
+            raise NotImplementedError
+        _catch_lzma_error(space, lzret)
+
+    @staticmethod
+    @unwrap_spec(format=int,
+                 w_check=WrappedDefault(None),


More information about the pypy-commit mailing list