Abstract

The current status of PyPy, with a particular focus on what happened in +the last two years, since the last EuroPython PyPy talk. We will give a +brief overview of the current speed and the on-going development efforts +on the JIT, the GC, NumPy, Python 3 compatibility, CFFI, STM...

Abstract

PyPy, the Python implementation written in Python, experimentally +supports Transactional Memory (TM). The strength of TM is to enable a +novel use of multithreading, inheritently safe, and not limited to +special use cases like other approaches. This talk will focus on how it +works under the hood.

Demo 1

Branch: reflex-support Changeset: r72456:0de503991f11 Date: 2014-07-16 12:16 -0700 http://bitbucket.org/pypy/pypy/changeset/0de503991f11/ Log: cling fixes diff --git a/pypy/module/cppyy/capi/builtin_capi.py b/pypy/module/cppyy/capi/builtin_capi.py --- a/pypy/module/cppyy/capi/builtin_capi.py +++ b/pypy/module/cppyy/capi/builtin_capi.py @@ -3,6 +3,7 @@ import reflex_capi as backend #import cint_capi as backend +#import cling_capi as backend from pypy.module.cppyy.capi.capi_types import C_SCOPE, C_TYPE, C_OBJECT,\ C_METHOD, C_INDEX, C_INDEX_ARRAY, WLAVC_INDEX,\ diff --git a/pypy/module/cppyy/capi/cling_capi.py b/pypy/module/cppyy/capi/cling_capi.py --- a/pypy/module/cppyy/capi/cling_capi.py +++ b/pypy/module/cppyy/capi/cling_capi.py @@ -14,12 +14,13 @@ (config_stat, incdir) = commands.getstatusoutput("root-config --incdir") if os.environ.get("ROOTSYS"): - if config_stat != 0: # presumably Reflex-only - rootincpath = [os.path.join(os.environ["ROOTSYS"], "interpreter/cling/include"), - os.path.join(os.environ["ROOTSYS"], "interpreter/llvm/inst/include")] + rootincpath = [os.path.join(os.environ["ROOTSYS"], "interpreter/cling/include"), + os.path.join(os.environ["ROOTSYS"], "interpreter/llvm/inst/include")] + if config_stat != 0: + rootincpath.append(os.path.join(os.environ["ROOTSYS"], "include")) rootlibpath = [os.path.join(os.environ["ROOTSYS"], "lib64"), os.path.join(os.environ["ROOTSYS"], "lib")] else: - rootincpath = [incdir] + rootincpath.append(incdir) rootlibpath = commands.getoutput("root-config --libdir").split() else: if config_stat == 0: @@ -45,7 +46,7 @@ includes=["clingcwrapper.h"], library_dirs=rootlibpath, libraries=["Cling"], - compile_extra=["-fno-strict-aliasing"], + compile_extra=["-fno-strict-aliasing", "-std=c++11"], use_cpp_linker=True, ) diff --git a/pypy/module/cppyy/src/clingcwrapper.cxx b/pypy/module/cppyy/src/clingcwrapper.cxx --- a/pypy/module/cppyy/src/clingcwrapper.cxx +++ b/pypy/module/cppyy/src/clingcwrapper.cxx @@ -21,7 +21,7 @@ #include "cling/Interpreter/DynamicLibraryManager.h" #include "cling/Interpreter/Interpreter.h" #include "cling/Interpreter/LookupHelper.h" -#include "cling/Interpreter/StoredValueRef.h" +#include "cling/Interpreter/Value.h" #include "cling/MetaProcessor/MetaProcessor.h" #include "llvm/ADT/SmallVector.h" @@ -157,7 +157,8 @@ cppyy_scope_t cppyy_get_scope(const char* scope_name) { const cling::LookupHelper& lh = gCppyy_Cling->getLookupHelper(); const Type* type = 0; - const Decl* decl = lh.findScope(scope_name, &type, /* intantiateTemplate= */ true); + const Decl* decl = lh.findScope(scope_name, + cling::LookupHelper::NoDiagnostics, &type, /* intantiateTemplate= */ true); if (!decl) { //std::string buf = TClassEdit::InsertStd(name); //decl = lh.findScope(buf, &type, /* intantiateTemplate= */ true); @@ -190,8 +191,8 @@ // TODO: expect the below to live in libCling.so static CPPYY_Cling_Wrapper_t make_wrapper(const FunctionDecl* fdecl); -static void exec_with_valref_return(void* address, cling::StoredValueRef* ret, const FunctionDecl*); -static long long sv_to_long_long(const cling::StoredValueRef& svref); +static void exec_with_valref_return(void* address, cling::Value*, const FunctionDecl*); +static long long sv_to_long_long(const cling::Value& val); // -- TODO: expect the above to live in libCling.so @@ -200,7 +201,7 @@ if (s_wrappers.find(method) == s_wrappers.end()) { make_wrapper((FunctionDecl*)method); } - cling::StoredValueRef ret; + cling::Value ret; // std::vector arguments = build_args(nargs, args); // CPPYY_Cling_Wrapper_t cb = (CPPYY_Cling_Wrapper_t)method; exec_with_valref_return((void*)self, &ret, (FunctionDecl*)method); @@ -362,7 +363,7 @@ // R__LOCKGUARD2(gInterpreterMutex); std::cout << " NOW LOADING: " << lib_name << std::endl; - cling::StoredValueRef call_res; + cling::Value call_res; cling::Interpreter::CompilationResult comp_res = cling::Interpreter::kSuccess; std::ostringstream line; line << "#include \"" << lib_name << ".h\""; @@ -377,8 +378,8 @@ // UpdateListOfLoadedSharedLibraries(); // } switch (res) { - case cling::DynamicLibraryManager::kLoadLibSuccess: return (void*)1; - case cling::DynamicLibraryManager::kLoadLibExists: return (void*)2; + case cling::DynamicLibraryManager::kLoadLibSuccess: return (void*)1; + case cling::DynamicLibraryManager::kLoadLibAlreadyLoaded: return (void*)2; default: break; }; return (void*)1; @@ -389,68 +390,130 @@ // TODO: expect the below to live in libCling.so -template -T sv_to_long_long_u_or_not(const cling::StoredValueRef& svref) { - const cling::Value& valref = svref.get(); - QualType QT = valref.getClangType(); - if (QT.isNull()) { - print_error("sv_to_long_long_u_or_not", "null type!"); - return 0; - } - llvm::GenericValue gv = valref.getGV(); - if (QT->isMemberPointerType()) { - const MemberPointerType* MPT = - QT->getAs(); - if (MPT->isMemberDataPointer()) { - return (T) (ptrdiff_t) gv.PointerVal; - } - return (T) gv.PointerVal; - } - if (QT->isPointerType() || QT->isArrayType() || QT->isRecordType() || - QT->isReferenceType()) { - return (T) gv.PointerVal; - } - if (const EnumType* ET = llvm::dyn_cast(&*QT)) { - if (ET->getDecl()->getIntegerType()->hasSignedIntegerRepresentation()) - return (T) gv.IntVal.getSExtValue(); - else - return (T) gv.IntVal.getZExtValue(); - } - if (const BuiltinType* BT = llvm::dyn_cast(&*QT)) { - if (BT->isSignedInteger()) { - return gv.IntVal.getSExtValue(); - } else if (BT->isUnsignedInteger()) { - return (T) gv.IntVal.getZExtValue(); - } else { - switch (BT->getKind()) { - case BuiltinType::Float: - return (T) gv.FloatVal; - case BuiltinType::Double: - return (T) gv.DoubleVal; - case BuiltinType::LongDouble: - // FIXME: Implement this! - break; - case BuiltinType::NullPtr: - // C++11 nullptr - return 0; - default: break; - } +template +returnType sv_to(const cling::Value& val) +{ + QualType QT = val.getType(); + if (QT->isMemberPointerType()) { + const MemberPointerType* MPT = QT->getAs(); + if (MPT->isMemberDataPointer()) { + return (returnType) (ptrdiff_t)val.getPtr(); } - } - print_error("sv_to_long_long_u_or_not", "cannot handle this type!"); - QT->dump(); - return 0; -} + return (returnType) (long) val.getPtr(); + } + if (QT->isPointerType() || QT->isArrayType() || QT->isRecordType() || + QT->isReferenceType()) { + return (returnType) (long) val.getPtr(); + } + if (const EnumType* ET = dyn_cast(&*QT)) { + if (ET->getDecl()->getIntegerType()->hasSignedIntegerRepresentation()) + return (returnType) val.getLL(); + else + return (returnType) val.getULL(); + } + if (const BuiltinType* BT = + dyn_cast(&*QT)) { + // + // WARNING!!! + // + // This switch is organized in order-of-declaration + // so that the produced assembly code is optimal. + // Do not reorder! + // + switch (BT->getKind()) { + case BuiltinType::Void: + // CINT used to expect a result of 0. + return (returnType) 0; + break; + // + // Unsigned Types + // + case BuiltinType::Bool: + case BuiltinType::Char_U: // char on targets where it is unsigned + case BuiltinType::UChar: + return (returnType) val.getULL(); + break; -static long long sv_to_long_long(const cling::StoredValueRef& svref) { - return sv_to_long_long_u_or_not(svref); + case BuiltinType::WChar_U: + // wchar_t on targets where it is unsigned + // The standard doesn't allow to specify signednedd of wchar_t + // thus this maps simply to wchar_t. + return (returnType) (wchar_t) val.getULL(); + break; + + case BuiltinType::Char16: + case BuiltinType::Char32: + case BuiltinType::UShort: + case BuiltinType::UInt: + case BuiltinType::ULong: + case BuiltinType::ULongLong: + return (returnType) val.getULL(); + break; + + case BuiltinType::UInt128: + // __uint128_t + break; + + // + // Signed Types + // + case BuiltinType::Char_S: // char on targets where it is signed + case BuiltinType::SChar: + return (returnType) val.getLL(); + break; + + case BuiltinType::WChar_S: + // wchar_t on targets where it is signed + // The standard doesn't allow to specify signednedd of wchar_t + // thus this maps simply to wchar_t. + return (returnType) (wchar_t) val.getLL(); + break; + + case BuiltinType::Short: + case BuiltinType::Int: + case BuiltinType::Long: + case BuiltinType::LongLong: + return (returnType) val.getLL(); + break; + + case BuiltinType::Int128: + break; + + case BuiltinType::Half: + // half in OpenCL, __fp16 in ARM NEON + break; + + case BuiltinType::Float: + return (returnType) val.getFloat(); + break; + case BuiltinType::Double: + return (returnType) val.getDouble(); + break; + case BuiltinType::LongDouble: + return (returnType) val.getLongDouble(); + break; + + case BuiltinType::NullPtr: + return (returnType) 0; + break; + + default: + break; + } + } + print_error("sv_to", "invalid type"); + QT->dump(); + return 0; } static -unsigned long long sv_to_ulong_long(const cling::StoredValueRef& svref) { - return sv_to_long_long_u_or_not(svref); +long long sv_to_long_long(const cling::Value& val) { + return sv_to(val); } - +static +unsigned long long sv_to_ulong_long(const cling::Value& val) { + return sv_to(val); +} namespace { @@ -772,230 +835,168 @@ (*wrapper)(address, (int)0/*num_args*/, (void**)vp_ary.data(), ret); } +static void +exec_with_valref_return(void* address, cling::Value* ret) const +{ + if (!ret) { + exec(address, 0); + return; + } + std::cout << " USING DECL: " << fdecl << std::endl; + const FunctionDecl* FD = fMethod->GetMethodDecl(); + ASTContext& Context = FD->getASTContext(); -static void exec_with_valref_return(void* address, cling::StoredValueRef* ret, const FunctionDecl* fdecl) { - if (!ret) { - exec(address, 0, fdecl); - return; - } - std::cout << " USING DECL: " << fdecl << std::endl; - fdecl->dump(); - ASTContext& Context = fdecl->getASTContext(); + if (const CXXConstructorDecl* CD = dyn_cast(FD)) { + const TypeDecl* TD = dyn_cast(CD->getDeclContext()); + QualType ClassTy(TD->getTypeForDecl(), 0); + QualType QT = Context.getLValueReferenceType(ClassTy); + *ret = cling::Value(QT, fInterp); + // Store the new()'ed address in getPtr() + exec(address, &ret->getPtr()); + return; + } + QualType QT = FD->getReturnType().getCanonicalType(); + if (QT->isReferenceType()) { + *ret = cling::Value(QT, 0); + exec(address, &ret->getPtr()); + return; + } + else if (QT->isMemberPointerType()) { + const MemberPointerType* MPT = QT->getAs(); + if (MPT->isMemberDataPointer()) { + // A member data pointer is a actually a struct with one + // member of ptrdiff_t, the offset from the base of the object + // storage to the storage for the designated data member. + // But that's not relevant: we use it as a non-builtin, allocated + // type. + *ret = cling::Value(QT, fInterp); + exec(address, ret->getPtr()); + return; + } + // We are a function member pointer. + *ret = cling::Value(QT, fInterp); + exec(address, &ret->getPtr()); + return; + } + else if (QT->isPointerType() || QT->isArrayType()) { + // Note: ArrayType is an illegal function return value type. + *ret = cling::Value(QT, 0); + exec(address, &ret->getPtr()); + return; + } + else if (QT->isRecordType()) { + *ret = cling::Value(QT, fInterp); + exec(address, ret->getPtr()); + return; + } + else if (const EnumType* ET = dyn_cast(&*QT)) { + // Note: We may need to worry about the underlying type + // of the enum here. + (void) ET; + *ret = cling::Value(QT, 0); + execWithLL(address, QT, ret); + return; + } + else if (const BuiltinType* BT = dyn_cast(&*QT)) { + *ret = cling::Value(QT, 0); + switch (BT->getKind()) { + case BuiltinType::Void: + exec(address, 0); + break; - if (const CXXConstructorDecl* CD = llvm::dyn_cast(fdecl)) { - const TypeDecl* TD = llvm::dyn_cast(CD->getDeclContext()); - QualType ClassTy(TD->getTypeForDecl(), 0); - QualType QT = Context.getLValueReferenceType(ClassTy); - llvm::GenericValue gv; - exec(address, &gv.PointerVal, fdecl); - *ret = cling::StoredValueRef::bitwiseCopy( - *gCppyy_Cling, cling::Value(gv, QT)); - return; - } - QualType QT = fdecl->getResultType().getCanonicalType(); - if (QT->isReferenceType()) { - llvm::GenericValue gv; - exec(address, &gv.PointerVal, fdecl); - *ret = cling::StoredValueRef::bitwiseCopy( - *gCppyy_Cling, cling::Value(gv, QT)); - return; - } - else if (QT->isMemberPointerType()) { - const MemberPointerType* MPT = - QT->getAs(); - if (MPT->isMemberDataPointer()) { - // A member data pointer is a actually a struct with one - // member of ptrdiff_t, the offset from the base of the object - // storage to the storage for the designated data member. - llvm::GenericValue gv; - exec(address, &gv.PointerVal, fdecl); - *ret = cling::StoredValueRef::bitwiseCopy( - *gCppyy_Cling, cling::Value(gv, QT)); - return; - } - // We are a function member pointer. - llvm::GenericValue gv; - exec(address, &gv.PointerVal, fdecl); - *ret = cling::StoredValueRef::bitwiseCopy( - *gCppyy_Cling, cling::Value(gv, QT)); - return; - } - else if (QT->isPointerType() || QT->isArrayType()) { - // Note: ArrayType is an illegal function return value type. - llvm::GenericValue gv; - exec(address, &gv.PointerVal, fdecl); - *ret = cling::StoredValueRef::bitwiseCopy( - *gCppyy_Cling, cling::Value(gv, QT)); - return; - } - else if (QT->isRecordType()) { - uint64_t size = Context.getTypeSizeInChars(QT).getQuantity(); - void* p = ::operator new(size); - exec(address, p, fdecl); - *ret = cling::StoredValueRef::bitwiseCopy( - *gCppyy_Cling, cling::Value(llvm::PTOGV(p), QT)); - return; - } - else if (const EnumType* ET = llvm::dyn_cast(&*QT)) { - // Note: We may need to worry about the underlying type - // of the enum here. - (void) ET; - uint64_t numBits = Context.getTypeSize(QT); - int retVal = 0; - exec(address, &retVal, fdecl); - llvm::GenericValue gv; - gv.IntVal = llvm::APInt(numBits, (uint64_t)retVal, true /*isSigned*/); - *ret = cling::StoredValueRef::bitwiseCopy( - *gCppyy_Cling, cling::Value(gv, QT)); - return; - } - else if (const BuiltinType* BT = llvm::dyn_cast(&*QT)) { - llvm::GenericValue gv; + // + // Unsigned Types + // + case BuiltinType::Bool: + execWithULL(address, QT, ret); + break; + case BuiltinType::Char_U: // char on targets where it is unsigned + case BuiltinType::UChar: + execWithULL(address, QT, ret); + break; + case BuiltinType::WChar_U: + // wchar_t on targets where it is unsigned. + // The standard doesn't allow to specify signednedd of wchar_t + // thus this maps simply to wchar_t. + execWithULL(address, QT, ret); + break; + case BuiltinType::Char16: + print_error("exec_with_valref_return", "invalid type: char16_t"); + break; + case BuiltinType::Char32: + print_error("exec_with_valref_return", "invalid type char32_t"); + break; + case BuiltinType::UShort: + execWithULL(address, QT, ret); + break; + case BuiltinType::UInt: + execWithULL(address, QT, ret); + break; + case BuiltinType::ULong: + execWithULL(address, QT, ret); + break; + case BuiltinType::ULongLong: + execWithULL(address, QT, ret); + break; + case BuiltinType::UInt128: + print_error("exec_with_valref_return", "invalid type __uint128_t"); + break; - uint64_t numBits = Context.getTypeSize(QT); - switch (BT->getKind()) { - // - // builtin types - // - case BuiltinType::Void: { - exec(address, 0, fdecl); - return; - } - // - // unsigned integral types - // - case BuiltinType::Bool: { - bool retVal = false; - exec(address, &retVal, fdecl); - gv.IntVal = llvm::APInt(numBits, (uint64_t)retVal, false /*isSigned*/); + // + // Signed Types + // + case BuiltinType::Char_S: // char on targets where it is signed + case BuiltinType::SChar: + execWithLL(address, QT, ret); break; - } - case BuiltinType::Char_U: { - // char on targets where it is unsigned - char retVal = '\0'; - exec(address, &retVal, fdecl); - gv.IntVal = llvm::APInt(numBits, (uint64_t) retVal, false /*isSigned*/); - break; - } - case BuiltinType::UChar: { - unsigned char retVal = '\0'; - exec(address, &retVal, fdecl); - gv.IntVal = llvm::APInt(numBits, (uint64_t) retVal, false /*isSigned*/); - break; - } - case BuiltinType::WChar_U: { - // wchar_t on targets where it is unsigned. - // The standard doesn't allow to specify signedness of wchar_t - // thus this maps simply to wchar_t. - wchar_t retVal = L'\0'; - exec(address, &retVal, fdecl); - gv.IntVal = llvm::APInt(numBits, (uint64_t) retVal, false /*isSigned*/); - break; - } - case BuiltinType::UShort: { - unsigned short retVal = 0; - exec(address, &retVal, fdecl); - gv.IntVal = llvm::APInt(numBits, (uint64_t) retVal, false /*isSigned*/); - break; - } - case BuiltinType::UInt: { - unsigned int retVal = 0; - exec(address, &retVal, fdecl); - gv.IntVal = llvm::APInt(numBits, (uint64_t) retVal, false /*isSigned*/); - break; - } - case BuiltinType::ULong: { - // unsigned long - unsigned long retVal = 0; - exec(address, &retVal, fdecl); - gv.IntVal = llvm::APInt(numBits, (uint64_t) retVal, false /*isSigned*/); - break; - } - case BuiltinType::ULongLong: { - // unsigned long long - unsigned long long retVal = 0; - exec(address, &retVal, fdecl); - gv.IntVal = llvm::APInt(numBits, (uint64_t) retVal, false /*isSigned*/); - break; - } - // - // signed integral types - // - case BuiltinType::Char_S: { - // char on targets where it is signed - char retVal = '\0'; - exec(address, &retVal, fdecl); - gv.IntVal = llvm::APInt(numBits, (uint64_t) retVal, true /*isSigned*/); - break; - } - case BuiltinType::SChar: { - // signed char - signed char retVal = '\0'; - exec(address, &retVal, fdecl); - gv.IntVal = llvm::APInt(numBits, (uint64_t) retVal, true /*isSigned*/); - break; - } - case BuiltinType::WChar_S: { + case BuiltinType::WChar_S: // wchar_t on targets where it is signed. // The standard doesn't allow to specify signednedd of wchar_t // thus this maps simply to wchar_t. - wchar_t retVal = L'\0'; - exec(address, &retVal, fdecl); - gv.IntVal = llvm::APInt(numBits, (uint64_t) retVal, true /*isSigned*/); + execWithLL(address, QT, ret); break; - } - case BuiltinType::Short: { - // short - short retVal = 0; - exec(address, &retVal, fdecl); - gv.IntVal = llvm::APInt(numBits, (uint64_t) retVal, true /*isSigned*/); + case BuiltinType::Short: + execWithLL(address, QT, ret); break; - } - case BuiltinType::Int: { - // int - int retVal = 0; - exec(address, &retVal, fdecl); - gv.IntVal = llvm::APInt(numBits, (uint64_t) retVal, true /*isSigned*/); + case BuiltinType::Int: + execWithLL(address, QT, ret); break; - } - case BuiltinType::Long: { - long retVal = 0; - exec(address, &retVal, fdecl); - gv.IntVal = llvm::APInt(numBits, (uint64_t) retVal, true /*isSigned*/); + case BuiltinType::Long: + execWithLL(address, QT, ret); break; - } - case BuiltinType::LongLong: { - long long retVal = 0; - exec(address, &retVal, fdecl); - gv.IntVal = llvm::APInt(numBits, (uint64_t) retVal, true /*isSigned*/); + case BuiltinType::LongLong: + execWithLL(address, QT, ret); break; - } - case BuiltinType::Float: { - exec(address, &gv.FloatVal, fdecl); + case BuiltinType::Int128: + print_error("exec_with_valref_return", "invalid type __int128_t"); break; - } - case BuiltinType::Double: { - exec(address, &gv.DoubleVal, fdecl); + case BuiltinType::Half: + // half in OpenCL, __fp16 in ARM NEON + print_error("exec_with_valref_return", "invalid type Half"); break; - } - case BuiltinType::Char16: - case BuiltinType::Char32: - case BuiltinType::Half: - case BuiltinType::Int128: - case BuiltinType::UInt128: - case BuiltinType::LongDouble: - case BuiltinType::NullPtr: - default: { - print_error("exec_with_valref", "unsupported return type"); - return; - } - } - - *ret = cling::StoredValueRef::bitwiseCopy(*gCppyy_Cling, cling::Value(gv, QT)); - return; - } - - std::cout << "exec_with_valref: some error occurred ... " << std::endl; + case BuiltinType::Float: + exec(address, &ret->getFloat()); + break; + case BuiltinType::Double: + exec(address, &ret->getDouble()); + break; + case BuiltinType::LongDouble: + exec(address, &ret->getLongDouble()); + break; + // + // Language-Specific Types + // + case BuiltinType::NullPtr: + // C++11 nullptr + print_error("exec_with_valref_return", "invalid type nullptr"); + break; + default: + break; + } + return; + } + print_error("exec_with_valref_return", "unrecognized return type"); + QT->dump(); } From noreply at buildbot.pypy.org Wed Jul 16 21:17:00 2014 From: noreply at buildbot.pypy.org (wlav) Date: Wed, 16 Jul 2014 21:17:00 +0200 (CEST) Subject: [pypy-commit] pypy reflex-support: updates for handling of null-ptrs and enums Message-ID: <20140716191700.3341F1C33B9@cobra.cs.uni-duesseldorf.de> Author: Wim Lavrijsen Branch: reflex-support Changeset: r72457:6fd647e28232 Date: 2014-07-16 12:16 -0700 http://bitbucket.org/pypy/pypy/changeset/6fd647e28232/ Log: updates for handling of null-ptrs and enums diff --git a/pypy/module/cppyy/test/test_datatypes.py b/pypy/module/cppyy/test/test_datatypes.py --- a/pypy/module/cppyy/test/test_datatypes.py +++ b/pypy/module/cppyy/test/test_datatypes.py @@ -391,6 +391,15 @@ CppyyTestData.s_ldouble = math.pi assert c.s_ldouble == math.pi + # enum types + assert raises(AttributeError, getattr, CppyyTestData, 'kBanana') + if self.capi_identity == 'Cling': # detailed enum support only in Cling + assert raises(TypeError, setattr, CppyyTestData, 'kLots', 42) + c.s_enum = CppyyTestData.kLots + assert CppyyTestData.s_enum == CppyyTestData.kLots + CppyyTestData.s_enum = CppyyTestData.kNothing + assert c.s_enum == CppyyTestData.kNothing + c.__destruct__() def test07_range_access(self): @@ -772,8 +781,13 @@ def address_equality_test(a, b): assert cppyy.addressof(a) == cppyy.addressof(b) b2 = cppyy.bind_object(a, CppyyTestData) + b.m_int = 888 + assert b.m_int == 888 + assert b == b2 and b.m_int == b2.m_int assert b is b2 # memory regulator recycles b3 = cppyy.bind_object(cppyy.addressof(a), CppyyTestData) + assert b3.m_int == 888 + assert b == b3 and b.m_int == b3.m_int assert b is b3 # likewise address_equality_test(c.m_voidp, c2) diff --git a/pypy/module/cppyy/test/test_fragile.py b/pypy/module/cppyy/test/test_fragile.py --- a/pypy/module/cppyy/test/test_fragile.py +++ b/pypy/module/cppyy/test/test_fragile.py @@ -70,7 +70,7 @@ assert fragile.D().check() == ord('D') d = fragile.D() - raises(TypeError, d.overload, None) + raises(TypeError, d.overload, 1.) raises(TypeError, d.overload, None, None, None) d.overload('a') @@ -88,8 +88,10 @@ assert fragile.E().check() == ord('E') e = fragile.E() - raises(TypeError, e.overload, None) - raises(TypeError, getattr, e, 'm_pp_no_such') + # TODO: figure out the desired behavior here; right now, an opaque + # pointer is returned to allow passing back to C++ code + #raises(TypeError, e.overload, None) + #raises(TypeError, getattr, e, 'm_pp_no_such') def test05_wrong_arg_addressof(self): """Test addressof() error reporting""" @@ -184,17 +186,17 @@ assert "TypeError: wrong number of arguments" in str(e) try: - d.overload(None) # raises TypeError + d.overload(1.) # raises TypeError assert 0 except TypeError, e: assert "fragile::D::overload()" in str(e) assert "TypeError: wrong number of arguments" in str(e) assert "fragile::D::overload(fragile::no_such_class*)" in str(e) - assert "TypeError: no converter available for 'fragile::no_such_class*'" in str(e) + # assert "TypeError: no converter available for 'fragile::no_such_class*'" in str(e) assert "fragile::D::overload(char, int)" in str(e) - assert "TypeError: expected string, got NoneType object" in str(e) + assert "TypeError: expected string, got float object" in str(e) assert "fragile::D::overload(int, fragile::no_such_class*)" in str(e) - assert "TypeError: expected integer, got NoneType object" in str(e) + assert "TypeError: expected integer, got float object" in str(e) j = fragile.J() assert fragile.J.method1.__doc__ == j.method1.__doc__ From noreply at buildbot.pypy.org Wed Jul 16 22:55:29 2014 From: noreply at buildbot.pypy.org (pjenvey) Date: Wed, 16 Jul 2014 22:55:29 +0200 (CEST) Subject: [pypy-commit] pypy default: prefer space from inter2app func args Message-ID: <20140716205529.243A51C0091@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: Changeset: r72458:5b3b60ba4e33 Date: 2014-07-16 13:54 -0700 http://bitbucket.org/pypy/pypy/changeset/5b3b60ba4e33/ Log: prefer space from inter2app func args diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py --- a/pypy/module/_ssl/interp_ssl.py +++ b/pypy/module/_ssl/interp_ssl.py @@ -133,11 +133,11 @@ self._issuer[0] = '\0' self.shutdown_seen_zero = False - def server(self): - return self.space.wrap(rffi.charp2str(self._server)) + def server(self, space): + return space.wrap(rffi.charp2str(self._server)) - def issuer(self): - return self.space.wrap(rffi.charp2str(self._issuer)) + def issuer(self, space): + return space.wrap(rffi.charp2str(self._issuer)) def __del__(self): self.enqueue_for_destruction(self.space, SSLObject.destructor, @@ -155,21 +155,21 @@ lltype.free(self._issuer, flavor='raw') @unwrap_spec(data='bufferstr') - def write(self, data): + def write(self, space, data): """write(s) -> len Writes the string s into the SSL object. Returns the number of bytes written.""" - self._refresh_nonblocking(self.space) + self._refresh_nonblocking(space) - sockstate = check_socket_and_wait_for_timeout(self.space, + sockstate = check_socket_and_wait_for_timeout(space, self.w_socket, True) if sockstate == SOCKET_HAS_TIMED_OUT: - raise ssl_error(self.space, "The write operation timed out") + raise ssl_error(space, "The write operation timed out") elif sockstate == SOCKET_HAS_BEEN_CLOSED: - raise ssl_error(self.space, "Underlying socket has been closed.") + raise ssl_error(space, "Underlying socket has been closed.") elif sockstate == SOCKET_TOO_LARGE_FOR_SELECT: - raise ssl_error(self.space, "Underlying socket too large for select().") + raise ssl_error(space, "Underlying socket too large for select().") num_bytes = 0 while True: @@ -179,18 +179,18 @@ err = libssl_SSL_get_error(self.ssl, num_bytes) if err == SSL_ERROR_WANT_READ: - sockstate = check_socket_and_wait_for_timeout(self.space, + sockstate = check_socket_and_wait_for_timeout(space, self.w_socket, False) elif err == SSL_ERROR_WANT_WRITE: - sockstate = check_socket_and_wait_for_timeout(self.space, + sockstate = check_socket_and_wait_for_timeout(space, self.w_socket, True) else: sockstate = SOCKET_OPERATION_OK if sockstate == SOCKET_HAS_TIMED_OUT: - raise ssl_error(self.space, "The write operation timed out") + raise ssl_error(space, "The write operation timed out") elif sockstate == SOCKET_HAS_BEEN_CLOSED: - raise ssl_error(self.space, "Underlying socket has been closed.") + raise ssl_error(space, "Underlying socket has been closed.") elif sockstate == SOCKET_IS_NONBLOCKING: break @@ -200,38 +200,38 @@ break if num_bytes > 0: - return self.space.wrap(num_bytes) + return space.wrap(num_bytes) else: - raise _ssl_seterror(self.space, self, num_bytes) + raise _ssl_seterror(space, self, num_bytes) - def pending(self): + def pending(self, space): """pending() -> count Returns the number of already decrypted bytes available for read, pending on the connection.""" count = libssl_SSL_pending(self.ssl) if count < 0: - raise _ssl_seterror(self.space, self, count) - return self.space.wrap(count) + raise _ssl_seterror(space, self, count) + return space.wrap(count) @unwrap_spec(num_bytes=int) - def read(self, num_bytes=1024): + def read(self, space, num_bytes=1024): """read([len]) -> string Read up to len bytes from the SSL socket.""" count = libssl_SSL_pending(self.ssl) if not count: - sockstate = check_socket_and_wait_for_timeout(self.space, - self.w_socket, False) + sockstate = check_socket_and_wait_for_timeout(space, self.w_socket, + False) if sockstate == SOCKET_HAS_TIMED_OUT: - raise ssl_error(self.space, "The read operation timed out") + raise ssl_error(space, "The read operation timed out") elif sockstate == SOCKET_TOO_LARGE_FOR_SELECT: - raise ssl_error(self.space, "Underlying socket too large for select().") + raise ssl_error(space, "Underlying socket too large for select().") elif sockstate == SOCKET_HAS_BEEN_CLOSED: if libssl_SSL_get_shutdown(self.ssl) == SSL_RECEIVED_SHUTDOWN: - return self.space.wrap('') - raise ssl_error(self.space, "Socket closed without SSL shutdown handshake") + return space.wrap('') + raise ssl_error(space, "Socket closed without SSL shutdown handshake") with rffi.scoped_alloc_buffer(num_bytes) as buf: while True: @@ -241,19 +241,19 @@ err = libssl_SSL_get_error(self.ssl, count) if err == SSL_ERROR_WANT_READ: - sockstate = check_socket_and_wait_for_timeout(self.space, + sockstate = check_socket_and_wait_for_timeout(space, self.w_socket, False) elif err == SSL_ERROR_WANT_WRITE: - sockstate = check_socket_and_wait_for_timeout(self.space, + sockstate = check_socket_and_wait_for_timeout(space, self.w_socket, True) elif (err == SSL_ERROR_ZERO_RETURN and libssl_SSL_get_shutdown(self.ssl) == SSL_RECEIVED_SHUTDOWN): - return self.space.wrap("") + return space.wrap("") else: sockstate = SOCKET_OPERATION_OK if sockstate == SOCKET_HAS_TIMED_OUT: - raise ssl_error(self.space, "The read operation timed out") + raise ssl_error(space, "The read operation timed out") elif sockstate == SOCKET_IS_NONBLOCKING: break @@ -263,11 +263,11 @@ break if count <= 0: - raise _ssl_seterror(self.space, self, count) + raise _ssl_seterror(space, self, count) result = buf.str(count) - return self.space.wrap(result) + return space.wrap(result) def _refresh_nonblocking(self, space): # just in case the blocking state of the socket has been changed @@ -361,18 +361,18 @@ ssl_err = libssl_SSL_get_error(self.ssl, ret) if ssl_err == SSL_ERROR_WANT_READ: sockstate = check_socket_and_wait_for_timeout( - self.space, self.w_socket, False) + space, self.w_socket, False) elif ssl_err == SSL_ERROR_WANT_WRITE: sockstate = check_socket_and_wait_for_timeout( - self.space, self.w_socket, True) + space, self.w_socket, True) else: break if sockstate == SOCKET_HAS_TIMED_OUT: if ssl_err == SSL_ERROR_WANT_READ: - raise ssl_error(self.space, "The read operation timed out") + raise ssl_error(space, "The read operation timed out") else: - raise ssl_error(self.space, "The write operation timed out") + raise ssl_error(space, "The write operation timed out") elif sockstate == SOCKET_TOO_LARGE_FOR_SELECT: raise ssl_error(space, "Underlying socket too large for select().") elif sockstate != SOCKET_OPERATION_OK: @@ -410,7 +410,7 @@ return space.newtuple([w_name, w_proto, w_bits]) @unwrap_spec(der=bool) - def peer_certificate(self, der=False): + def peer_certificate(self, space, der=False): """peer_certificate([der=False]) -> certificate Returns the certificate for the peer. If no certificate was provided, @@ -422,7 +422,7 @@ peer certificate, or None if no certificate was provided. This will return the certificate even if it wasn't validated.""" if not self.peer_cert: - return self.space.w_None + return space.w_None if der: # return cert in DER-encoded format @@ -430,20 +430,19 @@ buf_ptr[0] = lltype.nullptr(rffi.CCHARP.TO) length = libssl_i2d_X509(self.peer_cert, buf_ptr) if length < 0: - raise _ssl_seterror(self.space, self, length) + raise _ssl_seterror(space, self, length) try: # this is actually an immutable bytes sequence - return self.space.wrap(rffi.charpsize2str(buf_ptr[0], - length)) + return space.wrap(rffi.charpsize2str(buf_ptr[0], length)) finally: libssl_OPENSSL_free(buf_ptr[0]) else: verification = libssl_SSL_CTX_get_verify_mode( libssl_SSL_get_SSL_CTX(self.ssl)) if not verification & SSL_VERIFY_PEER: - return self.space.newdict() + return space.newdict() else: - return _decode_certificate(self.space, self.peer_cert) + return _decode_certificate(space, self.peer_cert) def _decode_certificate(space, certificate, verbose=False): w_retval = space.newdict() From noreply at buildbot.pypy.org Wed Jul 16 22:55:30 2014 From: noreply at buildbot.pypy.org (pjenvey) Date: Wed, 16 Jul 2014 22:55:30 +0200 (CEST) Subject: [pypy-commit] pypy default: cleanup Message-ID: <20140716205530.642A21C0091@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: Changeset: r72459:1fa3f31534b0 Date: 2014-07-16 13:54 -0700 http://bitbucket.org/pypy/pypy/changeset/1fa3f31534b0/ Log: cleanup diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py --- a/pypy/module/_ssl/interp_ssl.py +++ b/pypy/module/_ssl/interp_ssl.py @@ -1,14 +1,12 @@ -from __future__ import with_statement -from rpython.rtyper.lltypesystem import rffi, lltype +from rpython.rlib import rpoll, rsocket +from rpython.rlib.rarithmetic import intmask +from rpython.rlib.ropenssl import * +from rpython.rtyper.lltypesystem import lltype, rffi + +from pypy.interpreter.baseobjspace import W_Root from pypy.interpreter.error import OperationError -from pypy.interpreter.baseobjspace import W_Root +from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.typedef import TypeDef -from pypy.interpreter.gateway import interp2app, unwrap_spec - -from rpython.rlib.rarithmetic import intmask -from rpython.rlib import rpoll, rsocket -from rpython.rlib.ropenssl import * - from pypy.module._socket import interp_socket @@ -83,19 +81,15 @@ Mix string into the OpenSSL PRNG state. entropy (a float) is a lower bound on the entropy contained in string.""" - - buf = rffi.str2charp(string) - try: + with rffi.scoped_str2charp(string) as buf: libssl_RAND_add(buf, len(string), entropy) - finally: - rffi.free_charp(buf) def RAND_status(space): """RAND_status() -> 0 or 1 - Returns 1 if the OpenSSL PRNG has been seeded with enough data and 0 if not. - It is necessary to seed the PRNG with RAND_add() on some platforms before - using the ssl() function.""" + Returns 1 if the OpenSSL PRNG has been seeded with enough data + and 0 if not. It is necessary to seed the PRNG with RAND_add() + on some platforms before using the ssl() function.""" res = libssl_RAND_status() return space.wrap(res) @@ -107,16 +101,12 @@ Queries the entropy gather daemon (EGD) on socket path. Returns number of bytes read. Raises socket.sslerror if connection to EGD fails or if it does provide enough data to seed PRNG.""" - - socket_path = rffi.str2charp(path) - try: + with rffi.scoped_str2charp(path) as socket_path: bytes = libssl_RAND_egd(socket_path) - finally: - rffi.free_charp(socket_path) if bytes == -1: - msg = "EGD connection failed or EGD did not return" - msg += " enough data to seed the PRNG" - raise ssl_error(space, msg) + raise ssl_error(space, + "EGD connection failed or EGD did not return " + "enough data to seed the PRNG") return space.wrap(bytes) @@ -127,9 +117,11 @@ self.ctx = lltype.nullptr(SSL_CTX.TO) self.ssl = lltype.nullptr(SSL.TO) self.peer_cert = lltype.nullptr(X509.TO) - self._server = lltype.malloc(rffi.CCHARP.TO, X509_NAME_MAXLEN, flavor='raw') + self._server = lltype.malloc(rffi.CCHARP.TO, X509_NAME_MAXLEN, + flavor='raw') self._server[0] = '\0' - self._issuer = lltype.malloc(rffi.CCHARP.TO, X509_NAME_MAXLEN, flavor='raw') + self._issuer = lltype.malloc(rffi.CCHARP.TO, X509_NAME_MAXLEN, + flavor='raw') self._issuer[0] = '\0' self.shutdown_seen_zero = False @@ -162,8 +154,7 @@ of bytes written.""" self._refresh_nonblocking(space) - sockstate = check_socket_and_wait_for_timeout(space, - self.w_socket, True) + sockstate = checkwait(space, self.w_socket, True) if sockstate == SOCKET_HAS_TIMED_OUT: raise ssl_error(space, "The write operation timed out") elif sockstate == SOCKET_HAS_BEEN_CLOSED: @@ -179,11 +170,9 @@ err = libssl_SSL_get_error(self.ssl, num_bytes) if err == SSL_ERROR_WANT_READ: - sockstate = check_socket_and_wait_for_timeout(space, - self.w_socket, False) + sockstate = checkwait(space, self.w_socket, False) elif err == SSL_ERROR_WANT_WRITE: - sockstate = check_socket_and_wait_for_timeout(space, - self.w_socket, True) + sockstate = checkwait(space, self.w_socket, True) else: sockstate = SOCKET_OPERATION_OK @@ -219,19 +208,19 @@ """read([len]) -> string Read up to len bytes from the SSL socket.""" - count = libssl_SSL_pending(self.ssl) if not count: - sockstate = check_socket_and_wait_for_timeout(space, self.w_socket, - False) + sockstate = checkwait(space, self.w_socket, False) if sockstate == SOCKET_HAS_TIMED_OUT: raise ssl_error(space, "The read operation timed out") elif sockstate == SOCKET_TOO_LARGE_FOR_SELECT: - raise ssl_error(space, "Underlying socket too large for select().") + raise ssl_error(space, + "Underlying socket too large for select().") elif sockstate == SOCKET_HAS_BEEN_CLOSED: if libssl_SSL_get_shutdown(self.ssl) == SSL_RECEIVED_SHUTDOWN: return space.wrap('') - raise ssl_error(space, "Socket closed without SSL shutdown handshake") + raise ssl_error(space, + "Socket closed without SSL shutdown handshake") with rffi.scoped_alloc_buffer(num_bytes) as buf: while True: @@ -241,11 +230,9 @@ err = libssl_SSL_get_error(self.ssl, count) if err == SSL_ERROR_WANT_READ: - sockstate = check_socket_and_wait_for_timeout(space, - self.w_socket, False) + sockstate = checkwait(space, self.w_socket, False) elif err == SSL_ERROR_WANT_WRITE: - sockstate = check_socket_and_wait_for_timeout(space, - self.w_socket, True) + sockstate = checkwait(space, self.w_socket, True) elif (err == SSL_ERROR_ZERO_RETURN and libssl_SSL_get_shutdown(self.ssl) == SSL_RECEIVED_SHUTDOWN): return space.wrap("") @@ -286,11 +273,9 @@ err = libssl_SSL_get_error(self.ssl, ret) # XXX PyErr_CheckSignals() if err == SSL_ERROR_WANT_READ: - sockstate = check_socket_and_wait_for_timeout( - space, self.w_socket, False) + sockstate = checkwait(space, self.w_socket, False) elif err == SSL_ERROR_WANT_WRITE: - sockstate = check_socket_and_wait_for_timeout( - space, self.w_socket, True) + sockstate = checkwait(space, self.w_socket, True) else: sockstate = SOCKET_OPERATION_OK if sockstate == SOCKET_HAS_TIMED_OUT: @@ -298,7 +283,8 @@ elif sockstate == SOCKET_HAS_BEEN_CLOSED: raise ssl_error(space, "Underlying socket has been closed.") elif sockstate == SOCKET_TOO_LARGE_FOR_SELECT: - raise ssl_error(space, "Underlying socket too large for select().") + raise ssl_error(space, + "Underlying socket too large for select().") elif sockstate == SOCKET_IS_NONBLOCKING: break @@ -330,7 +316,6 @@ self._refresh_nonblocking(space) zeros = 0 - while True: # Disable read-ahead so that unwrap can work correctly. # Otherwise OpenSSL might read in too much data, @@ -360,11 +345,9 @@ # Possibly retry shutdown until timeout or failure ssl_err = libssl_SSL_get_error(self.ssl, ret) if ssl_err == SSL_ERROR_WANT_READ: - sockstate = check_socket_and_wait_for_timeout( - space, self.w_socket, False) + sockstate = checkwait(space, self.w_socket, False) elif ssl_err == SSL_ERROR_WANT_WRITE: - sockstate = check_socket_and_wait_for_timeout( - space, self.w_socket, True) + sockstate = checkwait(space, self.w_socket, True) else: break @@ -374,7 +357,8 @@ else: raise ssl_error(space, "The write operation timed out") elif sockstate == SOCKET_TOO_LARGE_FOR_SELECT: - raise ssl_error(space, "Underlying socket too large for select().") + raise ssl_error(space, + "Underlying socket too large for select().") elif sockstate != SOCKET_OPERATION_OK: # Retain the SSL error code break @@ -392,35 +376,29 @@ return space.w_None name = libssl_SSL_CIPHER_get_name(current) - if name: - w_name = space.wrap(rffi.charp2str(name)) - else: - w_name = space.w_None + w_name = space.wrap(rffi.charp2str(name)) if name else space.w_None proto = libssl_SSL_CIPHER_get_version(current) - if proto: - w_proto = space.wrap(rffi.charp2str(proto)) - else: - w_proto = space.w_None + w_proto = space.wrap(rffi.charp2str(proto)) if proto else space.w_None bits = libssl_SSL_CIPHER_get_bits(current, lltype.nullptr(rffi.INTP.TO)) w_bits = space.newint(bits) - return space.newtuple([w_name, w_proto, w_bits]) @unwrap_spec(der=bool) def peer_certificate(self, space, der=False): """peer_certificate([der=False]) -> certificate - Returns the certificate for the peer. If no certificate was provided, - returns None. If a certificate was provided, but not validated, returns - an empty dictionary. Otherwise returns a dict containing information - about the peer certificate. + Returns the certificate for the peer. If no certificate was + provided, returns None. If a certificate was provided, but not + validated, returns an empty dictionary. Otherwise returns a + dict containing information about the peer certificate. - If the optional argument is True, returns a DER-encoded copy of the - peer certificate, or None if no certificate was provided. This will - return the certificate even if it wasn't validated.""" + If the optional argument is True, returns a DER-encoded copy of + the peer certificate, or None if no certificate was provided. + This will return the certificate even if it wasn't validated. + """ if not self.peer_cert: return space.w_None @@ -579,15 +557,16 @@ name = libssl_sk_GENERAL_NAME_value(names, j) gntype = intmask(name[0].c_type) if gntype == GEN_DIRNAME: - # we special-case DirName as a tuple of tuples of attributes + # we special-case DirName as a tuple of tuples of + # attributes dirname = libssl_pypy_GENERAL_NAME_dirn(name) w_t = space.newtuple([ space.wrap("DirName"), _create_tuple_for_X509_NAME(space, dirname) ]) elif gntype in (GEN_EMAIL, GEN_DNS, GEN_URI): - # GENERAL_NAME_print() doesn't handle NULL bytes in ASN1_string - # correctly, CVE-2013-4238 + # GENERAL_NAME_print() doesn't handle NULL bytes in + # ASN1_string correctly, CVE-2013-4238 if gntype == GEN_EMAIL: v = space.wrap("email") elif gntype == GEN_DNS: @@ -664,26 +643,11 @@ sock_fd = space.int_w(space.call_method(w_sock, "fileno")) w_timeout = space.call_method(w_sock, "gettimeout") - if space.is_none(w_timeout): - has_timeout = False - else: - has_timeout = True - if space.is_none(w_key_file): - key_file = None - else: - key_file = space.str_w(w_key_file) - if space.is_none(w_cert_file): - cert_file = None - else: - cert_file = space.str_w(w_cert_file) - if space.is_none(w_cacerts_file): - cacerts_file = None - else: - cacerts_file = space.str_w(w_cacerts_file) - if space.is_none(w_ciphers): - ciphers = None - else: - ciphers = space.str_w(w_ciphers) + has_timeout = not space.is_none(w_timeout) + key_file = space.str_or_None_w(w_key_file) + cert_file = space.str_or_None_w(w_cert_file) + cacerts_file = space.str_or_None_w(w_cacerts_file) + ciphers = space.str_or_None_w(w_ciphers) if side == PY_SSL_SERVER and (not key_file or not cert_file): raise ssl_error(space, "Both the key & certificate files " @@ -746,8 +710,8 @@ libssl_SSL_set_fd(ss.ssl, sock_fd) # set the socket for SSL # The ACCEPT_MOVING_WRITE_BUFFER flag is necessary because the address # of a str object may be changed by the garbage collector. - libssl_SSL_set_mode(ss.ssl, - SSL_MODE_AUTO_RETRY | SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER) + libssl_SSL_set_mode( + ss.ssl, SSL_MODE_AUTO_RETRY | SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER) # If the socket is in non-blocking mode or timeout mode, set the BIO # to non-blocking mode (blocking is the default) @@ -764,7 +728,7 @@ ss.w_socket = w_sock return ss -def check_socket_and_wait_for_timeout(space, w_sock, writing): +def checkwait(space, w_sock, writing): """If the socket has a timeout, do a select()/poll() on the socket. The argument writing indicates the direction. Returns one of the possibilities in the timeout_state enum (above).""" From noreply at buildbot.pypy.org Wed Jul 16 22:55:31 2014 From: noreply at buildbot.pypy.org (pjenvey) Date: Wed, 16 Jul 2014 22:55:31 +0200 (CEST) Subject: [pypy-commit] pypy default: might as well use is_none here Message-ID: <20140716205531.B65E41C0091@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: Changeset: r72460:cb8e34d4eaf3 Date: 2014-07-16 13:54 -0700 http://bitbucket.org/pypy/pypy/changeset/cb8e34d4eaf3/ Log: might as well use is_none here diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -1501,9 +1501,7 @@ return buf.as_str() def str_or_None_w(self, w_obj): - if self.is_w(w_obj, self.w_None): - return None - return self.str_w(w_obj) + return None if self.is_none(w_obj) else self.str_w(w_obj) def str_w(self, w_obj): return w_obj.str_w(self) diff --git a/pypy/module/_winreg/interp_winreg.py b/pypy/module/_winreg/interp_winreg.py --- a/pypy/module/_winreg/interp_winreg.py +++ b/pypy/module/_winreg/interp_winreg.py @@ -667,11 +667,6 @@ space.wrap(nValues[0]), space.wrap(l)]) -def str_or_None_w(space, w_obj): - if space.is_w(w_obj, space.w_None): - return None - return space.str_w(w_obj) - def ConnectRegistry(space, w_machine, w_hkey): """key = ConnectRegistry(computer_name, key) @@ -683,7 +678,7 @@ The return value is the handle of the opened key. If the function fails, an EnvironmentError exception is raised.""" - machine = str_or_None_w(space, w_machine) + machine = space.str_or_None_w(w_machine) hkey = hkey_w(w_hkey, space) with lltype.scoped_alloc(rwinreg.PHKEY.TO, 1) as rethkey: ret = rwinreg.RegConnectRegistry(machine, hkey, rethkey) From noreply at buildbot.pypy.org Wed Jul 16 23:07:18 2014 From: noreply at buildbot.pypy.org (pjenvey) Date: Wed, 16 Jul 2014 23:07:18 +0200 (CEST) Subject: [pypy-commit] pypy py3k: merge default Message-ID: <20140716210718.D49591D2A23@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r72461:cd0604d4ee65 Date: 2014-07-16 14:05 -0700 http://bitbucket.org/pypy/pypy/changeset/cd0604d4ee65/ Log: merge default diff --git a/lib-python/2.7/xml/sax/saxutils.py b/lib-python/2.7/xml/sax/saxutils.py --- a/lib-python/2.7/xml/sax/saxutils.py +++ b/lib-python/2.7/xml/sax/saxutils.py @@ -98,13 +98,14 @@ except AttributeError: pass # wrap a binary writer with TextIOWrapper - class UnbufferedTextIOWrapper(io.TextIOWrapper): - def write(self, s): - super(UnbufferedTextIOWrapper, self).write(s) - self.flush() - return UnbufferedTextIOWrapper(buffer, encoding=encoding, + return _UnbufferedTextIOWrapper(buffer, encoding=encoding, errors='xmlcharrefreplace', newline='\n') +# PyPy: moved this class outside the function above +class _UnbufferedTextIOWrapper(io.TextIOWrapper): + def write(self, s): + super(_UnbufferedTextIOWrapper, self).write(s) + self.flush() class XMLGenerator(handler.ContentHandler): diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -1459,9 +1459,7 @@ return buf.as_str() def str_or_None_w(self, w_obj): - if self.is_w(w_obj, self.w_None): - return None - return self.str_w(w_obj) + return None if self.is_none(w_obj) else self.str_w(w_obj) def str_w(self, w_obj): """ diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py --- a/pypy/module/_ssl/interp_ssl.py +++ b/pypy/module/_ssl/interp_ssl.py @@ -1,17 +1,16 @@ -from __future__ import with_statement -from rpython.rtyper.lltypesystem import rffi, lltype -from pypy.interpreter.error import OperationError, oefmt, wrap_oserror -from pypy.interpreter.baseobjspace import W_Root -from pypy.interpreter.typedef import TypeDef, GetSetProperty -from pypy.interpreter.gateway import interp2app, unwrap_spec +import weakref +from rpython.rlib import rpoll, rsocket from rpython.rlib.rarithmetic import intmask -from rpython.rlib import rpoll, rsocket from rpython.rlib.ropenssl import * from rpython.rlib.rposix import get_errno, set_errno +from rpython.rtyper.lltypesystem import lltype, rffi +from pypy.interpreter.baseobjspace import W_Root +from pypy.interpreter.error import OperationError, oefmt, wrap_oserror +from pypy.interpreter.gateway import interp2app, unwrap_spec +from pypy.interpreter.typedef import GetSetProperty, TypeDef from pypy.module._socket import interp_socket -import weakref ## user defined constants @@ -296,19 +295,15 @@ Mix string into the OpenSSL PRNG state. entropy (a float) is a lower bound on the entropy contained in string.""" - - buf = rffi.str2charp(string) - try: + with rffi.scoped_str2charp(string) as buf: libssl_RAND_add(buf, len(string), entropy) - finally: - rffi.free_charp(buf) def RAND_status(space): """RAND_status() -> 0 or 1 - Returns 1 if the OpenSSL PRNG has been seeded with enough data and 0 if not. - It is necessary to seed the PRNG with RAND_add() on some platforms before - using the ssl() function.""" + Returns 1 if the OpenSSL PRNG has been seeded with enough data + and 0 if not. It is necessary to seed the PRNG with RAND_add() + on some platforms before using the ssl() function.""" res = libssl_RAND_status() return space.wrap(res) @@ -320,16 +315,12 @@ Queries the entropy gather daemon (EGD) on socket path. Returns number of bytes read. Raises socket.sslerror if connection to EGD fails or if it does provide enough data to seed PRNG.""" - - socket_path = rffi.str2charp(path) - try: + with rffi.scoped_str2charp(path) as socket_path: bytes = libssl_RAND_egd(socket_path) - finally: - rffi.free_charp(socket_path) if bytes == -1: - msg = "EGD connection failed or EGD did not return" - msg += " enough data to seed the PRNG" - raise ssl_error(space, msg) + raise ssl_error(space, + "EGD connection failed or EGD did not return " + "enough data to seed the PRNG") return space.wrap(bytes) @@ -354,7 +345,7 @@ of bytes written.""" w_socket = self._get_socket(space) - sockstate = check_socket_and_wait_for_timeout(space, w_socket, True) + sockstate = checkwait(space, w_socket, True) if sockstate == SOCKET_HAS_TIMED_OUT: raise ssl_error(space, "The write operation timed out") elif sockstate == SOCKET_HAS_BEEN_CLOSED: @@ -370,11 +361,9 @@ err = libssl_SSL_get_error(self.ssl, num_bytes) if err == SSL_ERROR_WANT_READ: - sockstate = check_socket_and_wait_for_timeout( - space, w_socket, False) + sockstate = checkwait(space, w_socket, False) elif err == SSL_ERROR_WANT_WRITE: - sockstate = check_socket_and_wait_for_timeout( - space, w_socket, True) + sockstate = checkwait(space, w_socket, True) else: sockstate = SOCKET_OPERATION_OK @@ -414,19 +403,20 @@ count = libssl_SSL_pending(self.ssl) if not count: - sockstate = check_socket_and_wait_for_timeout( - space, w_socket, False) + sockstate = checkwait(space, w_socket, False) if sockstate == SOCKET_HAS_TIMED_OUT: raise ssl_error(space, "The read operation timed out") elif sockstate == SOCKET_TOO_LARGE_FOR_SELECT: - raise ssl_error(space, "Underlying socket too large for select().") + raise ssl_error(space, + "Underlying socket too large for select().") elif sockstate == SOCKET_HAS_BEEN_CLOSED: if libssl_SSL_get_shutdown(self.ssl) == SSL_RECEIVED_SHUTDOWN: if space.is_none(w_buf): return space.wrapbytes('') else: return space.wrap(0) - raise ssl_error(space, "Socket closed without SSL shutdown handshake") + raise ssl_error(space, + "Socket closed without SSL shutdown handshake") rwbuffer = None if not space.is_none(w_buf): @@ -443,11 +433,9 @@ err = libssl_SSL_get_error(self.ssl, count) if err == SSL_ERROR_WANT_READ: - sockstate = check_socket_and_wait_for_timeout(self.space, - self.w_socket, False) + sockstate = checkwait(space, self.w_socket, False) elif err == SSL_ERROR_WANT_WRITE: - sockstate = check_socket_and_wait_for_timeout(self.space, - self.w_socket, True) + sockstate = checkwait(space, self.w_socket, True) elif (err == SSL_ERROR_ZERO_RETURN and libssl_SSL_get_shutdown(self.ssl) == SSL_RECEIVED_SHUTDOWN): if space.is_none(w_buf): @@ -458,7 +446,7 @@ sockstate = SOCKET_OPERATION_OK if sockstate == SOCKET_HAS_TIMED_OUT: - raise ssl_error(self.space, "The read operation timed out") + raise ssl_error(space, "The read operation timed out") elif sockstate == SOCKET_IS_NONBLOCKING: break @@ -468,7 +456,7 @@ break if count <= 0: - raise _ssl_seterror(self.space, self, count) + raise _ssl_seterror(space, self, count) result = buf.str(count) @@ -501,11 +489,9 @@ err = libssl_SSL_get_error(self.ssl, ret) # XXX PyErr_CheckSignals() if err == SSL_ERROR_WANT_READ: - sockstate = check_socket_and_wait_for_timeout( - space, w_socket, False) + sockstate = checkwait(space, w_socket, False) elif err == SSL_ERROR_WANT_WRITE: - sockstate = check_socket_and_wait_for_timeout( - space, w_socket, True) + sockstate = checkwait(space, w_socket, True) else: sockstate = SOCKET_OPERATION_OK if sockstate == SOCKET_HAS_TIMED_OUT: @@ -513,7 +499,8 @@ elif sockstate == SOCKET_HAS_BEEN_CLOSED: raise ssl_error(space, "Underlying socket has been closed.") elif sockstate == SOCKET_TOO_LARGE_FOR_SELECT: - raise ssl_error(space, "Underlying socket too large for select().") + raise ssl_error(space, + "Underlying socket too large for select().") elif sockstate == SOCKET_IS_NONBLOCKING: break @@ -538,7 +525,6 @@ raise ssl_error(space, "Underlying socket has been closed") zeros = 0 - while True: # Disable read-ahead so that unwrap can work correctly. # Otherwise OpenSSL might read in too much data, @@ -568,11 +554,9 @@ # Possibly retry shutdown until timeout or failure ssl_err = libssl_SSL_get_error(self.ssl, ret) if ssl_err == SSL_ERROR_WANT_READ: - sockstate = check_socket_and_wait_for_timeout( - space, w_socket, False) + sockstate = checkwait(space, w_socket, False) elif ssl_err == SSL_ERROR_WANT_WRITE: - sockstate = check_socket_and_wait_for_timeout( - space, w_socket, True) + sockstate = checkwait(space, w_socket, True) else: break @@ -582,7 +566,8 @@ else: raise ssl_error(space, "The write operation timed out") elif sockstate == SOCKET_TOO_LARGE_FOR_SELECT: - raise ssl_error(space, "Underlying socket too large for select().") + raise ssl_error(space, + "Underlying socket too large for select().") elif sockstate != SOCKET_OPERATION_OK: # Retain the SSL error code break @@ -600,35 +585,29 @@ return space.w_None name = libssl_SSL_CIPHER_get_name(current) - if name: - w_name = space.wrap(rffi.charp2str(name)) - else: - w_name = space.w_None + w_name = space.wrap(rffi.charp2str(name)) if name else space.w_None proto = libssl_SSL_CIPHER_get_version(current) - if proto: - w_proto = space.wrap(rffi.charp2str(proto)) - else: - w_proto = space.w_None + w_proto = space.wrap(rffi.charp2str(proto)) if proto else space.w_None bits = libssl_SSL_CIPHER_get_bits(current, lltype.nullptr(rffi.INTP.TO)) w_bits = space.newint(bits) - return space.newtuple([w_name, w_proto, w_bits]) @unwrap_spec(der=bool) def peer_certificate(self, space, der=False): """peer_certificate([der=False]) -> certificate - Returns the certificate for the peer. If no certificate was provided, - returns None. If a certificate was provided, but not validated, returns - an empty dictionary. Otherwise returns a dict containing information - about the peer certificate. + Returns the certificate for the peer. If no certificate was + provided, returns None. If a certificate was provided, but not + validated, returns an empty dictionary. Otherwise returns a + dict containing information about the peer certificate. - If the optional argument is True, returns a DER-encoded copy of the - peer certificate, or None if no certificate was provided. This will - return the certificate even if it wasn't validated.""" + If the optional argument is True, returns a DER-encoded copy of + the peer certificate, or None if no certificate was provided. + This will return the certificate even if it wasn't validated. + """ if not self.peer_cert: return space.w_None @@ -641,8 +620,7 @@ raise _ssl_seterror(space, self, length) try: # this is actually an immutable bytes sequence - return space.wrap(rffi.charpsize2str(buf_ptr[0], - length)) + return space.wrap(rffi.charpsize2str(buf_ptr[0], length)) finally: libssl_OPENSSL_free(buf_ptr[0]) else: @@ -788,15 +766,16 @@ name = libssl_sk_GENERAL_NAME_value(names, j) gntype = intmask(name[0].c_type) if gntype == GEN_DIRNAME: - # we special-case DirName as a tuple of tuples of attributes + # we special-case DirName as a tuple of tuples of + # attributes dirname = libssl_pypy_GENERAL_NAME_dirn(name) w_t = space.newtuple([ space.wrap("DirName"), _create_tuple_for_X509_NAME(space, dirname) ]) elif gntype in (GEN_EMAIL, GEN_DNS, GEN_URI): - # GENERAL_NAME_print() doesn't handle NULL bytes in ASN1_string - # correctly, CVE-2013-4238 + # GENERAL_NAME_print() doesn't handle NULL bytes in + # ASN1_string correctly, CVE-2013-4238 if gntype == GEN_EMAIL: v = space.wrap("email") elif gntype == GEN_DNS: @@ -870,17 +849,14 @@ sock_fd = space.int_w(space.call_method(w_sock, "fileno")) w_timeout = space.call_method(w_sock, "gettimeout") - if space.is_none(w_timeout): - has_timeout = False - else: - has_timeout = True + has_timeout = not space.is_none(w_timeout) ss.ssl = libssl_SSL_new(ctx) # new ssl struct libssl_SSL_set_fd(ss.ssl, sock_fd) # set the socket for SSL # The ACCEPT_MOVING_WRITE_BUFFER flag is necessary because the address # of a str object may be changed by the garbage collector. - libssl_SSL_set_mode(ss.ssl, - SSL_MODE_AUTO_RETRY | SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER) + libssl_SSL_set_mode( + ss.ssl, SSL_MODE_AUTO_RETRY | SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER) if server_hostname: libssl_SSL_set_tlsext_host_name(ss.ssl, server_hostname); @@ -900,7 +876,7 @@ ss.w_socket = weakref.ref(w_sock) return ss -def check_socket_and_wait_for_timeout(space, w_sock, writing): +def checkwait(space, w_sock, writing): """If the socket has a timeout, do a select()/poll() on the socket. The argument writing indicates the direction. Returns one of the possibilities in the timeout_state enum (above).""" diff --git a/pypy/module/_winreg/interp_winreg.py b/pypy/module/_winreg/interp_winreg.py --- a/pypy/module/_winreg/interp_winreg.py +++ b/pypy/module/_winreg/interp_winreg.py @@ -667,11 +667,6 @@ space.wrap(nValues[0]), space.wrap(l)]) -def str_or_None_w(space, w_obj): - if space.is_w(w_obj, space.w_None): - return None - return space.str_w(w_obj) - def ConnectRegistry(space, w_machine, w_hkey): """key = ConnectRegistry(computer_name, key) @@ -683,7 +678,7 @@ The return value is the handle of the opened key. If the function fails, an EnvironmentError exception is raised.""" - machine = str_or_None_w(space, w_machine) + machine = space.str_or_None_w(w_machine) hkey = hkey_w(w_hkey, space) with lltype.scoped_alloc(rwinreg.PHKEY.TO, 1) as rethkey: ret = rwinreg.RegConnectRegistry(machine, hkey, rethkey) diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py --- a/rpython/jit/metainterp/pyjitpl.py +++ b/rpython/jit/metainterp/pyjitpl.py @@ -1377,49 +1377,53 @@ def do_residual_call(self, funcbox, argboxes, descr, pc, assembler_call=False, assembler_call_jd=None): - # First build allboxes: it may need some reordering from the - # list provided in argboxes, depending on the order in which - # the arguments are expected by the function - # - allboxes = self._build_allboxes(funcbox, argboxes, descr) - effectinfo = descr.get_extra_info() - if (assembler_call or - effectinfo.check_forces_virtual_or_virtualizable()): - # residual calls require attention to keep virtualizables in-sync - self.metainterp.clear_exception() - if effectinfo.oopspecindex == EffectInfo.OS_JIT_FORCE_VIRTUAL: - resbox = self._do_jit_force_virtual(allboxes, descr, pc) + debug_start("jit-residual-call") + try: + # First build allboxes: it may need some reordering from the + # list provided in argboxes, depending on the order in which + # the arguments are expected by the function + # + allboxes = self._build_allboxes(funcbox, argboxes, descr) + effectinfo = descr.get_extra_info() + if (assembler_call or + effectinfo.check_forces_virtual_or_virtualizable()): + # residual calls require attention to keep virtualizables in-sync + self.metainterp.clear_exception() + if effectinfo.oopspecindex == EffectInfo.OS_JIT_FORCE_VIRTUAL: + resbox = self._do_jit_force_virtual(allboxes, descr, pc) + if resbox is not None: + return resbox + self.metainterp.vable_and_vrefs_before_residual_call() + resbox = self.metainterp.execute_and_record_varargs( + rop.CALL_MAY_FORCE, allboxes, descr=descr) + if effectinfo.is_call_release_gil(): + self.metainterp.direct_call_release_gil() + self.metainterp.vrefs_after_residual_call() + vablebox = None + if assembler_call: + vablebox = self.metainterp.direct_assembler_call( + assembler_call_jd) if resbox is not None: - return resbox - self.metainterp.vable_and_vrefs_before_residual_call() - resbox = self.metainterp.execute_and_record_varargs( - rop.CALL_MAY_FORCE, allboxes, descr=descr) - if effectinfo.is_call_release_gil(): - self.metainterp.direct_call_release_gil() - self.metainterp.vrefs_after_residual_call() - vablebox = None - if assembler_call: - vablebox = self.metainterp.direct_assembler_call( - assembler_call_jd) - if resbox is not None: - self.make_result_of_lastop(resbox) - self.metainterp.vable_after_residual_call(funcbox) - self.metainterp.generate_guard(rop.GUARD_NOT_FORCED, None) - if vablebox is not None: - self.metainterp.history.record(rop.KEEPALIVE, [vablebox], None) - self.metainterp.handle_possible_exception() - # XXX refactor: direct_libffi_call() is a hack - if effectinfo.oopspecindex == effectinfo.OS_LIBFFI_CALL: - self.metainterp.direct_libffi_call() - return resbox - else: - effect = effectinfo.extraeffect - if effect == effectinfo.EF_LOOPINVARIANT: - return self.execute_varargs(rop.CALL_LOOPINVARIANT, allboxes, - descr, False, False) - exc = effectinfo.check_can_raise() - pure = effectinfo.check_is_elidable() - return self.execute_varargs(rop.CALL, allboxes, descr, exc, pure) + self.make_result_of_lastop(resbox) + self.metainterp.vable_after_residual_call(funcbox) + self.metainterp.generate_guard(rop.GUARD_NOT_FORCED, None) + if vablebox is not None: + self.metainterp.history.record(rop.KEEPALIVE, [vablebox], None) + self.metainterp.handle_possible_exception() + # XXX refactor: direct_libffi_call() is a hack + if effectinfo.oopspecindex == effectinfo.OS_LIBFFI_CALL: + self.metainterp.direct_libffi_call() + return resbox + else: + effect = effectinfo.extraeffect + if effect == effectinfo.EF_LOOPINVARIANT: + return self.execute_varargs(rop.CALL_LOOPINVARIANT, allboxes, + descr, False, False) + exc = effectinfo.check_can_raise() + pure = effectinfo.check_is_elidable() + return self.execute_varargs(rop.CALL, allboxes, descr, exc, pure) + finally: + debug_stop("jit-residual-call") def do_conditional_call(self, condbox, funcbox, argboxes, descr, pc): if isinstance(condbox, ConstInt) and condbox.value == 0: diff --git a/rpython/jit/tool/oparser_model.py b/rpython/jit/tool/oparser_model.py --- a/rpython/jit/tool/oparser_model.py +++ b/rpython/jit/tool/oparser_model.py @@ -124,6 +124,15 @@ class ExtendedTreeLoop(model.TreeLoop): + def as_json(self): + return { + 'comment': self.comment, + 'name': self.name, + 'operations': [op.as_json() for op in self.operations], + 'inputargs': self.inputargs, + 'last_offset': self.last_offset + } + def getboxes(self): def opboxes(operations): for op in operations: diff --git a/rpython/tool/jitlogparser/logparser2json.py b/rpython/tool/jitlogparser/logparser2json.py new file mode 100755 --- /dev/null +++ b/rpython/tool/jitlogparser/logparser2json.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python +""" Convert logfile (from jit-log-opt and jit-backend) to json format. +Usage: + +logparser2json.py +""" + +import os +import sys +import json +from rpython.tool.jitlogparser.parser import import_log, parse_log_counts +from rpython.tool.logparser import extract_category +from rpython.tool.jitlogparser.storage import LoopStorage + +def mangle_descr(descr): + if descr.startswith('TargetToken('): + return descr[len('TargetToken('):-1] + if descr.startswith(' Author: Philip Jenvey Branch: py3k Changeset: r72462:bab7e4800a77 Date: 2014-07-16 14:49 -0700 http://bitbucket.org/pypy/pypy/changeset/bab7e4800a77/ Log: fix from previous merge diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py --- a/pypy/module/_ssl/interp_ssl.py +++ b/pypy/module/_ssl/interp_ssl.py @@ -433,9 +433,9 @@ err = libssl_SSL_get_error(self.ssl, count) if err == SSL_ERROR_WANT_READ: - sockstate = checkwait(space, self.w_socket, False) + sockstate = checkwait(space, w_socket, False) elif err == SSL_ERROR_WANT_WRITE: - sockstate = checkwait(space, self.w_socket, True) + sockstate = checkwait(space, w_socket, True) elif (err == SSL_ERROR_ZERO_RETURN and libssl_SSL_get_shutdown(self.ssl) == SSL_RECEIVED_SHUTDOWN): if space.is_none(w_buf): @@ -462,7 +462,7 @@ if rwbuffer is not None: rwbuffer.setslice(0, result) - return self.space.wrap(count) + return space.wrap(count) else: return space.wrapbytes(result) From noreply at buildbot.pypy.org Thu Jul 17 00:07:09 2014 From: noreply at buildbot.pypy.org (pjenvey) Date: Thu, 17 Jul 2014 00:07:09 +0200 (CEST) Subject: [pypy-commit] pypy py3.3: merge py3k Message-ID: <20140716220709.61E321D2857@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3.3 Changeset: r72463:7536507ca6e5 Date: 2014-07-16 14:50 -0700 http://bitbucket.org/pypy/pypy/changeset/7536507ca6e5/ Log: merge py3k diff --git a/lib-python/2.7/xml/sax/saxutils.py b/lib-python/2.7/xml/sax/saxutils.py --- a/lib-python/2.7/xml/sax/saxutils.py +++ b/lib-python/2.7/xml/sax/saxutils.py @@ -98,13 +98,14 @@ except AttributeError: pass # wrap a binary writer with TextIOWrapper - class UnbufferedTextIOWrapper(io.TextIOWrapper): - def write(self, s): - super(UnbufferedTextIOWrapper, self).write(s) - self.flush() - return UnbufferedTextIOWrapper(buffer, encoding=encoding, + return _UnbufferedTextIOWrapper(buffer, encoding=encoding, errors='xmlcharrefreplace', newline='\n') +# PyPy: moved this class outside the function above +class _UnbufferedTextIOWrapper(io.TextIOWrapper): + def write(self, s): + super(_UnbufferedTextIOWrapper, self).write(s) + self.flush() class XMLGenerator(handler.ContentHandler): diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -1461,9 +1461,7 @@ return buf.as_str() def str_or_None_w(self, w_obj): - if self.is_w(w_obj, self.w_None): - return None - return self.str_w(w_obj) + return None if self.is_none(w_obj) else self.str_w(w_obj) def str_w(self, w_obj): """ diff --git a/pypy/module/_ssl/interp_ssl.py b/pypy/module/_ssl/interp_ssl.py --- a/pypy/module/_ssl/interp_ssl.py +++ b/pypy/module/_ssl/interp_ssl.py @@ -1,18 +1,17 @@ -from __future__ import with_statement -from rpython.rtyper.lltypesystem import rffi, lltype -from pypy.interpreter.error import OperationError, oefmt, wrap_oserror -from pypy.interpreter.baseobjspace import W_Root -from pypy.interpreter.typedef import TypeDef, GetSetProperty -from pypy.interpreter.gateway import interp2app, unwrap_spec +import weakref +from rpython.rlib import rpoll, rsocket from rpython.rlib.rarithmetic import intmask -from rpython.rlib import rpoll, rsocket from rpython.rlib.ropenssl import * from rpython.rlib.rposix import get_errno, set_errno +from rpython.rtyper.lltypesystem import lltype, rffi +from pypy.interpreter.baseobjspace import W_Root +from pypy.interpreter.error import OperationError, oefmt, wrap_oserror +from pypy.interpreter.gateway import interp2app, unwrap_spec +from pypy.interpreter.typedef import GetSetProperty, TypeDef from pypy.module._socket import interp_socket from pypy.module.exceptions import interp_exceptions -import weakref ## user defined constants @@ -303,19 +302,15 @@ Mix string into the OpenSSL PRNG state. entropy (a float) is a lower bound on the entropy contained in string.""" - - buf = rffi.str2charp(string) - try: + with rffi.scoped_str2charp(string) as buf: libssl_RAND_add(buf, len(string), entropy) - finally: - rffi.free_charp(buf) def RAND_status(space): """RAND_status() -> 0 or 1 - Returns 1 if the OpenSSL PRNG has been seeded with enough data and 0 if not. - It is necessary to seed the PRNG with RAND_add() on some platforms before - using the ssl() function.""" + Returns 1 if the OpenSSL PRNG has been seeded with enough data + and 0 if not. It is necessary to seed the PRNG with RAND_add() + on some platforms before using the ssl() function.""" res = libssl_RAND_status() return space.wrap(res) @@ -327,16 +322,12 @@ Queries the entropy gather daemon (EGD) on socket path. Returns number of bytes read. Raises socket.sslerror if connection to EGD fails or if it does provide enough data to seed PRNG.""" - - socket_path = rffi.str2charp(path) - try: + with rffi.scoped_str2charp(path) as socket_path: bytes = libssl_RAND_egd(socket_path) - finally: - rffi.free_charp(socket_path) if bytes == -1: - msg = "EGD connection failed or EGD did not return" - msg += " enough data to seed the PRNG" - raise ssl_error(space, msg) + raise ssl_error(space, + "EGD connection failed or EGD did not return " + "enough data to seed the PRNG") return space.wrap(bytes) @@ -361,7 +352,7 @@ of bytes written.""" w_socket = self._get_socket(space) - sockstate = check_socket_and_wait_for_timeout(space, w_socket, True) + sockstate = checkwait(space, w_socket, True) if sockstate == SOCKET_HAS_TIMED_OUT: raise ssl_error(space, "The write operation timed out") elif sockstate == SOCKET_HAS_BEEN_CLOSED: @@ -377,11 +368,9 @@ err = libssl_SSL_get_error(self.ssl, num_bytes) if err == SSL_ERROR_WANT_READ: - sockstate = check_socket_and_wait_for_timeout( - space, w_socket, False) + sockstate = checkwait(space, w_socket, False) elif err == SSL_ERROR_WANT_WRITE: - sockstate = check_socket_and_wait_for_timeout( - space, w_socket, True) + sockstate = checkwait(space, w_socket, True) else: sockstate = SOCKET_OPERATION_OK @@ -421,19 +410,20 @@ count = libssl_SSL_pending(self.ssl) if not count: - sockstate = check_socket_and_wait_for_timeout( - space, w_socket, False) + sockstate = checkwait(space, w_socket, False) if sockstate == SOCKET_HAS_TIMED_OUT: raise ssl_error(space, "The read operation timed out") elif sockstate == SOCKET_TOO_LARGE_FOR_SELECT: - raise ssl_error(space, "Underlying socket too large for select().") + raise ssl_error(space, + "Underlying socket too large for select().") elif sockstate == SOCKET_HAS_BEEN_CLOSED: if libssl_SSL_get_shutdown(self.ssl) == SSL_RECEIVED_SHUTDOWN: if space.is_none(w_buf): return space.wrapbytes('') else: return space.wrap(0) - raise ssl_error(space, "Socket closed without SSL shutdown handshake") + raise ssl_error(space, + "Socket closed without SSL shutdown handshake") rwbuffer = None if not space.is_none(w_buf): @@ -450,11 +440,9 @@ err = libssl_SSL_get_error(self.ssl, count) if err == SSL_ERROR_WANT_READ: - sockstate = check_socket_and_wait_for_timeout(self.space, - self.w_socket, False) + sockstate = checkwait(space, w_socket, False) elif err == SSL_ERROR_WANT_WRITE: - sockstate = check_socket_and_wait_for_timeout(self.space, - self.w_socket, True) + sockstate = checkwait(space, w_socket, True) elif (err == SSL_ERROR_ZERO_RETURN and libssl_SSL_get_shutdown(self.ssl) == SSL_RECEIVED_SHUTDOWN): if space.is_none(w_buf): @@ -465,7 +453,7 @@ sockstate = SOCKET_OPERATION_OK if sockstate == SOCKET_HAS_TIMED_OUT: - raise ssl_error(self.space, "The read operation timed out") + raise ssl_error(space, "The read operation timed out") elif sockstate == SOCKET_IS_NONBLOCKING: break @@ -475,13 +463,13 @@ break if count <= 0: - raise _ssl_seterror(self.space, self, count) + raise _ssl_seterror(space, self, count) result = buf.str(count) if rwbuffer is not None: rwbuffer.setslice(0, result) - return self.space.wrap(count) + return space.wrap(count) else: return space.wrapbytes(result) @@ -508,11 +496,9 @@ err = libssl_SSL_get_error(self.ssl, ret) # XXX PyErr_CheckSignals() if err == SSL_ERROR_WANT_READ: - sockstate = check_socket_and_wait_for_timeout( - space, w_socket, False) + sockstate = checkwait(space, w_socket, False) elif err == SSL_ERROR_WANT_WRITE: - sockstate = check_socket_and_wait_for_timeout( - space, w_socket, True) + sockstate = checkwait(space, w_socket, True) else: sockstate = SOCKET_OPERATION_OK if sockstate == SOCKET_HAS_TIMED_OUT: @@ -520,7 +506,8 @@ elif sockstate == SOCKET_HAS_BEEN_CLOSED: raise ssl_error(space, "Underlying socket has been closed.") elif sockstate == SOCKET_TOO_LARGE_FOR_SELECT: - raise ssl_error(space, "Underlying socket too large for select().") + raise ssl_error(space, + "Underlying socket too large for select().") elif sockstate == SOCKET_IS_NONBLOCKING: break @@ -545,7 +532,6 @@ raise ssl_error(space, "Underlying socket has been closed") zeros = 0 - while True: # Disable read-ahead so that unwrap can work correctly. # Otherwise OpenSSL might read in too much data, @@ -575,11 +561,9 @@ # Possibly retry shutdown until timeout or failure ssl_err = libssl_SSL_get_error(self.ssl, ret) if ssl_err == SSL_ERROR_WANT_READ: - sockstate = check_socket_and_wait_for_timeout( - space, w_socket, False) + sockstate = checkwait(space, w_socket, False) elif ssl_err == SSL_ERROR_WANT_WRITE: - sockstate = check_socket_and_wait_for_timeout( - space, w_socket, True) + sockstate = checkwait(space, w_socket, True) else: break @@ -589,7 +573,8 @@ else: raise ssl_error(space, "The write operation timed out") elif sockstate == SOCKET_TOO_LARGE_FOR_SELECT: - raise ssl_error(space, "Underlying socket too large for select().") + raise ssl_error(space, + "Underlying socket too large for select().") elif sockstate != SOCKET_OPERATION_OK: # Retain the SSL error code break @@ -607,35 +592,29 @@ return space.w_None name = libssl_SSL_CIPHER_get_name(current) - if name: - w_name = space.wrap(rffi.charp2str(name)) - else: - w_name = space.w_None + w_name = space.wrap(rffi.charp2str(name)) if name else space.w_None proto = libssl_SSL_CIPHER_get_version(current) - if proto: - w_proto = space.wrap(rffi.charp2str(proto)) - else: - w_proto = space.w_None + w_proto = space.wrap(rffi.charp2str(proto)) if proto else space.w_None bits = libssl_SSL_CIPHER_get_bits(current, lltype.nullptr(rffi.INTP.TO)) w_bits = space.newint(bits) - return space.newtuple([w_name, w_proto, w_bits]) @unwrap_spec(der=bool) def peer_certificate(self, space, der=False): """peer_certificate([der=False]) -> certificate - Returns the certificate for the peer. If no certificate was provided, - returns None. If a certificate was provided, but not validated, returns - an empty dictionary. Otherwise returns a dict containing information - about the peer certificate. + Returns the certificate for the peer. If no certificate was + provided, returns None. If a certificate was provided, but not + validated, returns an empty dictionary. Otherwise returns a + dict containing information about the peer certificate. - If the optional argument is True, returns a DER-encoded copy of the - peer certificate, or None if no certificate was provided. This will - return the certificate even if it wasn't validated.""" + If the optional argument is True, returns a DER-encoded copy of + the peer certificate, or None if no certificate was provided. + This will return the certificate even if it wasn't validated. + """ if not self.peer_cert: return space.w_None @@ -648,8 +627,7 @@ raise _ssl_seterror(space, self, length) try: # this is actually an immutable bytes sequence - return space.wrap(rffi.charpsize2str(buf_ptr[0], - length)) + return space.wrap(rffi.charpsize2str(buf_ptr[0], length)) finally: libssl_OPENSSL_free(buf_ptr[0]) else: @@ -795,15 +773,16 @@ name = libssl_sk_GENERAL_NAME_value(names, j) gntype = intmask(name[0].c_type) if gntype == GEN_DIRNAME: - # we special-case DirName as a tuple of tuples of attributes + # we special-case DirName as a tuple of tuples of + # attributes dirname = libssl_pypy_GENERAL_NAME_dirn(name) w_t = space.newtuple([ space.wrap("DirName"), _create_tuple_for_X509_NAME(space, dirname) ]) elif gntype in (GEN_EMAIL, GEN_DNS, GEN_URI): - # GENERAL_NAME_print() doesn't handle NULL bytes in ASN1_string - # correctly, CVE-2013-4238 + # GENERAL_NAME_print() doesn't handle NULL bytes in + # ASN1_string correctly, CVE-2013-4238 if gntype == GEN_EMAIL: v = space.wrap("email") elif gntype == GEN_DNS: @@ -877,17 +856,14 @@ sock_fd = space.int_w(space.call_method(w_sock, "fileno")) w_timeout = space.call_method(w_sock, "gettimeout") - if space.is_none(w_timeout): - has_timeout = False - else: - has_timeout = True + has_timeout = not space.is_none(w_timeout) ss.ssl = libssl_SSL_new(ctx) # new ssl struct libssl_SSL_set_fd(ss.ssl, sock_fd) # set the socket for SSL # The ACCEPT_MOVING_WRITE_BUFFER flag is necessary because the address # of a str object may be changed by the garbage collector. - libssl_SSL_set_mode(ss.ssl, - SSL_MODE_AUTO_RETRY | SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER) + libssl_SSL_set_mode( + ss.ssl, SSL_MODE_AUTO_RETRY | SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER) if server_hostname: libssl_SSL_set_tlsext_host_name(ss.ssl, server_hostname); @@ -907,7 +883,7 @@ ss.w_socket = weakref.ref(w_sock) return ss -def check_socket_and_wait_for_timeout(space, w_sock, writing): +def checkwait(space, w_sock, writing): """If the socket has a timeout, do a select()/poll() on the socket. The argument writing indicates the direction. Returns one of the possibilities in the timeout_state enum (above).""" diff --git a/pypy/module/_winreg/interp_winreg.py b/pypy/module/_winreg/interp_winreg.py --- a/pypy/module/_winreg/interp_winreg.py +++ b/pypy/module/_winreg/interp_winreg.py @@ -667,11 +667,6 @@ space.wrap(nValues[0]), space.wrap(l)]) -def str_or_None_w(space, w_obj): - if space.is_w(w_obj, space.w_None): - return None - return space.str_w(w_obj) - def ConnectRegistry(space, w_machine, w_hkey): """key = ConnectRegistry(computer_name, key) @@ -683,7 +678,7 @@ The return value is the handle of the opened key. If the function fails, an EnvironmentError exception is raised.""" - machine = str_or_None_w(space, w_machine) + machine = space.str_or_None_w(w_machine) hkey = hkey_w(w_hkey, space) with lltype.scoped_alloc(rwinreg.PHKEY.TO, 1) as rethkey: ret = rwinreg.RegConnectRegistry(machine, hkey, rethkey) diff --git a/rpython/jit/metainterp/pyjitpl.py b/rpython/jit/metainterp/pyjitpl.py --- a/rpython/jit/metainterp/pyjitpl.py +++ b/rpython/jit/metainterp/pyjitpl.py @@ -1377,49 +1377,53 @@ def do_residual_call(self, funcbox, argboxes, descr, pc, assembler_call=False, assembler_call_jd=None): - # First build allboxes: it may need some reordering from the - # list provided in argboxes, depending on the order in which - # the arguments are expected by the function - # - allboxes = self._build_allboxes(funcbox, argboxes, descr) - effectinfo = descr.get_extra_info() - if (assembler_call or - effectinfo.check_forces_virtual_or_virtualizable()): - # residual calls require attention to keep virtualizables in-sync - self.metainterp.clear_exception() - if effectinfo.oopspecindex == EffectInfo.OS_JIT_FORCE_VIRTUAL: - resbox = self._do_jit_force_virtual(allboxes, descr, pc) + debug_start("jit-residual-call") + try: + # First build allboxes: it may need some reordering from the + # list provided in argboxes, depending on the order in which + # the arguments are expected by the function + # + allboxes = self._build_allboxes(funcbox, argboxes, descr) + effectinfo = descr.get_extra_info() + if (assembler_call or + effectinfo.check_forces_virtual_or_virtualizable()): + # residual calls require attention to keep virtualizables in-sync + self.metainterp.clear_exception() + if effectinfo.oopspecindex == EffectInfo.OS_JIT_FORCE_VIRTUAL: + resbox = self._do_jit_force_virtual(allboxes, descr, pc) + if resbox is not None: + return resbox + self.metainterp.vable_and_vrefs_before_residual_call() + resbox = self.metainterp.execute_and_record_varargs( + rop.CALL_MAY_FORCE, allboxes, descr=descr) + if effectinfo.is_call_release_gil(): + self.metainterp.direct_call_release_gil() + self.metainterp.vrefs_after_residual_call() + vablebox = None + if assembler_call: + vablebox = self.metainterp.direct_assembler_call( + assembler_call_jd) if resbox is not None: - return resbox - self.metainterp.vable_and_vrefs_before_residual_call() - resbox = self.metainterp.execute_and_record_varargs( - rop.CALL_MAY_FORCE, allboxes, descr=descr) - if effectinfo.is_call_release_gil(): - self.metainterp.direct_call_release_gil() - self.metainterp.vrefs_after_residual_call() - vablebox = None - if assembler_call: - vablebox = self.metainterp.direct_assembler_call( - assembler_call_jd) - if resbox is not None: - self.make_result_of_lastop(resbox) - self.metainterp.vable_after_residual_call(funcbox) - self.metainterp.generate_guard(rop.GUARD_NOT_FORCED, None) - if vablebox is not None: - self.metainterp.history.record(rop.KEEPALIVE, [vablebox], None) - self.metainterp.handle_possible_exception() - # XXX refactor: direct_libffi_call() is a hack - if effectinfo.oopspecindex == effectinfo.OS_LIBFFI_CALL: - self.metainterp.direct_libffi_call() - return resbox - else: - effect = effectinfo.extraeffect - if effect == effectinfo.EF_LOOPINVARIANT: - return self.execute_varargs(rop.CALL_LOOPINVARIANT, allboxes, - descr, False, False) - exc = effectinfo.check_can_raise() - pure = effectinfo.check_is_elidable() - return self.execute_varargs(rop.CALL, allboxes, descr, exc, pure) + self.make_result_of_lastop(resbox) + self.metainterp.vable_after_residual_call(funcbox) + self.metainterp.generate_guard(rop.GUARD_NOT_FORCED, None) + if vablebox is not None: + self.metainterp.history.record(rop.KEEPALIVE, [vablebox], None) + self.metainterp.handle_possible_exception() + # XXX refactor: direct_libffi_call() is a hack + if effectinfo.oopspecindex == effectinfo.OS_LIBFFI_CALL: + self.metainterp.direct_libffi_call() + return resbox + else: + effect = effectinfo.extraeffect + if effect == effectinfo.EF_LOOPINVARIANT: + return self.execute_varargs(rop.CALL_LOOPINVARIANT, allboxes, + descr, False, False) + exc = effectinfo.check_can_raise() + pure = effectinfo.check_is_elidable() + return self.execute_varargs(rop.CALL, allboxes, descr, exc, pure) + finally: + debug_stop("jit-residual-call") def do_conditional_call(self, condbox, funcbox, argboxes, descr, pc): if isinstance(condbox, ConstInt) and condbox.value == 0: diff --git a/rpython/jit/tool/oparser_model.py b/rpython/jit/tool/oparser_model.py --- a/rpython/jit/tool/oparser_model.py +++ b/rpython/jit/tool/oparser_model.py @@ -124,6 +124,15 @@ class ExtendedTreeLoop(model.TreeLoop): + def as_json(self): + return { + 'comment': self.comment, + 'name': self.name, + 'operations': [op.as_json() for op in self.operations], + 'inputargs': self.inputargs, + 'last_offset': self.last_offset + } + def getboxes(self): def opboxes(operations): for op in operations: diff --git a/rpython/tool/jitlogparser/logparser2json.py b/rpython/tool/jitlogparser/logparser2json.py new file mode 100755 --- /dev/null +++ b/rpython/tool/jitlogparser/logparser2json.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python +""" Convert logfile (from jit-log-opt and jit-backend) to json format. +Usage: + +logparser2json.py +""" + +import os +import sys +import json +from rpython.tool.jitlogparser.parser import import_log, parse_log_counts +from rpython.tool.logparser import extract_category +from rpython.tool.jitlogparser.storage import LoopStorage + +def mangle_descr(descr): + if descr.startswith('TargetToken('): + return descr[len('TargetToken('):-1] + if descr.startswith(' Author: Philip Jenvey Branch: Changeset: r72464:24db6697b691 Date: 2014-07-16 14:51 -0700 http://bitbucket.org/pypy/pypy/changeset/24db6697b691/ Log: py3k compat diff --git a/pypy/interpreter/test/test_generator.py b/pypy/interpreter/test/test_generator.py --- a/pypy/interpreter/test/test_generator.py +++ b/pypy/interpreter/test/test_generator.py @@ -17,7 +17,7 @@ yield 1 assert g.gi_running g = f() - assert g.gi_code is f.func_code + assert g.gi_code is f.__code__ assert g.__name__ == 'f' assert g.gi_frame is not None assert not g.gi_running @@ -26,7 +26,7 @@ raises(StopIteration, g.next) assert not g.gi_running assert g.gi_frame is None - assert g.gi_code is f.func_code + assert g.gi_code is f.__code__ assert g.__name__ == 'f' def test_generator3(self): @@ -286,13 +286,13 @@ w_co = space.appexec([], '''(): def g(x): yield x + 5 - return g.func_code + return g.__code__ ''') assert should_not_inline(w_co) == False w_co = space.appexec([], '''(): def g(x): yield x + 5 yield x + 6 - return g.func_code + return g.__code__ ''') assert should_not_inline(w_co) == True From noreply at buildbot.pypy.org Thu Jul 17 10:59:04 2014 From: noreply at buildbot.pypy.org (Conrad Calmez) Date: Thu, 17 Jul 2014 10:59:04 +0200 (CEST) Subject: [pypy-commit] lang-smalltalk stmgc-c7: changed argument parsing again for easier usage of smalltalk args Message-ID: <20140717085904.3126E1D2ADB@cobra.cs.uni-duesseldorf.de> Author: Conrad Calmez Branch: stmgc-c7 Changeset: r896:a0e39d69fbe0 Date: 2014-07-17 10:57 +0200 http://bitbucket.org/pypy/lang-smalltalk/changeset/a0e39d69fbe0/ Log: changed argument parsing again for easier usage of smalltalk args now you can stop regular argument parsing with "--" and all further arguments are for the smalltalk image diff --git a/images/Squeak4.5-12568.changes b/images/Squeak4.5-12568.changes --- a/images/Squeak4.5-12568.changes +++ b/images/Squeak4.5-12568.changes @@ -15094,4 +15094,4 @@ ifFalse: [ self organizeMeetingWith: chameleon. first := nil. ] ] atomic value. " tmpString := 'A Chameneos was in the meeting place and first is now: ' . SPyVM print: tmpString. "! ! !CPBChameneos methodsFor: 'as yet unclassified' stamp: 'hh 7/10/2014 15:12' prior: 34771049! run: meetingPlace [ color == #faded ] whileFalse: [ "SPyVM print: 'Chameneos goes to meeting place' , self color." meetingPlace reachedBy: self. - waitingForPair lock. "SPyVM print: 'Chameneos met another one and releases lock' , self color." ]! ! ----QUIT----{10 July 2014 . 3:13 pm} Squeak4.5-benchmarks.image priorSource: 1252802! ----STARTUP----{10 July 2014 . 3:16:52 pm} as /home/hub/hpi/stm/src/lang-smalltalk/images/images/Squeak4.5-benchmarks.image! ----QUIT/NOSAVE----{10 July 2014 . 3:17:21 pm} Squeak4.5-benchmarks.image priorSource: 1253908! ----QUIT/NOSAVE----{10 July 2014 . 1:19:47 pm} Squeak4.5-benchmarks.image priorSource: 1253908! ----STARTUP----{10 July 2014 . 5:00:09 pm} as /home/hub/hpi/stm/src/lang-smalltalk/images/images/Squeak4.5-benchmarks.image! !Integer methodsFor: '*SPy-Benchmarks' stamp: 'hh 7/10/2014 17:01'! OSLockTest | lock | lock := OSLock new. lock release.! ! ----QUIT----{10 July 2014 . 5:01:29 pm} Squeak4.5-benchmarks.image priorSource: 1253908! ----STARTUP----{10 July 2014 . 5:04:01 pm} as /home/hub/hpi/stm/src/lang-smalltalk/images/images/Squeak4.5-benchmarks.image! ----QUIT/NOSAVE----{10 July 2014 . 5:05:51 pm} Squeak4.5-benchmarks.image priorSource: 1254571! ----STARTUP----{10 July 2014 . 5:35:54 pm} as /home/hub/hpi/stm/src/lang-smalltalk/images/images/Squeak4.5-benchmarks.image! ----QUIT/NOSAVE----{10 July 2014 . 5:36:06 pm} Squeak4.5-benchmarks.image priorSource: 1254571! ----STARTUP----{10 July 2014 . 5:44:18 pm} as /home/hub/hpi/stm/src/lang-smalltalk/images/Squeak4.5-12568.image! ----QUIT/NOSAVE----{10 July 2014 . 5:44:25 pm} Squeak4.5-12568.image priorSource: 1254571! ----QUIT/NOSAVE----{10 July 2014 . 6:11:07 pm} Squeak4.5-12568.image priorSource: 1254571! ----STARTUP----{10 July 2014 . 6:11:32 pm} as /Users/conrad/Repositories/master-project/lang-smalltalk/images/Squeak4.5-12568.image! ----QUIT/NOSAVE----{10 July 2014 . 6:22:16 pm} Squeak4.5-12568.image priorSource: 1254571! \ No newline at end of file + waitingForPair lock. "SPyVM print: 'Chameneos met another one and releases lock' , self color." ]! ! ----QUIT----{10 July 2014 . 3:13 pm} Squeak4.5-benchmarks.image priorSource: 1252802! ----STARTUP----{10 July 2014 . 3:16:52 pm} as /home/hub/hpi/stm/src/lang-smalltalk/images/images/Squeak4.5-benchmarks.image! ----QUIT/NOSAVE----{10 July 2014 . 3:17:21 pm} Squeak4.5-benchmarks.image priorSource: 1253908! ----QUIT/NOSAVE----{10 July 2014 . 1:19:47 pm} Squeak4.5-benchmarks.image priorSource: 1253908! ----STARTUP----{10 July 2014 . 5:00:09 pm} as /home/hub/hpi/stm/src/lang-smalltalk/images/images/Squeak4.5-benchmarks.image! !Integer methodsFor: '*SPy-Benchmarks' stamp: 'hh 7/10/2014 17:01'! OSLockTest | lock | lock := OSLock new. lock release.! ! ----QUIT----{10 July 2014 . 5:01:29 pm} Squeak4.5-benchmarks.image priorSource: 1253908! ----STARTUP----{10 July 2014 . 5:04:01 pm} as /home/hub/hpi/stm/src/lang-smalltalk/images/images/Squeak4.5-benchmarks.image! ----QUIT/NOSAVE----{10 July 2014 . 5:05:51 pm} Squeak4.5-benchmarks.image priorSource: 1254571! ----STARTUP----{10 July 2014 . 5:35:54 pm} as /home/hub/hpi/stm/src/lang-smalltalk/images/images/Squeak4.5-benchmarks.image! ----QUIT/NOSAVE----{10 July 2014 . 5:36:06 pm} Squeak4.5-benchmarks.image priorSource: 1254571! ----STARTUP----{10 July 2014 . 5:44:18 pm} as /home/hub/hpi/stm/src/lang-smalltalk/images/Squeak4.5-12568.image! ----QUIT/NOSAVE----{10 July 2014 . 5:44:25 pm} Squeak4.5-12568.image priorSource: 1254571! ----QUIT/NOSAVE----{10 July 2014 . 6:11:07 pm} Squeak4.5-12568.image priorSource: 1254571! ----STARTUP----{10 July 2014 . 6:11:32 pm} as /Users/conrad/Repositories/master-project/lang-smalltalk/images/Squeak4.5-12568.image! ----QUIT/NOSAVE----{10 July 2014 . 6:22:16 pm} Squeak4.5-12568.image priorSource: 1254571! ----STARTUP----{17 July 2014 . 10:51:14 am} as /Users/conrad/Repositories/master-project/lang-smalltalk/images/Squeak4.5-12568.image! ----QUIT/NOSAVE----{17 July 2014 . 10:51:23 am} Squeak4.5-12568.image priorSource: 1254571! ----STARTUP----{17 July 2014 . 10:51:30 am} as /Users/conrad/Repositories/master-project/lang-smalltalk/images/Squeak4.5-12568.image! Smalltalk arguments! !Integer methodsFor: '*SPy-Benchmarks' stamp: 'cc 7/17/2014 10:52'! argTest | | SPyVM print: 'Arguments were', Smalltalk arguments.! ! ----SNAPSHOT----{17 July 2014 . 10:54:14 am} Squeak4.5-12568.image priorSource: 1254571! \ No newline at end of file diff --git a/images/Squeak4.5-12568.image b/images/Squeak4.5-12568.image index 72cb111b874a0858232e56cb11d5c95ddf8337eb..421974741f7d894e680a98bf8e91400220498d7a GIT binary patch [cut] diff --git a/targetimageloadingsmalltalk.py b/targetimageloadingsmalltalk.py --- a/targetimageloadingsmalltalk.py +++ b/targetimageloadingsmalltalk.py @@ -125,8 +125,8 @@ -r|--run [code string] -b|--benchmark [code string] -p|--poll_events - -s|--smalltalk-args [argument to pass] [image path, default: Squeak.image] + [-- [SMALLTALK_ARG [SMALLTALK_ARG ...]]] """ % argv[0] @@ -146,10 +146,13 @@ code = None as_benchmark = False smalltalk_args = [] + parse_smalltalk_args = False while idx < len(argv): arg = argv[idx] - if arg in ["-h", "--help"]: + if parse_smalltalk_args: + smalltalk_args.append(argv[idx]) + elif arg in ["-h", "--help"]: _usage(argv) return 0 elif arg in ["-j", "--jit"]: @@ -183,9 +186,8 @@ code = argv[idx + 1] as_benchmark = True idx += 1 - elif arg in ["-s", "--smalltalk-args"]: - smalltalk_args.append(argv[idx + 1]) - idx += 1 + elif arg in ["--"]: + parse_smalltalk_args = True elif path is None: path = argv[idx] else: From noreply at buildbot.pypy.org Thu Jul 17 15:44:43 2014 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 17 Jul 2014 15:44:43 +0200 (CEST) Subject: [pypy-commit] extradoc extradoc: Tweaks Message-ID: <20140717134443.21D961C0091@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: extradoc Changeset: r5361:49a0972a00d9 Date: 2014-07-17 15:44 +0200 http://bitbucket.org/pypy/extradoc/changeset/49a0972a00d9/ Log: Tweaks diff --git a/talk/ep2014/stm/talk.html b/talk/ep2014/stm/talk.html --- a/talk/ep2014/stm/talk.html +++ b/talk/ep2014/stm/talk.html @@ -383,6 +383,18 @@

Part 1 - Intro and Current Status

Introduction

PyPy-STM: Software Transactional Memory
On-going research project:
- by Remi Meier and myself
- helped by crowdfunding, thanks to all donors
+
Started as a EuroPython 2011 lightning talk

Why is there a GIL?

application-level locks still needed...
but can be very coarse:
- the idea is to make sure, internally, that one transaction -covers the whole time during which the lock was acquired
- even two big transactions can hopefully run in parallel
- even if they both acquire and release the same lock
- even if they both acquire and release the same lock

Demo 1

"Twisted apps made parallel out of the box"
Bottle web server

current status:
- basics work
- best case 25-40% overhead (much better than originally planned)
- tons of things to improve
- tons of things to improve
- tons of things to improve
- as long as they do multiple things that are "often independent"
Keep locks coarse-grained:
- need to track and fix issues in case of systematic conflicts
+

diff --git a/talk/ep2014/stm/talk.rst b/talk/ep2014/stm/talk.rst --- a/talk/ep2014/stm/talk.rst +++ b/talk/ep2014/stm/talk.rst @@ -17,6 +17,19 @@ --------------------------------- +Introduction +------------ + +* PyPy-STM: Software Transactional Memory + +* On-going research project: + + - by Remi Meier and myself + - helped by crowdfunding, thanks to all donors + +* Started as a EuroPython 2011 lightning talk + + Why is there a GIL? ------------------- @@ -132,12 +145,9 @@ * but *can be very coarse:* - - the idea is to make sure, internally, that one transaction - covers the whole time during which the lock was acquired - - even two big transactions can hopefully run in parallel - - even if they both acquire and release the *same* lock + - even if they both *acquire and release the same lock* Big Point @@ -149,6 +159,8 @@ Demo 1 ------ +* "Twisted apps made parallel out of the box" + * Bottle web server @@ -173,6 +185,7 @@ * current status: - basics work + - best case 25-40% overhead (much better than originally planned) - tons of things to improve - tons of things to improve - tons of things to improve @@ -201,6 +214,9 @@ - as long as they do multiple things that are "often independent" +* Keep locks coarse-grained: + + - need to track and fix issues in case of systematic conflicts Part 2 - Under The Hood From noreply at buildbot.pypy.org Thu Jul 17 22:14:26 2014 From: noreply at buildbot.pypy.org (yuyichao) Date: Thu, 17 Jul 2014 22:14:26 +0200 (CEST) Subject: [pypy-commit] pypy py3k-posix-decode: fix ascii decoding error in the posix module Message-ID: <20140717201426.B3E901C12F0@cobra.cs.uni-duesseldorf.de> Author: Yichao Yu Branch: py3k-posix-decode Changeset: r72465:9014f04de7c5 Date: 2014-07-09 12:41 +0800 http://bitbucket.org/pypy/pypy/changeset/9014f04de7c5/ Log: fix ascii decoding error in the posix module diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py --- a/pypy/module/posix/interp_posix.py +++ b/pypy/module/posix/interp_posix.py @@ -488,8 +488,7 @@ cur = os.getlogin() except OSError, e: raise wrap_oserror(space, e) - else: - return space.wrap(cur) + return space.fsdecode(space.wrapbytes(cur)) # ____________________________________________________________ @@ -693,14 +692,21 @@ except OSError, e: raise wrap_oserror(space, e) - at unwrap_spec(path='fsencode') -def readlink(space, path): +def readlink(space, w_path): "Return a string representing the path to which the symbolic link points." + is_unicode = space.isinstance_w(w_path, space.w_unicode) + if is_unicode: + path = space.fsencode_w(w_path) + else: + path = space.bytes0_w(w_path) try: result = os.readlink(path) except OSError, e: - raise wrap_oserror(space, e, path) - return space.wrap(result) + raise wrap_oserror2(space, e, w_path) + w_result = space.wrapbytes(result) + if is_unicode: + return space.fsdecode(w_result) + return w_result before_fork_hooks = [] after_fork_child_hooks = [] @@ -890,7 +896,8 @@ r = os.uname() except OSError, e: raise wrap_oserror(space, e) - l_w = [space.wrap(i) for i in [r[0], r[1], r[2], r[3], r[4]]] + l_w = [space.fsdecode(space.wrapbytes(i)) + for i in [r[0], r[1], r[2], r[3], r[4]]] return space.newtuple(l_w) def getuid(space): @@ -1217,7 +1224,7 @@ @unwrap_spec(fd=c_int) def ttyname(space, fd): try: - return space.wrap(os.ttyname(fd)) + return space.fsdecode(space.wrapbytes(os.ttyname(fd))) except OSError, e: raise wrap_oserror(space, e) @@ -1352,7 +1359,7 @@ Return the name of the controlling terminal for this process. """ - return space.wrap(os.ctermid()) + return space.fsdecode(space.wrapbytes(os.ctermid())) @unwrap_spec(fd=c_int) def device_encoding(space, fd): From noreply at buildbot.pypy.org Thu Jul 17 22:14:28 2014 From: noreply at buildbot.pypy.org (pjenvey) Date: Thu, 17 Jul 2014 22:14:28 +0200 (CEST) Subject: [pypy-commit] pypy py3k: Merged in yuyichao/pypy/py3k-posix-decode (pull request #247) Message-ID: <20140717201428.1F1991C12F0@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k Changeset: r72466:706409dc5979 Date: 2014-07-17 13:14 -0700 http://bitbucket.org/pypy/pypy/changeset/706409dc5979/ Log: Merged in yuyichao/pypy/py3k-posix-decode (pull request #247) fix ascii decoding error in the posix module diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py --- a/pypy/module/posix/interp_posix.py +++ b/pypy/module/posix/interp_posix.py @@ -488,8 +488,7 @@ cur = os.getlogin() except OSError, e: raise wrap_oserror(space, e) - else: - return space.wrap(cur) + return space.fsdecode(space.wrapbytes(cur)) # ____________________________________________________________ @@ -695,14 +694,21 @@ except OSError, e: raise wrap_oserror(space, e) - at unwrap_spec(path='fsencode') -def readlink(space, path): +def readlink(space, w_path): "Return a string representing the path to which the symbolic link points." + is_unicode = space.isinstance_w(w_path, space.w_unicode) + if is_unicode: + path = space.fsencode_w(w_path) + else: + path = space.bytes0_w(w_path) try: result = os.readlink(path) except OSError, e: - raise wrap_oserror(space, e, path) - return space.wrap(result) + raise wrap_oserror2(space, e, w_path) + w_result = space.wrapbytes(result) + if is_unicode: + return space.fsdecode(w_result) + return w_result before_fork_hooks = [] after_fork_child_hooks = [] @@ -892,7 +898,8 @@ r = os.uname() except OSError, e: raise wrap_oserror(space, e) - l_w = [space.wrap(i) for i in [r[0], r[1], r[2], r[3], r[4]]] + l_w = [space.fsdecode(space.wrapbytes(i)) + for i in [r[0], r[1], r[2], r[3], r[4]]] return space.newtuple(l_w) def getuid(space): @@ -1219,7 +1226,7 @@ @unwrap_spec(fd=c_int) def ttyname(space, fd): try: - return space.wrap(os.ttyname(fd)) + return space.fsdecode(space.wrapbytes(os.ttyname(fd))) except OSError, e: raise wrap_oserror(space, e) @@ -1354,7 +1361,7 @@ Return the name of the controlling terminal for this process. """ - return space.wrap(os.ctermid()) + return space.fsdecode(space.wrapbytes(os.ctermid())) @unwrap_spec(fd=c_int) def device_encoding(space, fd): From noreply at buildbot.pypy.org Thu Jul 17 22:14:45 2014 From: noreply at buildbot.pypy.org (pjenvey) Date: Thu, 17 Jul 2014 22:14:45 +0200 (CEST) Subject: [pypy-commit] pypy py3k-posix-decode: Close branch py3k-posix-decode Message-ID: <20140717201445.70AE41C12F0@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3k-posix-decode Changeset: r72467:42120ecaf5ae Date: 2014-07-17 13:14 -0700 http://bitbucket.org/pypy/pypy/changeset/42120ecaf5ae/ Log: Close branch py3k-posix-decode From noreply at buildbot.pypy.org Fri Jul 18 14:08:22 2014 From: noreply at buildbot.pypy.org (anton_gulenko) Date: Fri, 18 Jul 2014 14:08:22 +0200 (CEST) Subject: [pypy-commit] lang-smalltalk storage: Filed in BalloonEngine class form the image on the vref branch. Message-ID: <20140718120822.581AC1C01D2@cobra.cs.uni-duesseldorf.de> Author: Anton Gulenko Branch: storage Changeset: r897:f60624a57139 Date: 2014-07-10 15:50 +0200 http://bitbucket.org/pypy/lang-smalltalk/changeset/f60624a57139/ Log: Filed in BalloonEngine class form the image on the vref branch. This removes some errors at startup, but also adds some new errors. Image feels smoother, fps is up. diff too long, truncating to 2000 out of 2375 lines diff --git a/images/Squeak4.5-noBitBlt.changes b/images/Squeak4.5-noBitBlt.changes --- a/images/Squeak4.5-noBitBlt.changes +++ b/images/Squeak4.5-noBitBlt.changes @@ -12208,4 +12208,144 @@ ]. "self footer." - ^ self! ! ----QUIT----{2 April 2014 . 11:59:41 am} Squeak4.5-noBitBlt.image priorSource: 15812182! ----STARTUP----{3 July 2014 . 11:14:14 am} as C:\Dev\lang-smalltalk\images\Squeak4.5-noBitBlt.image! SystemOrganization addCategory: #Anton! Object subclass: #AntonMatrix instanceVariableNames: 'fields columns rows' classVariableNames: '' poolDictionaries: '' category: 'Anton'! Object subclass: #AntonMatrix instanceVariableNames: 'fields columns rows' classVariableNames: '' poolDictionaries: '' category: 'Anton'! !AntonMatrix methodsFor: 'accessing' stamp: 'ag 7/3/2014 10:17'! at: point ^ self x: point x y: point y! ! !AntonMatrix methodsFor: 'accessing' stamp: 'ag 7/3/2014 10:16'! at: point put: number ^ self x: point x y: point y put: number! ! !AntonMatrix methodsFor: 'accessing' stamp: 'ag 7/3/2014 10:12'! columns ^ columns! ! !AntonMatrix methodsFor: 'accessing' stamp: 'ag 7/3/2014 10:55'! fieldsDo: block (1 to: self rows) do: [ :row | (1 to: self columns) do: [ :column | block value: row value: column ] ].! ! !AntonMatrix methodsFor: 'accessing' stamp: 'ag 7/3/2014 10:56'! fill: block self fieldsDo: [ :x :y | self x: x y: y put: (block value: x value: y) ].! ! !AntonMatrix methodsFor: 'accessing' stamp: 'ag 7/3/2014 10:12'! rows ^ rows! ! !AntonMatrix methodsFor: 'accessing' stamp: 'ag 7/3/2014 10:17'! x: x y: y ^ fields at: (self offsetX: x y: y)! ! !AntonMatrix methodsFor: 'accessing' stamp: 'ag 7/3/2014 10:17'! x: x y: y put: number fields at: (self offsetX: x y: y) put: number! ! !AntonMatrix methodsFor: 'private' stamp: 'ag 7/3/2014 10:44'! offsetX: x y: y ^ (y-1) * columns + x! ! !AntonMatrix methodsFor: 'initialization' stamp: 'ag 7/3/2014 10:43'! initializeFields: f rows: r rows := r. (f size \\ r) = 0 ifFalse: [ self error: 'Illegal initialization.' ]. columns := f size / r. fields := f.! ! !AntonMatrix methodsFor: 'initialization' stamp: 'ag 7/3/2014 10:18'! initializeRows: r columns: c rows := r. columns := c. fields := Array new: rows * columns.! ! !AntonMatrix methodsFor: 'math' stamp: 'ag 7/3/2014 10:30'! * other | result | (self columns = other rows and: [ self rows = other columns ]) ifFalse: [ ^ self error: 'Cannot multiply, wrong dimensions.' ]. result := AntonMatrix rows: self rows columns: other columns. (1 to: self rows) do: [ :row | (1 to: other columns) do: [ :column | | value | value := 0. (1 to: self columns) do: [ :i | value := value + ((self x: i y: row) * (other x: column y: i)) ]. result x: column y: row put: value ] ]. ^ result! ! !AntonMatrix methodsFor: 'printing' stamp: 'ag 7/3/2014 10:47'! printOn: s (1 to: self rows) do: [ :row | (1 to: self columns) do: [ :column | s nextPutAll: (self x: column y: row) asString. s nextPutAll: ' ' ]. s nextPutAll: String cr ].! ! !AntonMatrix methodsFor: 'benchmarking' stamp: 'ag 7/3/2014 11:01'! fillRandomFloats: generator | max | max := SmallInteger maxVal sqrt asInteger. self fill: [ :x :y | max atRandom: generator ].! ! !AntonMatrix methodsFor: 'benchmarking' stamp: 'ag 7/3/2014 11:02'! fillRandomInts: generator "Fill with SmallInteger values small enough to stay SmallIntegers after multiplication." self fill: [ :x :y | generator next * 100 ].! ! "-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- "! AntonMatrix class instanceVariableNames: ''! !AntonMatrix class methodsFor: 'benchmarking' stamp: 'ag 7/3/2014 11:13'! benchFloats: numOfRuns multiplicationsPerRun: mults rows: r columns: c | generator | generator := Random seed: 23456432. numOfRuns timesRepeat: [ | a b | a := AntonMatrix rows: r columns: c. b := AntonMatrix rows: r columns: c. a fillRandomFloats: generator. b fillRandomFloats: generator. mults timesRepeat: [ a * b ] ].! ! !AntonMatrix class methodsFor: 'benchmarking' stamp: 'ag 7/3/2014 11:03'! benchInts: numOfRuns multiplicationsPerRun: mults rows: r columns: c | generator | generator := Random seed: 23456432. numOfRuns timesRepeat: [ | a b | a := AntonMatrix rows: r columns: c. b := AntonMatrix rows: r columns: c. a fillRandomInts: generator. b fillRandomInts: generator. mults timesRepeat: [ a * b ] ].! ! !AntonMatrix class methodsFor: 'instance creation' stamp: 'ag 7/3/2014 10:35'! fields: fields rows: r ^ self basicNew initializeFields: fields rows: r! ! !AntonMatrix class methodsFor: 'instance creation' stamp: 'ag 7/3/2014 10:19'! rows: r columns: c ^ self basicNew initializeRows: r columns: c; yourself! ! Object subclass: #AntonMatrixBenchmark instanceVariableNames: '' classVariableNames: 'Cols Mults NumOfRuns Rows' poolDictionaries: '' category: 'Anton'! Object subclass: #AntonMatrixBenchmark instanceVariableNames: '' classVariableNames: 'Cols Mults NumOfRuns Rows' poolDictionaries: '' category: 'Anton'! !AntonMatrixBenchmark methodsFor: 'bench' stamp: 'ag 7/3/2014 11:19'! benchFloats AntonMatrix benchFloats: NumOfRuns multiplicationsPerRun: Mults rows: Rows columns: Cols.! ! !AntonMatrixBenchmark methodsFor: 'bench' stamp: 'ag 7/3/2014 11:18'! benchInts AntonMatrix benchInts: NumOfRuns multiplicationsPerRun: Mults rows: Rows columns: Cols.! ! "-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- "! AntonMatrixBenchmark class instanceVariableNames: ''! !AntonMatrixBenchmark class methodsFor: 'initialization' stamp: 'ag 7/3/2014 11:24'! config: spec | tokens nextInt | tokens := spec findTokens: ' '. nextInt := [ :default | (tokens ifEmpty: [ nil ] ifNotEmptyDo: #removeFirst) asInteger ifNil: [ default ] ]. NumOfRuns := nextInt value: 10. Mults := nextInt value: 100. Rows := nextInt value: 100. Cols := nextInt value: 100.! ! !AntonMatrixBenchmark class methodsFor: 'initialization' stamp: 'ag 7/3/2014 11:20'! initialize super initialize. NumOfRuns := 10. Mults := 100. Cols := 100. Rows := 100.! ! AntonMatrixBenchmark initialize! ----End fileIn of C:\Dev\lang-smalltalk\Anton.st----! !SmallInteger methodsFor: 'as yet unclassified' stamp: 'ag 7/3/2014 11:27'! benchMatrixInt: spec AntonMatrixBenchmark config: spec. ! ! !SmallInteger methodsFor: 'as yet unclassified' stamp: 'ag 7/3/2014 11:28' prior: 49374034! benchMatrixInt: spec AntonMatrixBenchmark config: spec. ^ Benchmarks runMatching: 'AntonMatrix' iterations: self benchmarkIterations! ! !SmallInteger methodsFor: 'as yet unclassified' stamp: 'ag 7/3/2014 11:28'! benchMatrix: spec AntonMatrixBenchmark config: spec. ^ Benchmarks runMatching: 'AntonMatrix' iterations: self benchmarkIterations! ! SystemOrganization renameCategory: #Anton toBe: #'Matrix-Benchmarks'! Smalltalk renameClassNamed: #AntonMatrix as: #BenchMatrix! Object subclass: #SimpleMatrixBenchmark instanceVariableNames: '' classVariableNames: 'Cols Mults NumOfRuns Rows' poolDictionaries: '' category: 'Matrix-Benchmarks'! Smalltalk removeClassNamed: #SimpleMatrixBenchmark! Smalltalk renameClassNamed: #AntonMatrixBenchmark as: #SimpleMatrixBenchmark! SmallInteger removeSelector: #benchMatrixInt:! !SmallInteger methodsFor: 'as yet unclassified' stamp: 'ag 7/3/2014 11:30' prior: 49374406! benchMatrix: spec SimpleMatrixBenchmark config: spec. ^ Benchmarks runMatching: 'SimpleMatrixBenchmark' iterations: self benchmarkIterations! ! !Benchmarks class methodsFor: 'benchmarks' stamp: 'ag 7/3/2014 11:31' prior: 49367383! allBenchmarks ^ { CPBAStarBenchmark. CPBBinaryTreeBenchmark. CPBBlowfishSuite. CPBChameneosBenchmark. CPBDeltaBlueBenchmark. CPBMandelbrotBenchmarkSuite. CPBNBodyBenchmark. "CPBPolymorphyBenchmark." "Commented out because it compiled code in setup." CPBRichardsBenchmark. CPBSplayTreeBenchmark. SimpleMatrixBenchmark. }! ! ----QUIT----{3 July 2014 . 11:32:10 am} Squeak4.5-noBitBlt.image priorSource: 15813551! ----STARTUP----{3 July 2014 . 11:34:49 am} as C:\Dev\lang-smalltalk\images\Squeak4.5-noBitBlt.image! SMarkSuite subclass: #SimpleMatrixBenchmark instanceVariableNames: '' classVariableNames: 'Cols Mults NumOfRuns Rows' poolDictionaries: '' category: 'Matrix-Benchmarks'! !SimpleMatrixBenchmark methodsFor: 'bench' stamp: 'ag 7/3/2014 11:37' prior: 49372902! benchFloats BenchMatrix benchFloats: NumOfRuns multiplicationsPerRun: Mults rows: Rows columns: Cols.! ! !SimpleMatrixBenchmark methodsFor: 'bench' stamp: 'ag 7/3/2014 11:37' prior: 49373080! benchInts BenchMatrix benchInts: NumOfRuns multiplicationsPerRun: Mults rows: Rows columns: Cols.! ! SimpleMatrixBenchmark config: '5 5 5 5'! Benchmarks runMatching: 'SimpleMatrixBenchmark' iterations: 1! !SimpleMatrixBenchmark class methodsFor: 'initialization' stamp: 'ag 7/3/2014 11:38' prior: 49373773! initialize super initialize. NumOfRuns := 10. Mults := 10. Cols := 10. Rows := 10.! ! self initialize! !SimpleMatrixBenchmark class methodsFor: 'initialization' stamp: 'ag 7/3/2014 11:39' prior: 49376651! initialize "self initialize" super initialize. NumOfRuns := 10. Mults := 10. Cols := 10. Rows := 10.! ! ----QUIT----{3 July 2014 . 11:39:08 am} Squeak4.5-noBitBlt.image priorSource: 15821257! ----STARTUP----{3 July 2014 . 11:48:06 am} as C:\Dev\lang-smalltalk\images\Squeak4.5-noBitBlt.image! !BenchMatrix class methodsFor: 'benchmarking' stamp: 'ag 7/3/2014 11:49' prior: 49371447! benchFloats: numOfRuns multiplicationsPerRun: mults rows: r columns: c | generator | generator := Random seed: 23456432. numOfRuns timesRepeat: [ | a b | a := BenchMatrix rows: r columns: c. b := BenchMatrix rows: c columns: r. a fillRandomFloats: generator. b fillRandomFloats: generator. mults timesRepeat: [ a * b ] ].! ! !BenchMatrix class methodsFor: 'benchmarking' stamp: 'ag 7/3/2014 11:49' prior: 49371861! benchInts: numOfRuns multiplicationsPerRun: mults rows: r columns: c | generator | generator := Random seed: 23456432. numOfRuns timesRepeat: [ | a b | a := BenchMatrix rows: r columns: c. b := BenchMatrix rows: c columns: r. a fillRandomInts: generator. b fillRandomInts: generator. mults timesRepeat: [ a * b ] ].! ! !BenchMatrix methodsFor: 'accessing' stamp: 'ag 7/3/2014 11:51' prior: 49368902! fieldsDo: block (1 to: self rows) do: [ :row | (1 to: self columns) do: [ :column | block value: column value: row ] ].! ! 1 benchMatrix: '1 10 100 10'! 1 benchMatrix: '1 10 100 10'! 1 benchMatrix: '1 10 10 100'! 1 benchMatrix: '1 10 10 1000'! ----QUIT----{3 July 2014 . 11:51:44 am} Squeak4.5-noBitBlt.image priorSource: 15822543! ----STARTUP----{3 July 2014 . 12:30:20 pm} as C:\Dev\lang-smalltalk\images\Squeak4.5-noBitBlt.image! Array variableSubclass: #BenchMatrix instanceVariableNames: 'columns rows' classVariableNames: '' poolDictionaries: '' category: 'Matrix-Benchmarks'! Array variableSubclass: #BenchMatrix instanceVariableNames: 'rows' classVariableNames: '' poolDictionaries: '' category: 'Matrix-Benchmarks'! BenchMatrix removeSelector: #at:! BenchMatrix removeSelector: #at:put:! !BenchMatrix methodsFor: 'accessing' stamp: 'ag 7/3/2014 12:31' prior: 49368813! columns ^ self size / rows! ! !BenchMatrix methodsFor: 'accessing' stamp: 'ag 7/3/2014 12:31' prior: 49379008! columns ^ self size / rows! ! 11/2! 11//2! !BenchMatrix methodsFor: 'accessing' stamp: 'ag 7/3/2014 12:33' prior: 49378103! fieldsDo: block (1 to: self size) do: [ :i | block value: i \\ rows value: i // rows ].! ! !BenchMatrix methodsFor: 'accessing' stamp: 'ag 7/3/2014 12:34' prior: 49379251! fieldsDo: block 1 to: self size do: [ :i | block value: i \\ rows value: i // rows ].! ! !BenchMatrix methodsFor: 'accessing' stamp: 'ag 7/3/2014 12:34' prior: 49369340! x: x y: y ^ self at: (self offsetX: x y: y)! ! !BenchMatrix methodsFor: 'accessing' stamp: 'ag 7/3/2014 12:34' prior: 49369457! x: x y: y put: number self at: (self offsetX: x y: y) put: number! ! a := BenchMatrix fields: #( 3 2 1 1 0 2 ) rows: 2! b := BenchMatrix fields: #( 1 2 0 1 4 0 ) rows: 3! a! a! a rows! a columns! BenchMatrix removeSelector: #initializeRows:columns:! !BenchMatrix methodsFor: 'initialization' stamp: 'ag 7/3/2014 12:36'! initializeRows: r rows := r.! ! BenchMatrix removeSelector: #initializeFields:rows:! !BenchMatrix class methodsFor: 'instance creation' stamp: 'ag 7/3/2014 12:37' prior: 49372274! fields: fields rows: r | columns f rows | rows := r. (f size \\ r) = 0 ifFalse: [ self error: 'Illegal initialization.' ]. columns := f size / r. " fields := f." ^ self basicNew initializeFields: fields rows: r! ! !BenchMatrix methodsFor: 'initialization' stamp: 'ag 7/3/2014 12:37'! rows: r rows := r.! ! BenchMatrix removeSelector: #initializeRows:! Array withAll: #(1 2 3)! !BenchMatrix class methodsFor: 'instance creation' stamp: 'ag 7/3/2014 12:39' prior: 49380248! fields: fields rows: r (fields size \\ r) = 0 ifFalse: [ self error: 'Illegal initialization.' ]. ^ (self withAll: fields) rows: r; yourself! ! !BenchMatrix methodsFor: 'accessing' stamp: 'ag 7/3/2014 12:39' prior: 49379122! columns ^ self size // rows! ! !BenchMatrix class methodsFor: 'instance creation' stamp: 'ag 7/3/2014 12:40' prior: 49372433! rows: r columns: c ^ (self new: r * c) rows: r; fillZeros; yourself! ! !BenchMatrix methodsFor: 'initialization' stamp: 'ag 7/3/2014 12:40'! fillZeros self fill: [ :x :y | 0 ].! ! i! i \\ rows! i //rows! rows! !BenchMatrix methodsFor: 'accessing' stamp: 'ag 7/3/2014 12:42' prior: 49379428! fieldsDo: block 1 to: self size do: [ :i | block value: i \\ rows + 1 value: i // rows + 1 ].! ! x := BenchMatrix rows: 4 columns: 3.! x! o := OrderedCollection new.! x fieldsDo: [ :x :y | o add: x -> y ].! o! !BenchMatrix methodsFor: 'accessing' stamp: 'ag 7/3/2014 12:43' prior: 49381404! fieldsDo: block 0 to: self size do: [ :i | block value: i \\ rows + 1 value: i // rows + 1 ].! ! o := OrderedCollection new.! x fieldsDo: [ :x :y | o add: x -> y ].! o! x := BenchMatrix rows: 4 columns: 3.! x! !BenchMatrix methodsFor: 'accessing' stamp: 'ag 7/3/2014 12:44' prior: 49381705! fieldsDo: block 0 to: self size + 1 do: [ :i | block value: i \\ rows + 1 value: i // rows + 1 ].! ! x := BenchMatrix rows: 4 columns: 3.! x! o := OrderedCollection new.! x fieldsDo: [ :x :y | o add: x -> y ].! o! o size! x size! o size! o asSet size! !BenchMatrix methodsFor: 'accessing' stamp: 'ag 7/3/2014 12:46' prior: 49382006! fieldsDo: block 1 to: self size do: [ :i | block value: i \\ rows + 1 value: i // rows + 1 ].! ! o := OrderedCollection new.! x fieldsDo: [ :x :y | o add: x -> y ].! o! o size! o ! 1 \\ 4! !BenchMatrix methodsFor: 'accessing' stamp: 'ag 7/3/2014 12:46' prior: 49382353! fieldsDo: block 1 to: self size do: [ :i | block value: i \\ rows value: i // rows + 1 ].! ! o := OrderedCollection new.! x fieldsDo: [ :x :y | o add: x -> y ].! o size! o ! !BenchMatrix methodsFor: 'accessing' stamp: 'ag 7/3/2014 12:50' prior: 49382634! fieldsDo: block | columns | columns := self columns. 1 to: self size do: [ :i | block value: i \\ columns value: i // columns + 1 ].! ! Array variableSubclass: #BenchMatrix instanceVariableNames: 'rows columns' classVariableNames: '' poolDictionaries: '' category: 'Matrix-Benchmarks'! Array variableSubclass: #BenchMatrix instanceVariableNames: 'rows columns' classVariableNames: '' poolDictionaries: '' category: 'Matrix-Benchmarks'! !BenchMatrix methodsFor: 'accessing' stamp: 'ag 7/3/2014 12:51' prior: 49382898! fieldsDo: block 1 to: self size do: [ :i | block value: i \\ columns value: i // columns + 1 ].! ! !BenchMatrix methodsFor: 'accessing' stamp: 'ag 7/3/2014 12:51' prior: 49380969! columns ^ columns! ! !BenchMatrix methodsFor: 'initialization' stamp: 'ag 7/3/2014 12:51' prior: 49380543! rows: r rows := r. columns := self size // r.! ! x := BenchMatrix rows: 4 columns: 3.! x! o := OrderedCollection new. x fieldsDo: [ :x :y | o add: x -> y ]. ! ox! o! !BenchMatrix methodsFor: 'initialization' stamp: 'ag 7/3/2014 12:52' prior: 49381247! fillZeros self atAllPut: 0.! ! !BenchMatrix methodsFor: 'accessing' stamp: 'ag 7/3/2014 12:53' prior: 49383432! fieldsDo: block 0 to: self size - 1 do: [ :i | block value: i \\ columns + 1 value: i // columns + 1 ].! ! o := OrderedCollection new. x fieldsDo: [ :x :y | o add: x -> y ].! o size! o! a := BenchMatrix fields: #( 3 2 1 1 0 2 ) rows: 2! b := BenchMatrix fields: #( 1 2 0 1 4 0 ) rows: 3! !BenchMatrix methodsFor: 'math' stamp: 'ag 7/3/2014 12:55' prior: 49370092! * other | result | (self columns = other rows and: [ self rows = other columns ]) ifFalse: [ ^ self error: 'Cannot multiply, wrong dimensions.' ]. result := BenchMatrix rows: self rows columns: other columns. (1 to: self rows) do: [ :row | (1 to: other columns) do: [ :column | | value | value := 0. (1 to: self columns) do: [ :i | value := value + ((self x: i y: row) * (other x: column y: i)) ]. result x: column y: row put: value ] ]. ^ result! ! a * b! self assert: (Array withAll: (a * b)) = #(7 8 9 2)! BenchMatrix class organization addCategory: #test! !BenchMatrix class methodsFor: 'test' stamp: 'ag 7/3/2014 12:57'! tinyTest "self tinyTest" | a b | a := BenchMatrix fields: #( 3 2 1 1 0 2 ) rows: 2. b := BenchMatrix fields: #( 1 2 0 1 4 0 ) rows: 3. self assert: (Array withAll: (a * b)) = #(7 8 9 2).! ! self tinyTest! 1 benchMatrix: '1 3 5 5'! 1 benchMatrix: '1 10 5 5'! ----QUIT----{3 July 2014 . 12:58:52 pm} Squeak4.5-noBitBlt.image priorSource: 15823926! ----STARTUP----{3 July 2014 . 1:05:04 pm} as C:\Dev\lang-smalltalk\images\Squeak4.5-noBitBlt.image! a := BenchMatrix rows: 20 columns: 20. b := BenchMatrix rows: 20 columns: 20. generator := Random seed: 13243456. a fillRandomInts: generator. b fillRandomInts: generator.! (a collect: #class) asSet! (b collect: #class) asSet! a := BenchMatrix rows: 20 columns: 20. b := BenchMatrix rows: 20 columns: 20. generator := Random seed: 13243456. a fillRandomFloats: generator. b fillRandomInts: generator.! (b collect: #class) asSet! (a collect: #class) asSet! !BenchMatrix methodsFor: 'benchmarking' stamp: 'ag 7/3/2014 13:08' prior: 49371100! fillRandomInts: generator "Fill with SmallInteger values small enough to stay SmallIntegers after multiplication." | max | max := SmallInteger maxVal sqrt asInteger. self fill: [ :x :y | max atRandom: generator ]. ! ! !BenchMatrix methodsFor: 'benchmarking' stamp: 'ag 7/3/2014 13:08' prior: 49370897! fillRandomFloats: generator self fill: [ :x :y | generator next * 100 ].! ! a := BenchMatrix rows: 20 columns: 20. b := BenchMatrix rows: 20 columns: 20. generator := Random seed: 13243456. a fillRandomFloats: generator. b fillRandomInts: generator. (a collect: #class) asSet ! a := BenchMatrix rows: 20 columns: 20. b := BenchMatrix rows: 20 columns: 20. generator := Random seed: 13243456. a fillRandomFloats: generator. b fillRandomInts: generator. (b collect: #class) asSet ! a := BenchMatrix rows: 20 columns: 20. b := BenchMatrix rows: 20 columns: 20. generator := Random seed: 13243456. a fillRandomInts: generator. b fillRandomInts: generator. ! c := a * b! (c collect: #class) asSet! !BenchMatrix methodsFor: 'benchmarking' stamp: 'ag 7/3/2014 13:09' prior: 49386143! fillRandomInts: generator "Fill with SmallInteger values small enough to stay SmallIntegers after multiplication." | max | max := 1000. self fill: [ :x :y | max atRandom: generator ]. ! ! a := BenchMatrix rows: 20 columns: 20. b := BenchMatrix rows: 20 columns: 20. generator := Random seed: 13243456. a fillRandomInts: generator. b fillRandomInts: generator. c := a * b. (c collect: #class) asSet! ----QUIT----{3 July 2014 . 1:09:37 pm} Squeak4.5-noBitBlt.image priorSource: 15830973! ----STARTUP----{3 July 2014 . 8:26:43 pm} as C:\Dev\lang-smalltalk\images\Squeak4.5-noBitBlt.image! a := BenchMatrix rows: 20 columns: 20. b := BenchMatrix rows: 20 columns: 20. generator := Random seed: 13243456. a fillRandomInts: generator. b fillRandomInts: generator.! c := a * b.! (c collect: #class) asSet! c! !SmallInteger methodsFor: 'as yet unclassified' stamp: 'ag 7/3/2014 20:27'! testMatrix ! ! !SmallInteger methodsFor: 'as yet unclassified' stamp: 'ag 7/3/2014 20:28' prior: 49388134! testMatrix | a b c generator | a := BenchMatrix rows: 20 columns: 20. b := BenchMatrix rows: 20 columns: 20. generator := Random seed: 13243456. a fillRandomInts: generator. b fillRandomInts: generator. c := a * b. ^ (c collect: #class) asSet asString! ! 5 testMatrix! ----SNAPSHOT----{3 July 2014 . 8:28:40 pm} Squeak4.5-noBitBlt.1.image priorSource: 15833215! ----QUIT----{3 July 2014 . 8:28:49 pm} Squeak4.5-noBitBlt.1.image priorSource: 15834093! ----STARTUP----{3 July 2014 . 9:02:43 pm} as C:\Dev\lang-smalltalk\images\Squeak4.5-noBitBlt.image! !BenchMatrix methodsFor: 'initialization' stamp: 'ag 7/3/2014 21:03' prior: 49383727! rows: r rows := r asFloat. columns := (self size // r) asFloat.! ! !BenchMatrix methodsFor: 'accessing' stamp: 'ag 7/3/2014 21:03' prior: 49369257! rows ^ rows asInteger! ! !BenchMatrix methodsFor: 'accessing' stamp: 'ag 7/3/2014 21:03' prior: 49383617! columns ^ columns asInteger! ! !BenchMatrix methodsFor: 'accessing' stamp: 'ag 7/3/2014 21:04' prior: 49384103! fieldsDo: block 0 to: self size - 1 do: [ :i | block value: i \\ self columns + 1 value: i // self columns + 1 ].! ! ----QUIT----{3 July 2014 . 9:04:33 pm} Squeak4.5-noBitBlt.image priorSource: 15834187! \ No newline at end of file + ^ self! ! ----QUIT----{2 April 2014 . 11:59:41 am} Squeak4.5-noBitBlt.image priorSource: 15812182! ----STARTUP----{3 July 2014 . 11:14:14 am} as C:\Dev\lang-smalltalk\images\Squeak4.5-noBitBlt.image! SystemOrganization addCategory: #Anton! Object subclass: #AntonMatrix instanceVariableNames: 'fields columns rows' classVariableNames: '' poolDictionaries: '' category: 'Anton'! Object subclass: #AntonMatrix instanceVariableNames: 'fields columns rows' classVariableNames: '' poolDictionaries: '' category: 'Anton'! !AntonMatrix methodsFor: 'accessing' stamp: 'ag 7/3/2014 10:17'! at: point ^ self x: point x y: point y! ! !AntonMatrix methodsFor: 'accessing' stamp: 'ag 7/3/2014 10:16'! at: point put: number ^ self x: point x y: point y put: number! ! !AntonMatrix methodsFor: 'accessing' stamp: 'ag 7/3/2014 10:12'! columns ^ columns! ! !AntonMatrix methodsFor: 'accessing' stamp: 'ag 7/3/2014 10:55'! fieldsDo: block (1 to: self rows) do: [ :row | (1 to: self columns) do: [ :column | block value: row value: column ] ].! ! !AntonMatrix methodsFor: 'accessing' stamp: 'ag 7/3/2014 10:56'! fill: block self fieldsDo: [ :x :y | self x: x y: y put: (block value: x value: y) ].! ! !AntonMatrix methodsFor: 'accessing' stamp: 'ag 7/3/2014 10:12'! rows ^ rows! ! !AntonMatrix methodsFor: 'accessing' stamp: 'ag 7/3/2014 10:17'! x: x y: y ^ fields at: (self offsetX: x y: y)! ! !AntonMatrix methodsFor: 'accessing' stamp: 'ag 7/3/2014 10:17'! x: x y: y put: number fields at: (self offsetX: x y: y) put: number! ! !AntonMatrix methodsFor: 'private' stamp: 'ag 7/3/2014 10:44'! offsetX: x y: y ^ (y-1) * columns + x! ! !AntonMatrix methodsFor: 'initialization' stamp: 'ag 7/3/2014 10:43'! initializeFields: f rows: r rows := r. (f size \\ r) = 0 ifFalse: [ self error: 'Illegal initialization.' ]. columns := f size / r. fields := f.! ! !AntonMatrix methodsFor: 'initialization' stamp: 'ag 7/3/2014 10:18'! initializeRows: r columns: c rows := r. columns := c. fields := Array new: rows * columns.! ! !AntonMatrix methodsFor: 'math' stamp: 'ag 7/3/2014 10:30'! * other | result | (self columns = other rows and: [ self rows = other columns ]) ifFalse: [ ^ self error: 'Cannot multiply, wrong dimensions.' ]. result := AntonMatrix rows: self rows columns: other columns. (1 to: self rows) do: [ :row | (1 to: other columns) do: [ :column | | value | value := 0. (1 to: self columns) do: [ :i | value := value + ((self x: i y: row) * (other x: column y: i)) ]. result x: column y: row put: value ] ]. ^ result! ! !AntonMatrix methodsFor: 'printing' stamp: 'ag 7/3/2014 10:47'! printOn: s (1 to: self rows) do: [ :row | (1 to: self columns) do: [ :column | s nextPutAll: (self x: column y: row) asString. s nextPutAll: ' ' ]. s nextPutAll: String cr ].! ! !AntonMatrix methodsFor: 'benchmarking' stamp: 'ag 7/3/2014 11:01'! fillRandomFloats: generator | max | max := SmallInteger maxVal sqrt asInteger. self fill: [ :x :y | max atRandom: generator ].! ! !AntonMatrix methodsFor: 'benchmarking' stamp: 'ag 7/3/2014 11:02'! fillRandomInts: generator "Fill with SmallInteger values small enough to stay SmallIntegers after multiplication." self fill: [ :x :y | generator next * 100 ].! ! "-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- "! AntonMatrix class instanceVariableNames: ''! !AntonMatrix class methodsFor: 'benchmarking' stamp: 'ag 7/3/2014 11:13'! benchFloats: numOfRuns multiplicationsPerRun: mults rows: r columns: c | generator | generator := Random seed: 23456432. numOfRuns timesRepeat: [ | a b | a := AntonMatrix rows: r columns: c. b := AntonMatrix rows: r columns: c. a fillRandomFloats: generator. b fillRandomFloats: generator. mults timesRepeat: [ a * b ] ].! ! !AntonMatrix class methodsFor: 'benchmarking' stamp: 'ag 7/3/2014 11:03'! benchInts: numOfRuns multiplicationsPerRun: mults rows: r columns: c | generator | generator := Random seed: 23456432. numOfRuns timesRepeat: [ | a b | a := AntonMatrix rows: r columns: c. b := AntonMatrix rows: r columns: c. a fillRandomInts: generator. b fillRandomInts: generator. mults timesRepeat: [ a * b ] ].! ! !AntonMatrix class methodsFor: 'instance creation' stamp: 'ag 7/3/2014 10:35'! fields: fields rows: r ^ self basicNew initializeFields: fields rows: r! ! !AntonMatrix class methodsFor: 'instance creation' stamp: 'ag 7/3/2014 10:19'! rows: r columns: c ^ self basicNew initializeRows: r columns: c; yourself! ! Object subclass: #AntonMatrixBenchmark instanceVariableNames: '' classVariableNames: 'Cols Mults NumOfRuns Rows' poolDictionaries: '' category: 'Anton'! Object subclass: #AntonMatrixBenchmark instanceVariableNames: '' classVariableNames: 'Cols Mults NumOfRuns Rows' poolDictionaries: '' category: 'Anton'! !AntonMatrixBenchmark methodsFor: 'bench' stamp: 'ag 7/3/2014 11:19'! benchFloats AntonMatrix benchFloats: NumOfRuns multiplicationsPerRun: Mults rows: Rows columns: Cols.! ! !AntonMatrixBenchmark methodsFor: 'bench' stamp: 'ag 7/3/2014 11:18'! benchInts AntonMatrix benchInts: NumOfRuns multiplicationsPerRun: Mults rows: Rows columns: Cols.! ! "-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- "! AntonMatrixBenchmark class instanceVariableNames: ''! !AntonMatrixBenchmark class methodsFor: 'initialization' stamp: 'ag 7/3/2014 11:24'! config: spec | tokens nextInt | tokens := spec findTokens: ' '. nextInt := [ :default | (tokens ifEmpty: [ nil ] ifNotEmptyDo: #removeFirst) asInteger ifNil: [ default ] ]. NumOfRuns := nextInt value: 10. Mults := nextInt value: 100. Rows := nextInt value: 100. Cols := nextInt value: 100.! ! !AntonMatrixBenchmark class methodsFor: 'initialization' stamp: 'ag 7/3/2014 11:20'! initialize super initialize. NumOfRuns := 10. Mults := 100. Cols := 100. Rows := 100.! ! AntonMatrixBenchmark initialize! ----End fileIn of C:\Dev\lang-smalltalk\Anton.st----! !SmallInteger methodsFor: 'as yet unclassified' stamp: 'ag 7/3/2014 11:27'! benchMatrixInt: spec AntonMatrixBenchmark config: spec. ! ! !SmallInteger methodsFor: 'as yet unclassified' stamp: 'ag 7/3/2014 11:28' prior: 49374034! benchMatrixInt: spec AntonMatrixBenchmark config: spec. ^ Benchmarks runMatching: 'AntonMatrix' iterations: self benchmarkIterations! ! !SmallInteger methodsFor: 'as yet unclassified' stamp: 'ag 7/3/2014 11:28'! benchMatrix: spec AntonMatrixBenchmark config: spec. ^ Benchmarks runMatching: 'AntonMatrix' iterations: self benchmarkIterations! ! SystemOrganization renameCategory: #Anton toBe: #'Matrix-Benchmarks'! Smalltalk renameClassNamed: #AntonMatrix as: #BenchMatrix! Object subclass: #SimpleMatrixBenchmark instanceVariableNames: '' classVariableNames: 'Cols Mults NumOfRuns Rows' poolDictionaries: '' category: 'Matrix-Benchmarks'! Smalltalk removeClassNamed: #SimpleMatrixBenchmark! Smalltalk renameClassNamed: #AntonMatrixBenchmark as: #SimpleMatrixBenchmark! SmallInteger removeSelector: #benchMatrixInt:! !SmallInteger methodsFor: 'as yet unclassified' stamp: 'ag 7/3/2014 11:30' prior: 49374406! benchMatrix: spec SimpleMatrixBenchmark config: spec. ^ Benchmarks runMatching: 'SimpleMatrixBenchmark' iterations: self benchmarkIterations! ! !Benchmarks class methodsFor: 'benchmarks' stamp: 'ag 7/3/2014 11:31' prior: 49367383! allBenchmarks ^ { CPBAStarBenchmark. CPBBinaryTreeBenchmark. CPBBlowfishSuite. CPBChameneosBenchmark. CPBDeltaBlueBenchmark. CPBMandelbrotBenchmarkSuite. CPBNBodyBenchmark. "CPBPolymorphyBenchmark." "Commented out because it compiled code in setup." CPBRichardsBenchmark. CPBSplayTreeBenchmark. SimpleMatrixBenchmark. }! ! ----QUIT----{3 July 2014 . 11:32:10 am} Squeak4.5-noBitBlt.image priorSource: 15813551! ----STARTUP----{3 July 2014 . 11:34:49 am} as C:\Dev\lang-smalltalk\images\Squeak4.5-noBitBlt.image! SMarkSuite subclass: #SimpleMatrixBenchmark instanceVariableNames: '' classVariableNames: 'Cols Mults NumOfRuns Rows' poolDictionaries: '' category: 'Matrix-Benchmarks'! !SimpleMatrixBenchmark methodsFor: 'bench' stamp: 'ag 7/3/2014 11:37' prior: 49372902! benchFloats BenchMatrix benchFloats: NumOfRuns multiplicationsPerRun: Mults rows: Rows columns: Cols.! ! !SimpleMatrixBenchmark methodsFor: 'bench' stamp: 'ag 7/3/2014 11:37' prior: 49373080! benchInts BenchMatrix benchInts: NumOfRuns multiplicationsPerRun: Mults rows: Rows columns: Cols.! ! SimpleMatrixBenchmark config: '5 5 5 5'! Benchmarks runMatching: 'SimpleMatrixBenchmark' iterations: 1! !SimpleMatrixBenchmark class methodsFor: 'initialization' stamp: 'ag 7/3/2014 11:38' prior: 49373773! initialize super initialize. NumOfRuns := 10. Mults := 10. Cols := 10. Rows := 10.! ! self initialize! !SimpleMatrixBenchmark class methodsFor: 'initialization' stamp: 'ag 7/3/2014 11:39' prior: 49376651! initialize "self initialize" super initialize. NumOfRuns := 10. Mults := 10. Cols := 10. Rows := 10.! ! ----QUIT----{3 July 2014 . 11:39:08 am} Squeak4.5-noBitBlt.image priorSource: 15821257! ----STARTUP----{3 July 2014 . 11:48:06 am} as C:\Dev\lang-smalltalk\images\Squeak4.5-noBitBlt.image! !BenchMatrix class methodsFor: 'benchmarking' stamp: 'ag 7/3/2014 11:49' prior: 49371447! benchFloats: numOfRuns multiplicationsPerRun: mults rows: r columns: c | generator | generator := Random seed: 23456432. numOfRuns timesRepeat: [ | a b | a := BenchMatrix rows: r columns: c. b := BenchMatrix rows: c columns: r. a fillRandomFloats: generator. b fillRandomFloats: generator. mults timesRepeat: [ a * b ] ].! ! !BenchMatrix class methodsFor: 'benchmarking' stamp: 'ag 7/3/2014 11:49' prior: 49371861! benchInts: numOfRuns multiplicationsPerRun: mults rows: r columns: c | generator | generator := Random seed: 23456432. numOfRuns timesRepeat: [ | a b | a := BenchMatrix rows: r columns: c. b := BenchMatrix rows: c columns: r. a fillRandomInts: generator. b fillRandomInts: generator. mults timesRepeat: [ a * b ] ].! ! !BenchMatrix methodsFor: 'accessing' stamp: 'ag 7/3/2014 11:51' prior: 49368902! fieldsDo: block (1 to: self rows) do: [ :row | (1 to: self columns) do: [ :column | block value: column value: row ] ].! ! 1 benchMatrix: '1 10 100 10'! 1 benchMatrix: '1 10 100 10'! 1 benchMatrix: '1 10 10 100'! 1 benchMatrix: '1 10 10 1000'! ----QUIT----{3 July 2014 . 11:51:44 am} Squeak4.5-noBitBlt.image priorSource: 15822543! ----STARTUP----{3 July 2014 . 12:30:20 pm} as C:\Dev\lang-smalltalk\images\Squeak4.5-noBitBlt.image! Array variableSubclass: #BenchMatrix instanceVariableNames: 'columns rows' classVariableNames: '' poolDictionaries: '' category: 'Matrix-Benchmarks'! Array variableSubclass: #BenchMatrix instanceVariableNames: 'rows' classVariableNames: '' poolDictionaries: '' category: 'Matrix-Benchmarks'! BenchMatrix removeSelector: #at:! BenchMatrix removeSelector: #at:put:! !BenchMatrix methodsFor: 'accessing' stamp: 'ag 7/3/2014 12:31' prior: 49368813! columns ^ self size / rows! ! !BenchMatrix methodsFor: 'accessing' stamp: 'ag 7/3/2014 12:31' prior: 49379008! columns ^ self size / rows! ! 11/2! 11//2! !BenchMatrix methodsFor: 'accessing' stamp: 'ag 7/3/2014 12:33' prior: 49378103! fieldsDo: block (1 to: self size) do: [ :i | block value: i \\ rows value: i // rows ].! ! !BenchMatrix methodsFor: 'accessing' stamp: 'ag 7/3/2014 12:34' prior: 49379251! fieldsDo: block 1 to: self size do: [ :i | block value: i \\ rows value: i // rows ].! ! !BenchMatrix methodsFor: 'accessing' stamp: 'ag 7/3/2014 12:34' prior: 49369340! x: x y: y ^ self at: (self offsetX: x y: y)! ! !BenchMatrix methodsFor: 'accessing' stamp: 'ag 7/3/2014 12:34' prior: 49369457! x: x y: y put: number self at: (self offsetX: x y: y) put: number! ! a := BenchMatrix fields: #( 3 2 1 1 0 2 ) rows: 2! b := BenchMatrix fields: #( 1 2 0 1 4 0 ) rows: 3! a! a! a rows! a columns! BenchMatrix removeSelector: #initializeRows:columns:! !BenchMatrix methodsFor: 'initialization' stamp: 'ag 7/3/2014 12:36'! initializeRows: r rows := r.! ! BenchMatrix removeSelector: #initializeFields:rows:! !BenchMatrix class methodsFor: 'instance creation' stamp: 'ag 7/3/2014 12:37' prior: 49372274! fields: fields rows: r | columns f rows | rows := r. (f size \\ r) = 0 ifFalse: [ self error: 'Illegal initialization.' ]. columns := f size / r. " fields := f." ^ self basicNew initializeFields: fields rows: r! ! !BenchMatrix methodsFor: 'initialization' stamp: 'ag 7/3/2014 12:37'! rows: r rows := r.! ! BenchMatrix removeSelector: #initializeRows:! Array withAll: #(1 2 3)! !BenchMatrix class methodsFor: 'instance creation' stamp: 'ag 7/3/2014 12:39' prior: 49380248! fields: fields rows: r (fields size \\ r) = 0 ifFalse: [ self error: 'Illegal initialization.' ]. ^ (self withAll: fields) rows: r; yourself! ! !BenchMatrix methodsFor: 'accessing' stamp: 'ag 7/3/2014 12:39' prior: 49379122! columns ^ self size // rows! ! !BenchMatrix class methodsFor: 'instance creation' stamp: 'ag 7/3/2014 12:40' prior: 49372433! rows: r columns: c ^ (self new: r * c) rows: r; fillZeros; yourself! ! !BenchMatrix methodsFor: 'initialization' stamp: 'ag 7/3/2014 12:40'! fillZeros self fill: [ :x :y | 0 ].! ! i! i \\ rows! i //rows! rows! !BenchMatrix methodsFor: 'accessing' stamp: 'ag 7/3/2014 12:42' prior: 49379428! fieldsDo: block 1 to: self size do: [ :i | block value: i \\ rows + 1 value: i // rows + 1 ].! ! x := BenchMatrix rows: 4 columns: 3.! x! o := OrderedCollection new.! x fieldsDo: [ :x :y | o add: x -> y ].! o! !BenchMatrix methodsFor: 'accessing' stamp: 'ag 7/3/2014 12:43' prior: 49381404! fieldsDo: block 0 to: self size do: [ :i | block value: i \\ rows + 1 value: i // rows + 1 ].! ! o := OrderedCollection new.! x fieldsDo: [ :x :y | o add: x -> y ].! o! x := BenchMatrix rows: 4 columns: 3.! x! !BenchMatrix methodsFor: 'accessing' stamp: 'ag 7/3/2014 12:44' prior: 49381705! fieldsDo: block 0 to: self size + 1 do: [ :i | block value: i \\ rows + 1 value: i // rows + 1 ].! ! x := BenchMatrix rows: 4 columns: 3.! x! o := OrderedCollection new.! x fieldsDo: [ :x :y | o add: x -> y ].! o! o size! x size! o size! o asSet size! !BenchMatrix methodsFor: 'accessing' stamp: 'ag 7/3/2014 12:46' prior: 49382006! fieldsDo: block 1 to: self size do: [ :i | block value: i \\ rows + 1 value: i // rows + 1 ].! ! o := OrderedCollection new.! x fieldsDo: [ :x :y | o add: x -> y ].! o! o size! o ! 1 \\ 4! !BenchMatrix methodsFor: 'accessing' stamp: 'ag 7/3/2014 12:46' prior: 49382353! fieldsDo: block 1 to: self size do: [ :i | block value: i \\ rows value: i // rows + 1 ].! ! o := OrderedCollection new.! x fieldsDo: [ :x :y | o add: x -> y ].! o size! o ! !BenchMatrix methodsFor: 'accessing' stamp: 'ag 7/3/2014 12:50' prior: 49382634! fieldsDo: block | columns | columns := self columns. 1 to: self size do: [ :i | block value: i \\ columns value: i // columns + 1 ].! ! Array variableSubclass: #BenchMatrix instanceVariableNames: 'rows columns' classVariableNames: '' poolDictionaries: '' category: 'Matrix-Benchmarks'! Array variableSubclass: #BenchMatrix instanceVariableNames: 'rows columns' classVariableNames: '' poolDictionaries: '' category: 'Matrix-Benchmarks'! !BenchMatrix methodsFor: 'accessing' stamp: 'ag 7/3/2014 12:51' prior: 49382898! fieldsDo: block 1 to: self size do: [ :i | block value: i \\ columns value: i // columns + 1 ].! ! !BenchMatrix methodsFor: 'accessing' stamp: 'ag 7/3/2014 12:51' prior: 49380969! columns ^ columns! ! !BenchMatrix methodsFor: 'initialization' stamp: 'ag 7/3/2014 12:51' prior: 49380543! rows: r rows := r. columns := self size // r.! ! x := BenchMatrix rows: 4 columns: 3.! x! o := OrderedCollection new. x fieldsDo: [ :x :y | o add: x -> y ]. ! ox! o! !BenchMatrix methodsFor: 'initialization' stamp: 'ag 7/3/2014 12:52' prior: 49381247! fillZeros self atAllPut: 0.! ! !BenchMatrix methodsFor: 'accessing' stamp: 'ag 7/3/2014 12:53' prior: 49383432! fieldsDo: block 0 to: self size - 1 do: [ :i | block value: i \\ columns + 1 value: i // columns + 1 ].! ! o := OrderedCollection new. x fieldsDo: [ :x :y | o add: x -> y ].! o size! o! a := BenchMatrix fields: #( 3 2 1 1 0 2 ) rows: 2! b := BenchMatrix fields: #( 1 2 0 1 4 0 ) rows: 3! !BenchMatrix methodsFor: 'math' stamp: 'ag 7/3/2014 12:55' prior: 49370092! * other | result | (self columns = other rows and: [ self rows = other columns ]) ifFalse: [ ^ self error: 'Cannot multiply, wrong dimensions.' ]. result := BenchMatrix rows: self rows columns: other columns. (1 to: self rows) do: [ :row | (1 to: other columns) do: [ :column | | value | value := 0. (1 to: self columns) do: [ :i | value := value + ((self x: i y: row) * (other x: column y: i)) ]. result x: column y: row put: value ] ]. ^ result! ! a * b! self assert: (Array withAll: (a * b)) = #(7 8 9 2)! BenchMatrix class organization addCategory: #test! !BenchMatrix class methodsFor: 'test' stamp: 'ag 7/3/2014 12:57'! tinyTest "self tinyTest" | a b | a := BenchMatrix fields: #( 3 2 1 1 0 2 ) rows: 2. b := BenchMatrix fields: #( 1 2 0 1 4 0 ) rows: 3. self assert: (Array withAll: (a * b)) = #(7 8 9 2).! ! self tinyTest! 1 benchMatrix: '1 3 5 5'! 1 benchMatrix: '1 10 5 5'! ----QUIT----{3 July 2014 . 12:58:52 pm} Squeak4.5-noBitBlt.image priorSource: 15823926! ----STARTUP----{3 July 2014 . 1:05:04 pm} as C:\Dev\lang-smalltalk\images\Squeak4.5-noBitBlt.image! a := BenchMatrix rows: 20 columns: 20. b := BenchMatrix rows: 20 columns: 20. generator := Random seed: 13243456. a fillRandomInts: generator. b fillRandomInts: generator.! (a collect: #class) asSet! (b collect: #class) asSet! a := BenchMatrix rows: 20 columns: 20. b := BenchMatrix rows: 20 columns: 20. generator := Random seed: 13243456. a fillRandomFloats: generator. b fillRandomInts: generator.! (b collect: #class) asSet! (a collect: #class) asSet! !BenchMatrix methodsFor: 'benchmarking' stamp: 'ag 7/3/2014 13:08' prior: 49371100! fillRandomInts: generator "Fill with SmallInteger values small enough to stay SmallIntegers after multiplication." | max | max := SmallInteger maxVal sqrt asInteger. self fill: [ :x :y | max atRandom: generator ]. ! ! !BenchMatrix methodsFor: 'benchmarking' stamp: 'ag 7/3/2014 13:08' prior: 49370897! fillRandomFloats: generator self fill: [ :x :y | generator next * 100 ].! ! a := BenchMatrix rows: 20 columns: 20. b := BenchMatrix rows: 20 columns: 20. generator := Random seed: 13243456. a fillRandomFloats: generator. b fillRandomInts: generator. (a collect: #class) asSet ! a := BenchMatrix rows: 20 columns: 20. b := BenchMatrix rows: 20 columns: 20. generator := Random seed: 13243456. a fillRandomFloats: generator. b fillRandomInts: generator. (b collect: #class) asSet ! a := BenchMatrix rows: 20 columns: 20. b := BenchMatrix rows: 20 columns: 20. generator := Random seed: 13243456. a fillRandomInts: generator. b fillRandomInts: generator. ! c := a * b! (c collect: #class) asSet! !BenchMatrix methodsFor: 'benchmarking' stamp: 'ag 7/3/2014 13:09' prior: 49386143! fillRandomInts: generator "Fill with SmallInteger values small enough to stay SmallIntegers after multiplication." | max | max := 1000. self fill: [ :x :y | max atRandom: generator ]. ! ! a := BenchMatrix rows: 20 columns: 20. b := BenchMatrix rows: 20 columns: 20. generator := Random seed: 13243456. a fillRandomInts: generator. b fillRandomInts: generator. c := a * b. (c collect: #class) asSet! ----QUIT----{3 July 2014 . 1:09:37 pm} Squeak4.5-noBitBlt.image priorSource: 15830973! ----STARTUP----{3 July 2014 . 8:26:43 pm} as C:\Dev\lang-smalltalk\images\Squeak4.5-noBitBlt.image! a := BenchMatrix rows: 20 columns: 20. b := BenchMatrix rows: 20 columns: 20. generator := Random seed: 13243456. a fillRandomInts: generator. b fillRandomInts: generator.! c := a * b.! (c collect: #class) asSet! c! !SmallInteger methodsFor: 'as yet unclassified' stamp: 'ag 7/3/2014 20:27'! testMatrix ! ! !SmallInteger methodsFor: 'as yet unclassified' stamp: 'ag 7/3/2014 20:28' prior: 49388134! testMatrix | a b c generator | a := BenchMatrix rows: 20 columns: 20. b := BenchMatrix rows: 20 columns: 20. generator := Random seed: 13243456. a fillRandomInts: generator. b fillRandomInts: generator. c := a * b. ^ (c collect: #class) asSet asString! ! 5 testMatrix! ----SNAPSHOT----{3 July 2014 . 8:28:40 pm} Squeak4.5-noBitBlt.1.image priorSource: 15833215! ----QUIT----{3 July 2014 . 8:28:49 pm} Squeak4.5-noBitBlt.1.image priorSource: 15834093! ----STARTUP----{3 July 2014 . 9:02:43 pm} as C:\Dev\lang-smalltalk\images\Squeak4.5-noBitBlt.image! !BenchMatrix methodsFor: 'initialization' stamp: 'ag 7/3/2014 21:03' prior: 49383727! rows: r rows := r asFloat. columns := (self size // r) asFloat.! ! !BenchMatrix methodsFor: 'accessing' stamp: 'ag 7/3/2014 21:03' prior: 49369257! rows ^ rows asInteger! ! !BenchMatrix methodsFor: 'accessing' stamp: 'ag 7/3/2014 21:03' prior: 49383617! columns ^ columns asInteger! ! !BenchMatrix methodsFor: 'accessing' stamp: 'ag 7/3/2014 21:04' prior: 49384103! fieldsDo: block 0 to: self size - 1 do: [ :i | block value: i \\ self columns + 1 value: i // self columns + 1 ].! ! ----QUIT----{3 July 2014 . 9:04:33 pm} Squeak4.5-noBitBlt.image priorSource: 15834187! ----STARTUP----{10 July 2014 . 3:46:58 pm} as C:\Dev\lang-smalltalk\images\Squeak4.5-noBitBlt.image! ----SNAPSHOT----{10 July 2014 . 3:47:56 pm} Squeak4.5-noBitBlt.image priorSource: 15834963! 'From Squeak4.4 of 28 April 2013 [latest update: #12627] on 10 July 2014 at 3:45:28 pm'! Object subclass: #BalloonEngine instanceVariableNames: 'workBuffer span bitBlt forms clipRect destOffset externals aaLevel edgeTransform colorTransform deferred postFlushNeeded' classVariableNames: 'BezierStats BufferCache CacheProtect Counts Debug Times' poolDictionaries: 'BalloonEngineConstants' category: 'Balloon-Engine'! Object subclass: #BalloonEngine instanceVariableNames: 'workBuffer span bitBlt forms clipRect destOffset externals aaLevel edgeTransform colorTransform deferred postFlushNeeded' classVariableNames: 'BezierStats BufferCache CacheProtect Counts Debug Times' poolDictionaries: 'BalloonEngineConstants' category: 'Balloon-Engine'! !BalloonEngine commentStamp: '' prior: 17214650! BalloonEngine is the representative for the Balloon engine inside Squeak. For most purposes it should not be used directly but via BalloonCanvas since this ensures proper initialization and is polymorphic with other canvas uses.! !BalloonEngine methodsFor: 'accessing' stamp: 'ar 11/11/1998 23:04' prior: 17214951! aaLevel ^aaLevel ifNil:[1]! ! !BalloonEngine methodsFor: 'accessing' stamp: 'ar 11/11/1998 23:04' prior: 17215052! aaLevel: anInteger aaLevel := (anInteger min: 4) max: 1.! ! !BalloonEngine methodsFor: 'accessing' stamp: 'ar 10/29/1998 01:51' prior: 17215183! aaTransform "Return a transformation for the current anti-aliasing level" | matrix | matrix := MatrixTransform2x3 withScale: (self aaLevel) asFloat asPoint. matrix offset: (self aaLevel // 2) asFloat asPoint. ^matrix composedWith:(MatrixTransform2x3 withOffset: destOffset asFloatPoint)! ! !BalloonEngine methodsFor: 'accessing' stamp: 'ar 10/13/1998 03:04' prior: 17215548! bitBlt ^bitBlt! ! !BalloonEngine methodsFor: 'accessing' stamp: 'ar 5/28/2000 15:02' prior: 17215636! bitBlt: aBitBlt bitBlt := aBitBlt. bitBlt isNil ifTrue:[^self]. self class primitiveSetBitBltPlugin: bitBlt getPluginName. self clipRect: bitBlt clipRect. bitBlt sourceForm: (Form extent: span size @ 1 depth: 32 bits: span); sourceRect: (0 at 0 extent: 1 at span size); colorMap: (Color colorMapIfNeededFrom: 32 to: bitBlt destForm depth); combinationRule: (bitBlt destForm depth >= 8 ifTrue:[34] ifFalse:[Form paint]).! ! !BalloonEngine methodsFor: 'accessing' stamp: 'ar 11/1/1998 02:57' prior: 17216136! clipRect ^clipRect! ! !BalloonEngine methodsFor: 'accessing' stamp: 'ar 10/13/1998 02:44' prior: 17216229! clipRect: aRect clipRect := aRect truncated! ! !BalloonEngine methodsFor: 'accessing' stamp: 'ar 11/24/1998 15:04' prior: 17216347! colorTransform ^colorTransform! ! !BalloonEngine methodsFor: 'accessing' stamp: 'ar 11/24/1998 15:04' prior: 17216452! colorTransform: aColorTransform colorTransform := aColorTransform! ! !BalloonEngine methodsFor: 'accessing' stamp: 'ar 12/30/1998 11:24' prior: 17216592! deferred ^deferred! ! !BalloonEngine methodsFor: 'accessing' stamp: 'ar 12/30/1998 11:24' prior: 17216685! deferred: aBoolean deferred := aBoolean.! ! !BalloonEngine methodsFor: 'accessing' stamp: 'ar 11/1/1998 02:56' prior: 17216799! destOffset ^destOffset! ! !BalloonEngine methodsFor: 'accessing' stamp: 'ar 11/12/1998 00:22' prior: 17216896! destOffset: aPoint destOffset := aPoint asIntegerPoint. bitBlt destX: aPoint x; destY: aPoint y.! ! !BalloonEngine methodsFor: 'accessing' stamp: 'ar 11/25/1998 22:34' prior: 17217068! edgeTransform ^edgeTransform! ! !BalloonEngine methodsFor: 'accessing' stamp: 'ar 11/25/1998 22:34' prior: 17217171! edgeTransform: aTransform edgeTransform := aTransform.! ! !BalloonEngine methodsFor: 'accessing' stamp: 'ar 10/29/1998 01:51' prior: 17217300! fullTransformFrom: aMatrix | m | m := self aaTransform composedWith: aMatrix. "m offset: m offset + destOffset." ^m! ! !BalloonEngine methodsFor: 'copying' stamp: 'ar 11/25/1998 00:45' prior: 17217492! canProceedAfter: failureReason "Check if we can proceed after the failureReason indicated." | newBuffer | failureReason = GErrorNeedFlush ifTrue:[ "Need to flush engine before proceeding" self copyBits. self reset. ^true]. failureReason = GErrorNoMoreSpace ifTrue:[ "Work buffer is too small" newBuffer := workBuffer species new: workBuffer size * 2. self primCopyBufferFrom: workBuffer to: newBuffer. workBuffer := newBuffer. ^true]. "Not handled" ^false! ! !BalloonEngine methodsFor: 'copying' stamp: 'ar 3/6/2001 12:06' prior: 17218041! copyBits (bitBlt notNil and:[bitBlt destForm notNil]) ifTrue:[bitBlt destForm unhibernate]. self copyLoopFaster.! ! !BalloonEngine methodsFor: 'copying' stamp: 'ar 11/14/1998 19:32' prior: 17218227! copyLoop "This is the basic rendering loop using as little primitive support as possible." | finished edge fill | edge := BalloonEdgeData new. fill := BalloonFillData new. self primInitializeProcessing. "Initialize the GE for processing" [self primFinishedProcessing] whileFalse:[ "Step 1: Process the edges in the global edge table that will be added in this step" [finished := self primNextGlobalEdgeEntryInto: edge. finished] whileFalse:[ edge source: (externals at: edge index). edge stepToFirstScanLine. self primAddActiveEdgeTableEntryFrom: edge]. "Step 2: Scan the active edge table" [finished := self primNextFillEntryInto: fill. finished] whileFalse:[ fill source: (externals at: fill index). "Compute the new fill" fill computeFill. "And mix it in the out buffer" self primMergeFill: fill destForm bits from: fill]. "Step 3: Display the current span buffer if necessary" self primDisplaySpanBuffer. "Step 4: Advance and resort the active edge table" [finished := self primNextActiveEdgeEntryInto: edge. finished] whileFalse:[ "If the index is zero then the edge has been handled by the GE" edge source: (externals at: edge index). edge stepToNextScanLine. self primChangeActiveEdgeTableEntryFrom: edge]. ]. self primGetTimes: Times. self primGetCounts: Counts. self primGetBezierStats: BezierStats.! ! !BalloonEngine methodsFor: 'copying' stamp: 'ar 11/14/1998 19:32' prior: 17219678! copyLoopFaster "This is a copy loop drawing one scan line at a time" | edge fill reason | edge := BalloonEdgeData new. fill := BalloonFillData new. [self primFinishedProcessing] whileFalse:[ reason := self primRenderScanline: edge with: fill. "reason ~= 0 means there has been a problem" reason = 0 ifFalse:[ self processStopReason: reason edge: edge fill: fill. ]. ]. self primGetTimes: Times. self primGetCounts: Counts. self primGetBezierStats: BezierStats.! ! !BalloonEngine methodsFor: 'copying' stamp: 'ar 11/14/1998 19:33' prior: 17220230! copyLoopFastest "This is a copy loop drawing the entire image" | edge fill reason | edge := BalloonEdgeData new. fill := BalloonFillData new. [self primFinishedProcessing] whileFalse:[ reason := self primRenderImage: edge with: fill. "reason ~= 0 means there has been a problem" reason = 0 ifFalse:[ self processStopReason: reason edge: edge fill: fill. ]. ]. self primGetTimes: Times. self primGetCounts: Counts. self primGetBezierStats: BezierStats.! ! !BalloonEngine methodsFor: 'copying' stamp: 'ar 11/11/1998 21:19' prior: 17220773! processStopReason: reason edge: edge fill: fill "The engine has stopped because of some reason. Try to figure out how to respond and do the necessary actions." "Note: The order of operations below can affect the speed" "Process unknown fills first" reason = GErrorFillEntry ifTrue:[ fill source: (externals at: fill index). "Compute the new fill" fill computeFill. "And mix it in the out buffer" ^self primMergeFill: fill destForm bits from: fill]. "Process unknown steppings in the AET second" reason = GErrorAETEntry ifTrue:[ edge source: (externals at: edge index). edge stepToNextScanLine. ^self primChangeActiveEdgeTableEntryFrom: edge]. "Process unknown entries in the GET third" reason = GErrorGETEntry ifTrue:[ edge source: (externals at: edge index). edge stepToFirstScanLine. ^self primAddActiveEdgeTableEntryFrom: edge]. "Process generic problems last" (self canProceedAfter: reason) ifTrue:[^self]. "Okay." ^self error:'Unkown stop reason in graphics engine' ! ! !BalloonEngine methodsFor: 'drawing' stamp: 'ar 10/11/1999 16:49' prior: 17221856! drawBezierShape: points fill: fillStyle borderWidth: borderWidth borderColor: borderFill transform: aTransform | fills | self edgeTransform: aTransform. self resetIfNeeded. fills := self registerFill: fillStyle and: borderFill. self primAddBezierShape: points segments: (points size) // 3 fill: (fills at: 1) lineWidth: borderWidth lineFill: (fills at: 2). self postFlushIfNeeded.! ! !BalloonEngine methodsFor: 'drawing' stamp: 'ar 11/26/1998 19:44' prior: 17222322! drawCompressedShape: shape transform: aTransform | fillIndexList | self edgeTransform: aTransform. self resetIfNeeded. fillIndexList := self registerFills: shape fillStyles. self primAddCompressedShape: shape points segments: shape numSegments leftFills: shape leftFills rightFills: shape rightFills lineWidths: shape lineWidths lineFills: shape lineFills fillIndexList: fillIndexList. self postFlushIfNeeded.! ! !BalloonEngine methodsFor: 'drawing' stamp: 'ar 1/15/1999 03:02' prior: 17222822! drawGeneralBezierShape: contours fill: fillStyle borderWidth: borderWidth borderColor: borderFill transform: aTransform | fills | self edgeTransform: aTransform. self resetIfNeeded. fills := self registerFill: fillStyle and: borderFill. contours do:[:points| self primAddBezierShape: points segments: (points size // 3) fill: (fills at: 1) lineWidth: borderWidth lineFill: (fills at: 2). "Note: To avoid premature flushing of the pipeline we need to reset the flush bit within the engine." self primFlushNeeded: false. ]. "And set the flush bit afterwards" self primFlushNeeded: true. self postFlushIfNeeded.! ! !BalloonEngine methodsFor: 'drawing' stamp: 'ar 1/15/1999 03:02' prior: 17223531! drawGeneralPolygon: contours fill: fillStyle borderWidth: borderWidth borderColor: borderFill transform: aTransform | fills | self edgeTransform: aTransform. self resetIfNeeded. fills := self registerFill: fillStyle and: borderFill. contours do:[:points| self primAddPolygon: points segments: points size fill: (fills at: 1) lineWidth: borderWidth lineFill: (fills at: 2). "Note: To avoid premature flushing of the pipeline we need to reset the flush bit within the engine." self primFlushNeeded: false. ]. "And set the flush bit afterwards" self primFlushNeeded: true. self postFlushIfNeeded.! ! !BalloonEngine methodsFor: 'drawing' stamp: 'ar 11/26/1998 19:45' prior: 17224226! drawOval: rect fill: fillStyle borderWidth: borderWidth borderColor: borderColor transform: aMatrix | fills | self edgeTransform: aMatrix. self resetIfNeeded. fills := self registerFill: fillStyle and: borderColor. self primAddOvalFrom: rect origin to: rect corner fillIndex: (fills at: 1) borderWidth: borderWidth borderColor: (fills at: 2). self postFlushIfNeeded.! ! !BalloonEngine methodsFor: 'drawing' stamp: 'ar 11/26/1998 19:45' prior: 17224684! drawPolygon: points fill: fillStyle borderWidth: borderWidth borderColor: borderFill transform: aTransform | fills | self edgeTransform: aTransform. self resetIfNeeded. fills := self registerFill: fillStyle and: borderFill. self primAddPolygon: points segments: points size fill: (fills at: 1) lineWidth: borderWidth lineFill: (fills at: 2). self postFlushIfNeeded.! ! !BalloonEngine methodsFor: 'drawing' stamp: 'ar 11/26/1998 19:45' prior: 17225136! drawRectangle: rect fill: fillStyle borderWidth: borderWidth borderColor: borderColor transform: aMatrix | fills | self edgeTransform: aMatrix. self resetIfNeeded. fills := self registerFill: fillStyle and: borderColor. self primAddRectFrom: rect origin to: rect corner fillIndex: (fills at: 1) borderWidth: borderWidth borderColor: (fills at: 2). self postFlushIfNeeded.! ! !BalloonEngine methodsFor: 'drawing' stamp: 'bf 4/3/2004 01:36' prior: 17225597! registerFill: aFillStyle "Register the given fill style." | theForm | aFillStyle ifNil:[^0]. aFillStyle isSolidFill ifTrue:[^aFillStyle scaledPixelValue32]. aFillStyle isGradientFill ifTrue:[ ^self primAddGradientFill: aFillStyle pixelRamp from: aFillStyle origin along: aFillStyle direction normal: aFillStyle normal radial: aFillStyle isRadialFill ]. aFillStyle isBitmapFill ifTrue:[ theForm := aFillStyle form asSourceForm. theForm unhibernate. forms := forms copyWith: theForm. ^self primAddBitmapFill: theForm colormap: (theForm colormapIfNeededForDepth: 32) tile: aFillStyle isTiled from: aFillStyle origin along: aFillStyle direction normal: aFillStyle normal xIndex: forms size]. ^0! ! !BalloonEngine methodsFor: 'drawing' stamp: 'ar 11/26/1998 19:45' prior: 17226418! registerFill: fill1 and: fill2 ^self registerFills: (Array with: fill1 with: fill2)! ! !BalloonEngine methodsFor: 'drawing' stamp: 'ar 1/14/1999 15:24' prior: 17226573! registerFill: aFillStyle transform: aTransform aFillStyle ifNil:[^0]. aFillStyle isSolidFill ifTrue:[^aFillStyle scaledPixelValue32]. aFillStyle isGradientFill ifTrue:[ ^self primAddGradientFill: aFillStyle pixelRamp from: aFillStyle origin along: aFillStyle direction normal: aFillStyle normal radial: aFillStyle isRadialFill matrix: aTransform. ]. ^0! ! !BalloonEngine methodsFor: 'drawing' stamp: 'di 11/21/1999 20:15' prior: 17227025! registerFills: fills | fillIndexList index fillIndex | ((colorTransform notNil and:[colorTransform isAlphaTransform]) or:[ fills anySatisfy: [:any| any notNil and:[any isTranslucent]]]) ifTrue:[ self flush. self reset. postFlushNeeded := true]. fillIndexList := WordArray new: fills size. index := 1. [index <= fills size] whileTrue:[ fillIndex := self registerFill: (fills at: index). fillIndex == nil ifTrue:[index := 1] "Need to start over" ifFalse:[fillIndexList at: index put: fillIndex. index := index+1] ]. ^fillIndexList! ! !BalloonEngine methodsFor: 'experimental' stamp: 'ar 11/12/1998 19:53' prior: 17227670! registerBezier: aCurve transformation: aMatrix self primAddBezierFrom: aCurve start to: aCurve end via: aCurve via leftFillIndex: (self registerFill: aCurve leftFill transform: aMatrix) rightFillIndex: (self registerFill: aCurve rightFill transform: aMatrix) matrix: aMatrix! ! !BalloonEngine methodsFor: 'experimental' stamp: 'ar 11/11/1998 21:15' prior: 17228032! registerBoundary: boundaryObject transformation: aMatrix | external | external := boundaryObject asEdgeRepresentation: (self fullTransformFrom: aMatrix). self subdivideExternalEdge: external from: boundaryObject. ! ! !BalloonEngine methodsFor: 'experimental' stamp: 'ar 11/12/1998 19:54' prior: 17228325! registerExternalEdge: externalEdge from: boundaryObject externals addLast: externalEdge. self primAddExternalEdge: externals size initialX: externalEdge initialX initialY: externalEdge initialY initialZ: externalEdge initialZ leftFillIndex: (self registerFill: boundaryObject leftFill transform: nil) rightFillIndex: (self registerFill: boundaryObject rightFill transform: nil)! ! !BalloonEngine methodsFor: 'experimental' stamp: 'ar 11/12/1998 19:54' prior: 17228791! registerLine: aLine transformation: aMatrix self primAddLineFrom: aLine start to: aLine end leftFillIndex: (self registerFill: aLine leftFill transform: aMatrix) rightFillIndex: (self registerFill: aLine rightFill transform: aMatrix) matrix: aMatrix! ! !BalloonEngine methodsFor: 'experimental' stamp: 'ar 11/11/1998 21:15' prior: 17229124! subdivideExternalEdge: external from: boundaryObject | external2 | external2 := external subdivide. external2 notNil ifTrue:[ self subdivideExternalEdge: external from: boundaryObject. self subdivideExternalEdge: external2 from: boundaryObject. ] ifFalse:[ self registerExternalEdge: external from: boundaryObject. ].! ! !BalloonEngine methodsFor: 'initialize' stamp: 'ar 11/25/1998 22:29' prior: 17229528! flush "Force all pending primitives onscreen" workBuffer ifNil:[^self]. self copyBits. self release.! ! !BalloonEngine methodsFor: 'initialize' stamp: 'nk 9/26/2003 10:52' prior: 17229706! initialize | w | w := Display width > 2048 ifTrue: [ 4096 ] ifFalse: [ 2048 ]. externals := OrderedCollection new: 100. span := Bitmap new: w. bitBlt := nil. self bitBlt: ((BitBlt toForm: Display) destRect: Display boundingBox; yourself). forms := #(). deferred := false.! ! !BalloonEngine methodsFor: 'initialize' stamp: 'ar 11/25/1998 22:42' prior: 17230060! postFlushIfNeeded From noreply at buildbot.pypy.org Fri Jul 18 14:08:23 2014 From: noreply at buildbot.pypy.org (anton_gulenko) Date: Fri, 18 Jul 2014 14:08:23 +0200 (CEST) Subject: [pypy-commit] lang-smalltalk storage: Fixed error-reporting and other outputs after parsing arguments. Message-ID: <20140718120823.8477F1C01D2@cobra.cs.uni-duesseldorf.de> Author: Anton Gulenko Branch: storage Changeset: r898:3146c682cb30 Date: 2014-07-10 16:39 +0200 http://bitbucket.org/pypy/lang-smalltalk/changeset/3146c682cb30/ Log: Fixed error-reporting and other outputs after parsing arguments. Added flag to disable specialized storage strategies. diff --git a/spyvm/display.py b/spyvm/display.py --- a/spyvm/display.py +++ b/spyvm/display.py @@ -229,7 +229,7 @@ return elif c_type == RSDL.QUIT: from spyvm.error import Exit - raise Exit("Window closed..") + raise Exit("Window closed") finally: lltype.free(event, flavor='raw') diff --git a/spyvm/objspace.py b/spyvm/objspace.py --- a/spyvm/objspace.py +++ b/spyvm/objspace.py @@ -8,6 +8,10 @@ class ObjSpace(object): def __init__(self): + # If this is True, then no optimizing storage strategies will be used. + # Intended for performance comparisons. Breaks tests. + self.no_specialized_storage = [False] + self.classtable = {} self.objtable = {} self._executable_path = [""] # XXX: we cannot set the attribute diff --git a/spyvm/primitives.py b/spyvm/primitives.py --- a/spyvm/primitives.py +++ b/spyvm/primitives.py @@ -374,7 +374,6 @@ @expose_primitive(FAIL) def func(interp, s_frame, argcount): - from spyvm.error import Exit if s_frame.w_method().lookup_selector == 'doesNotUnderstand:': print '' print s_frame.print_stack() @@ -837,7 +836,7 @@ @expose_primitive(QUIT, unwrap_spec=[object]) def func(interp, s_frame, w_rcvr): from spyvm.error import Exit - raise Exit('Quit-Primitive called..') + raise Exit('Quit-Primitive called') @expose_primitive(EXIT_TO_DEBUGGER, unwrap_spec=[object]) def func(interp, s_frame, w_rcvr): diff --git a/spyvm/shadow.py b/spyvm/shadow.py --- a/spyvm/shadow.py +++ b/spyvm/shadow.py @@ -8,10 +8,6 @@ from rpython.rlib.rstruct.runpack import runpack from rpython.rtyper.lltypesystem import rffi, lltype -# If this is True, then no optimizing storage strategies will be used. -# Intended for performance comparisons. Breaks tests. -no_specialized_storage = False - class AbstractShadow(object): """A shadow is an optional extra bit of information that can be attached at run-time to any Smalltalk object. @@ -170,14 +166,14 @@ def empty_storage(space, w_self, size, weak=False): if weak: return WeakListStorageShadow(space, w_self, size) - if no_specialized_storage: + if space.no_specialized_storage[0]: return ListStorageShadow(space, w_self, size) return AllNilStorageShadow(space, w_self, size) def find_storage_for_objects(space, vars, weak=False): if weak: return WeakListStorageShadow - if no_specialized_storage: + if space.no_specialized_storage[0]: return ListStorageShadow specialized_strategies = 3 all_nil_can_handle = True diff --git a/targetimageloadingsmalltalk.py b/targetimageloadingsmalltalk.py --- a/targetimageloadingsmalltalk.py +++ b/targetimageloadingsmalltalk.py @@ -38,6 +38,7 @@ -s - After num stack frames, the entire stack will be dumped to the heap. This breaks performance, but protects agains stack overflow. num <= 0 disables stack protection (default: %d) + -S - Disable specialized storage strategies; always use generic ListStorage Logging parameters: -t|--trace - Output a trace of each message, primitive, return value and process switch. @@ -49,13 +50,25 @@ def get_parameter(argv, idx, arg): if len(argv) < idx + 1: - raise RuntimeError("Error: missing argument after %s" % arg) + raise error.Exit("Missing argument after %s" % arg) return argv[idx], idx + 1 +def get_int_parameter(argv, idx, arg): + param, idx = get_parameter(argv, idx, arg) + try: + result = int(param) + except ValueError, e: + raise error.Exit("Non-int argument after %s" % arg) + return result, idx + +def print_error(str): + os.write(2, str + os.linesep) + prebuilt_space = objspace.ObjSpace() def entry_point(argv): # == Main execution parameters + path = None selector = None code = "" number = 0 @@ -68,54 +81,61 @@ max_stack_depth = constants.MAX_LOOP_DEPTH trace = False - path = argv[1] if len(argv) > 1 else "Squeak.image" - idx = 2 - - while idx < len(argv): - arg = argv[idx] - idx += 1 - if arg in ["-h", "--help"]: - _usage(argv) - return 0 - elif arg in ["-j", "--jit"]: - jitarg, idx = get_parameter(argv, idx, arg) - jit.set_user_param(interpreter.Interpreter.jit_driver, jitarg) - elif arg in ["-n", "--number"]: - numarg, idx = get_parameter(argv, idx, arg) - number = int(numarg) - have_number = True - elif arg in ["-m", "--method"]: - selector, idx = get_parameter(argv, idx, arg) - elif arg in ["-t", "--trace"]: - trace = True - elif arg in ["-p", "--poll"]: - poll = True - elif arg in ["-a", "--arg"]: - stringarg, idx = get_parameter(argv, idx, arg) - elif arg in ["-r", "--run"]: - code, idx = get_parameter(argv, idx, arg) - elif arg in ["-i", "--no-interrupts"]: - interrupts = False - elif arg in ["-s"]: - arg, idx = get_parameter(argv, idx, arg) - max_stack_depth = int(arg) - elif arg in ["-P", "--process"]: - headless = False - elif arg in ["-u"]: - from spyvm.plugins.vmdebugging import stop_ui_process - stop_ui_process() - elif arg in ["-l", "--storage-log"]: - storage_logger.activate() - elif arg in ["-L", "--storage-log-aggregate"]: - storage_logger.activate(aggregate=True) - elif arg in ["-E", "--storage-log-elements"]: - storage_logger.activate(elements=True) - else: - _usage(argv) - return -1 - - if code and selector: - raise RuntimeError("Cannot handle both -r and -m.") + space = prebuilt_space + idx = 1 + try: + while idx < len(argv): + arg = argv[idx] + idx += 1 + if arg in ["-h", "--help"]: + _usage(argv) + return 0 + elif arg in ["-j", "--jit"]: + jitarg, idx = get_parameter(argv, idx, arg) + jit.set_user_param(interpreter.Interpreter.jit_driver, jitarg) + elif arg in ["-n", "--number"]: + number, idx = get_int_parameter(argv, idx, arg) + have_number = True + elif arg in ["-m", "--method"]: + selector, idx = get_parameter(argv, idx, arg) + elif arg in ["-t", "--trace"]: + trace = True + elif arg in ["-p", "--poll"]: + poll = True + elif arg in ["-a", "--arg"]: + stringarg, idx = get_parameter(argv, idx, arg) + elif arg in ["-r", "--run"]: + code, idx = get_parameter(argv, idx, arg) + elif arg in ["-i", "--no-interrupts"]: + interrupts = False + elif arg in ["-s"]: + max_stack_depth, idx = get_int_parameter(argv, idx, arg) + elif arg in ["-P", "--process"]: + headless = False + elif arg in ["-S"]: + space.no_specialized_storage[0] = True + elif arg in ["-u"]: + from spyvm.plugins.vmdebugging import stop_ui_process + stop_ui_process() + elif arg in ["-l", "--storage-log"]: + storage_logger.activate() + elif arg in ["-L", "--storage-log-aggregate"]: + storage_logger.activate(aggregate=True) + elif arg in ["-E", "--storage-log-elements"]: + storage_logger.activate(elements=True) + elif path is None: + path = arg + else: + _usage(argv) + return -1 + + if path is None: + path = "Squeak.image" + if code and selector: + raise error.Exit("Cannot handle both -r and -m.") + except error.Exit as e: + print_error("Parameter error: %s" % e.msg) + return 1 path = rpath.rabspath(path) try: @@ -125,17 +145,17 @@ finally: f.close() except OSError as e: - os.write(2, "%s -- %s (LoadError)\n" % (os.strerror(e.errno), path)) + print_error("%s -- %s (LoadError)" % (os.strerror(e.errno), path)) return 1 # Load & prepare image and environment - space = prebuilt_space image_reader = squeakimage.reader_for_image(space, squeakimage.Stream(data=imagedata)) image = create_image(space, image_reader) interp = interpreter.Interpreter(space, image, image_name=path, trace=trace, evented=not poll, interrupts=interrupts, max_stack_depth=max_stack_depth) space.runtime_setup(argv[0]) + print_error("") # Line break after image-loading characters # Create context to be executed if code or selector: @@ -172,30 +192,31 @@ selector = "DoIt%d" % int(time.time()) space = interp.space w_receiver_class = w_receiver.getclass(space) + + # The suppress_process_switch flag is a hack/workaround to enable compiling code + # before having initialized the image cleanly. The problem is that the TimingSemaphore is not yet + # registered (primitive 136 not called), so the idle process will never be left once it is entered. + # TODO - Find a way to cleanly initialize the image, without executing the active_context of the image. + # Instead, we want to execute our own context. Then remove this flag (and all references to it) + interp.space.suppress_process_switch[0] = True try: - try: - # The suppress_process_switch flag is a hack/workaround to enable compiling code - # before having initialized the image cleanly. The problem is that the TimingSemaphore is not yet - # registered (primitive 136 not called), so the idle process will never be left once it is entered. - # TODO - Find a way to cleanly initialize the image, without executing the active_context of the image. - # Instead, we want to execute our own context. Then remove this flag (and all references to it) - interp.space.suppress_process_switch[0] = True - w_result = interp.perform( - w_receiver_class, - "compile:classified:notifying:", - w_arguments = [space.wrap_string("%s\r\n%s" % (selector, code)), - space.wrap_string("spy-run-code"), - space.w_nil] - ) - finally: - interp.space.suppress_process_switch[0] = False + w_result = interp.perform( + w_receiver_class, + "compile:classified:notifying:", + w_arguments = [space.wrap_string("%s\r\n%s" % (selector, code)), + space.wrap_string("spy-run-code"), + space.w_nil] + ) + # TODO - is this expected in every image? if not isinstance(w_result, model.W_BytesObject) or w_result.as_string() != selector: - print "Compilation failed, unexpected result: %s" % result_string(w_result) + print_error("Compilation failed, unexpected result: %s" % result_string(w_result)) return None except error.Exit, e: - print "Exited while compiling code: %s" % e.msg + print_error("Exited while compiling code: %s" % e.msg) return None + finally: + interp.space.suppress_process_switch[0] = False w_receiver_class.as_class_get_shadow(space).s_methoddict().sync_method_cache() return selector @@ -234,11 +255,10 @@ return w_active_context.as_context_get_shadow(space) def execute_context(interp, s_frame, measure=False): - print "" # Line break after image-loading-indicator characters try: return interp.interpret_toplevel(s_frame.w_self()) except error.Exit, e: - print "Exited: %s" % e.msg + print_error("Exited: %s" % e.msg) return None # _____ Target and Main _____ From noreply at buildbot.pypy.org Fri Jul 18 14:08:24 2014 From: noreply at buildbot.pypy.org (anton_gulenko) Date: Fri, 18 Jul 2014 14:08:24 +0200 (CEST) Subject: [pypy-commit] lang-smalltalk storage: Optimizing storage strategies. Message-ID: <20140718120824.B9F771C01D2@cobra.cs.uni-duesseldorf.de> Author: Anton Gulenko Branch: storage Changeset: r899:2c3b6f965b1a Date: 2014-07-10 21:29 +0200 http://bitbucket.org/pypy/lang-smalltalk/changeset/2c3b6f965b1a/ Log: Optimizing storage strategies. diff --git a/spyvm/model.py b/spyvm/model.py --- a/spyvm/model.py +++ b/spyvm/model.py @@ -614,7 +614,7 @@ def switch_shadow(self, new_shadow, w_element=None): old_shadow = self.assert_shadow() - new_shadow.copy_from(old_shadow) + old_shadow.copy_into(new_shadow) self.store_shadow(new_shadow) new_shadow.attach_shadow() self.log_storage("Switched", old_shadow, w_element=w_element) diff --git a/spyvm/shadow.py b/spyvm/shadow.py --- a/spyvm/shadow.py +++ b/spyvm/shadow.py @@ -38,16 +38,27 @@ def size(self): raise NotImplementedError("Abstract class") + # This will invoke an appropriate copy_from_* method. + # Overwriting this allows optimized transitions between certain storage types. + def copy_into(self, other_shadow): + other_shadow.copy_from(self) + def attach_shadow(self): pass def copy_field_from(self, n0, other_shadow): self.store(n0, other_shadow.fetch(n0)) - # This can be overwritten to change the order of initialization. def copy_from(self, other_shadow): assert self.size() == other_shadow.size() for i in range(self.size()): self.copy_field_from(i, other_shadow) + + def copy_from_AllNil(self, all_nil_storage): + self.copy_from(all_nil_storage) + def copy_from_SmallIntegerOrNil(self, small_int_storage): + self.copy_from(small_int_storage) + def copy_from_FloatOrNil(self, float_storage): + self.copy_from(float_storage) class AbstractStorageShadow(AbstractShadow): _attrs_ = [] @@ -80,6 +91,8 @@ if n0 >= self._size: raise IndexError return self.space.w_nil + def copy_into(self, other_shadow): + other_shadow.copy_from_AllNil(self) def do_store(self, n0, w_value): pass def size(self): @@ -117,6 +130,9 @@ self.storage[n0] = self.nil_value else: self.storage[n0] = self.unwrap(self.space, w_val) + + def copy_from_AllNil(self, all_nil_storage): + pass # Already initialized # This is to avoid code duplication @objectmodel.specialize.arg(0) @@ -143,6 +159,8 @@ @staticmethod def unwrap(space, w_val): return space.unwrap_int(w_val) + def copy_into(self, other_shadow): + other_shadow.copy_from_SmallIntegerOrNil(self) class FloatOrNilStorageShadow(AbstractStorageShadow): repr_classname = "FloatOrNilStorageShadow" @@ -162,6 +180,8 @@ @staticmethod def unwrap(space, w_val): return space.unwrap_float(w_val) + def copy_into(self, other_shadow): + other_shadow.copy_from_FloatOrNil(self) def empty_storage(space, w_self, size, weak=False): if weak: @@ -170,6 +190,7 @@ return ListStorageShadow(space, w_self, size) return AllNilStorageShadow(space, w_self, size) + at jit.unroll_safe def find_storage_for_objects(space, vars, weak=False): if weak: return WeakListStorageShadow @@ -212,7 +233,10 @@ def copy_from(self, other_shadow): if self.size() != other_shadow.size(): self.initialize_storage(other_shadow.size()) - AbstractShadow.copy_from(self, other_shadow) + for i in range(self.size()): + w_val = other_shadow.fetch(i) + if not w_val.is_nil(self.space): + self.store(i, w_val) class ListStorageShadow(AbstractStorageShadow): _attrs_ = ['storage'] From noreply at buildbot.pypy.org Fri Jul 18 14:08:25 2014 From: noreply at buildbot.pypy.org (anton_gulenko) Date: Fri, 18 Jul 2014 14:08:25 +0200 (CEST) Subject: [pypy-commit] lang-smalltalk storage: Merged. Message-ID: <20140718120825.EDE171C01D2@cobra.cs.uni-duesseldorf.de> Author: Anton Gulenko Branch: storage Changeset: r900:f71c6de0dab8 Date: 2014-07-10 21:31 +0200 http://bitbucket.org/pypy/lang-smalltalk/changeset/f71c6de0dab8/ Log: Merged. diff --git a/spyvm/constants.py b/spyvm/constants.py --- a/spyvm/constants.py +++ b/spyvm/constants.py @@ -190,6 +190,5 @@ # Interpreter constants # -MAX_LOOP_DEPTH = 100 INTERRUPT_COUNTER_SIZE = 10000 CompileTime = time.time() diff --git a/spyvm/interpreter.py b/spyvm/interpreter.py --- a/spyvm/interpreter.py +++ b/spyvm/interpreter.py @@ -4,7 +4,7 @@ from spyvm import model, constants, primitives, conftest, wrapper from spyvm.tool.bitmanipulation import splitter -from rpython.rlib import jit +from rpython.rlib import jit, rstackovf from rpython.rlib import objectmodel, unroll class MissingBytecode(Exception): @@ -24,7 +24,7 @@ class Interpreter(object): _immutable_fields_ = ["space", "image", "image_name", - "max_stack_depth", "interrupt_counter_size", + "interrupt_counter_size", "startup_time", "evented", "interrupts"] jit_driver = jit.JitDriver( @@ -35,8 +35,7 @@ ) def __init__(self, space, image=None, image_name="", - trace=False, evented=True, interrupts=True, - max_stack_depth=constants.MAX_LOOP_DEPTH): + trace=False, evented=True, interrupts=True): import time # === Initialize immutable variables @@ -47,7 +46,6 @@ self.startup_time = image.startup_time else: self.startup_time = constants.CompileTime - self.max_stack_depth = max_stack_depth self.evented = evented self.interrupts = interrupts try: @@ -57,7 +55,6 @@ # === Initialize mutable variables self.interrupt_check_counter = self.interrupt_counter_size - self.current_stack_depth = 0 self.next_wakeup_tick = 0 self.trace = trace self.trace_proxy = False @@ -66,7 +63,6 @@ # This is the top-level loop and is not invoked recursively. s_new_context = w_active_context.as_context_get_shadow(self.space) while True: - assert self.current_stack_depth == 0 s_sender = s_new_context.s_sender() try: self.loop_bytecodes(s_new_context) @@ -76,11 +72,13 @@ print "====== StackOverflow, contexts forced to heap at: %s" % e.s_new_context.short_str() s_new_context = e.s_new_context except Return, nlr: + assert nlr.s_target_context or nlr.is_local s_new_context = s_sender - while s_new_context is not nlr.s_target_context: - s_sender = s_new_context.s_sender() - s_new_context._activate_unwind_context(self) - s_new_context = s_sender + if not nlr.is_local: + while s_new_context is not nlr.s_target_context: + s_sender = s_new_context.s_sender() + s_new_context._activate_unwind_context(self) + s_new_context = s_sender s_new_context.push(nlr.value) except ProcessSwitch, p: assert not self.space.suppress_process_switch[0], "ProcessSwitch should be disabled..." @@ -110,11 +108,16 @@ try: self.step(s_context) except Return, nlr: - if nlr.s_target_context is not s_context: + if nlr.s_target_context is s_context or nlr.is_local: + s_context.push(nlr.value) + else: + if nlr.s_target_context is None: + # This is the case where we are returning to our sender. + # Mark the return as local, so our sender will take it + nlr.is_local = True s_context._activate_unwind_context(self) raise nlr - else: - s_context.push(nlr.value) + # This is a wrapper around loop_bytecodes that cleanly enters/leaves the frame # and handles the stack overflow protection mechanism. @@ -122,16 +125,11 @@ try: if s_frame._s_sender is None and s_sender is not None: s_frame.store_s_sender(s_sender, raise_error=False) - - self.current_stack_depth += 1 - if self.max_stack_depth > 0: - if self.current_stack_depth >= self.max_stack_depth: - raise StackOverflow(s_frame) - # Now (continue to) execute the context bytecodes self.loop_bytecodes(s_frame, may_context_switch) - finally: - self.current_stack_depth -= 1 + except rstackovf.StackOverflow: + rstackovf.check_stack_overflow() + raise StackOverflow(s_frame) def step(self, context): bytecode = context.fetch_next_bytecode() @@ -205,7 +203,7 @@ s_frame = self.create_toplevel_context(w_receiver, selector, w_selector, w_arguments) self.interrupt_check_counter = self.interrupt_counter_size return self.interpret_toplevel(s_frame.w_self()) - + def create_toplevel_context(self, w_receiver, selector="", w_selector=None, w_arguments=[]): if w_selector is None: assert selector, "Need either string or W_Object selector" @@ -213,7 +211,7 @@ w_selector = self.image.w_asSymbol else: w_selector = self.perform(self.space.wrap_string(selector), "asSymbol") - + w_method = model.W_CompiledMethod(self.space, header=512) w_method.literalatput0(self.space, 1, w_selector) assert len(w_arguments) <= 7 @@ -225,7 +223,7 @@ return s_frame def padding(self, symbol=' '): - return symbol * self.current_stack_depth + return symbol class ReturnFromTopLevel(Exception): _attrs_ = ["object"] @@ -233,10 +231,11 @@ self.object = object class Return(Exception): - _attrs_ = ["value", "s_target_context"] + _attrs_ = ["value", "s_target_context", "is_local"] def __init__(self, s_target_context, w_result): self.value = w_result self.s_target_context = s_target_context + self.is_local = False class ContextSwitchException(Exception): """General Exception that causes the interpreter to leave @@ -632,7 +631,7 @@ interp.padding(), code, w_method.safe_identifier_string(), w_selector.str_content()) raise e - def _return(self, return_value, interp, s_return_to): + def _return(self, return_value, interp, local_return=False): # unfortunately, this assert is not true for some tests. TODO fix this. # assert self._stack_ptr == self.tempsize() @@ -640,36 +639,47 @@ if interp.trace: print '%s<- %s' % (interp.padding(), return_value.as_repr_string()) - if s_return_to is None: - # This should never happen while executing a normal image. - raise ReturnFromTopLevel(return_value) + if self.home_is_self() or local_return: + # a local return just needs to go up the stack once. there + # it will find the sender as a local, and we don't have to + # force the reference + s_return_to = None + if self.s_sender() is None: + # This should never happen while executing a normal image. + raise ReturnFromTopLevel(return_value) + else: + s_return_to = self.s_home().s_sender() + if s_return_to is None: + # This should never happen while executing a normal image. + raise ReturnFromTopLevel(return_value) + raise Return(s_return_to, return_value) # ====== Send/Return bytecodes ====== @bytecode_implementation() def returnReceiverBytecode(self, interp, current_bytecode): - return self._return(self.w_receiver(), interp, self.s_home().s_sender()) + return self._return(self.w_receiver(), interp) @bytecode_implementation() def returnTrueBytecode(self, interp, current_bytecode): - return self._return(interp.space.w_true, interp, self.s_home().s_sender()) + return self._return(interp.space.w_true, interp) @bytecode_implementation() def returnFalseBytecode(self, interp, current_bytecode): - return self._return(interp.space.w_false, interp, self.s_home().s_sender()) + return self._return(interp.space.w_false, interp) @bytecode_implementation() def returnNilBytecode(self, interp, current_bytecode): - return self._return(interp.space.w_nil, interp, self.s_home().s_sender()) + return self._return(interp.space.w_nil, interp) @bytecode_implementation() def returnTopFromMethodBytecode(self, interp, current_bytecode): - return self._return(self.pop(), interp, self.s_home().s_sender()) + return self._return(self.pop(), interp) @bytecode_implementation() def returnTopFromBlockBytecode(self, interp, current_bytecode): - return self._return(self.pop(), interp, self.s_sender()) + return self._return(self.pop(), interp, local_return=True) @bytecode_implementation() def sendLiteralSelectorBytecode(self, interp, current_bytecode): @@ -738,9 +748,7 @@ # ====== Misc ====== def _activate_unwind_context(self, interp): - # TODO put the constant somewhere else. - # Primitive 198 is used in BlockClosure >> ensure: - if self.is_closure_context() or self.w_method().primitive() != 198: + if self.is_closure_context() or not self.is_BlockClosure_ensure(): self.mark_returned() return # The first temp is executed flag for both #ensure: and #ifCurtailed: @@ -750,7 +758,8 @@ try: self.bytecodePrimValue(interp, 0) except Return, nlr: - if self is not nlr.s_target_context: + assert nlr.s_target_context or nlr.is_local + if self is not nlr.s_target_context and not nlr.is_local: raise nlr finally: self.mark_returned() @@ -964,11 +973,9 @@ # in order to enable tracing/jumping for message sends etc. def debugging(): def stepping_debugger_init(original): - def meth(self, space, image=None, image_name="", trace=False, - max_stack_depth=constants.MAX_LOOP_DEPTH): + def meth(self, space, image=None, image_name="", trace=False): return_value = original(self, space, image=image, - image_name=image_name, trace=trace, - max_stack_depth=max_stack_depth) + image_name=image_name, trace=trace) # ############################################################## self.message_stepping = False diff --git a/spyvm/shadow.py b/spyvm/shadow.py --- a/spyvm/shadow.py +++ b/spyvm/shadow.py @@ -797,6 +797,12 @@ def is_closure_context(self): raise NotImplementedError() + def is_BlockClosure_ensure(self): + raise NotImplementedError() + + def home_is_self(self): + raise NotImplementedError() + # === Other properties of Contexts === def mark_returned(self): @@ -1015,6 +1021,12 @@ def is_closure_context(self): return True + def is_BlockClosure_ensure(self): + return False + + def home_is_self(self): + return False + # === Temporary variables === def gettemp(self, index): @@ -1094,7 +1106,7 @@ return '[] in %s' % self.w_method().get_identifier_string() class MethodContextShadow(ContextPartShadow): - _attrs_ = ['closure', '_w_receiver', '_w_method'] + _attrs_ = ['closure', '_w_receiver', '_w_method', '_is_BlockClosure_ensure'] repr_classname = "MethodContextShadow" # === Initialization === @@ -1116,6 +1128,7 @@ self.init_stack_and_temps() else: self._w_method = None + self._is_BlockClosure_ensure = False argc = len(arguments) for i0 in range(argc): @@ -1188,6 +1201,9 @@ def store_w_method(self, w_method): assert isinstance(w_method, model.W_CompiledMethod) self._w_method = w_method + if w_method: + # Primitive 198 is used in BlockClosure >> ensure: + self._is_BlockClosure_ensure = (w_method.primitive() == 198) def w_receiver(self): return self._w_receiver @@ -1206,6 +1222,12 @@ def is_closure_context(self): return self.closure is not None + def is_BlockClosure_ensure(self): + return self._is_BlockClosure_ensure + + def home_is_self(self): + return not self.is_closure_context() + # ______________________________________________________________________ # Marriage of MethodContextShadows with PointerObjects only when required diff --git a/spyvm/test/jittest/base.py b/spyvm/test/jittest/base.py --- a/spyvm/test/jittest/base.py +++ b/spyvm/test/jittest/base.py @@ -18,7 +18,7 @@ def run(self, spy, tmpdir, code): logfile = str(tmpdir.join("x.pypylog")) proc = subprocess.Popen( - [str(spy), "-r", code.replace("\n", "\r\n"), BenchmarkImage], + [str(spy), BenchmarkImage, "-r", code.replace("\n", "\r\n")], cwd=str(tmpdir), env={"PYPYLOG": "jit-log-opt:%s" % logfile, "SDL_VIDEODRIVER": "dummy"} diff --git a/targetimageloadingsmalltalk.py b/targetimageloadingsmalltalk.py --- a/targetimageloadingsmalltalk.py +++ b/targetimageloadingsmalltalk.py @@ -14,13 +14,13 @@ print """ Usage: %s [-r|-m] [-naHu] [-jpis] [-tlLE] - image path (default: Squeak.image) - + Execution mode: (no flags) - Image will be normally opened. -r|--run

        - Code will be compiled and executed, result printed.
             -m|--method  - Selector will be sent to a SmallInteger, result printed.
             -h|--help              - Output this and exit.
-            
+
           Execution parameters:
             -n|--num  - Only with -m or -r, SmallInteger to be used as receiver (default: nil).
             -a|--arg  - Only with -m, will be used as single String argument.
@@ -30,7 +30,7 @@
                              in the image and execute the context directly. The image window will
                              probably not open. Good for benchmarking.
             -u             - Only with -m or -r, try to stop UI-process at startup. Can help benchmarking.
-            
+
           Other parameters:
             -j|--jit  - jitargs will be passed to the jit configuration.
             -p|--poll          - Actively poll for events. Try this if the image is not responding well.
@@ -45,8 +45,8 @@
             -l|--storage-log           - Output a log of storage operations.
             -L|--storage-log-aggregate - Output an aggregated storage log at the end of execution.
             -E|--storage-log-elements  - Include classnames of elements into the storage log.
-            
-    """ % (argv[0], constants.MAX_LOOP_DEPTH)
+
+    """ % argv[0]
 
 def get_parameter(argv, idx, arg):
     if len(argv) < idx + 1:
@@ -78,7 +78,6 @@
     # == Other parameters
     poll = False
     interrupts = True
-    max_stack_depth = constants.MAX_LOOP_DEPTH
     trace = False
     
     space = prebuilt_space
@@ -147,13 +146,13 @@
     except OSError as e:
         print_error("%s -- %s (LoadError)" % (os.strerror(e.errno), path))
         return 1
-    
+
     # Load & prepare image and environment
     image_reader = squeakimage.reader_for_image(space, squeakimage.Stream(data=imagedata))
     image = create_image(space, image_reader)
     interp = interpreter.Interpreter(space, image, image_name=path,
                 trace=trace, evented=not poll,
-                interrupts=interrupts, max_stack_depth=max_stack_depth)
+                interrupts=interrupts)
     space.runtime_setup(argv[0])
     print_error("") # Line break after image-loading characters
     
@@ -175,7 +174,7 @@
             context = active_context(interp.space)
     else:
         context = active_context(interp.space)
-    
+
     w_result = execute_context(interp, context)
     print result_string(w_result)
     storage_logger.print_aggregated_log()
@@ -219,13 +218,13 @@
             interp.space.suppress_process_switch[0] = False
     w_receiver_class.as_class_get_shadow(space).s_methoddict().sync_method_cache()
     return selector
-    
+
 def create_context(interp, w_receiver, selector, stringarg):
     args = []
     if stringarg:
         args.append(interp.space.wrap_string(stringarg))
     return interp.create_toplevel_context(w_receiver, selector, w_arguments = args)
-    
+
 def create_process(interp, s_frame):
     space = interp.space
     w_active_process = wrapper.scheduler(space).active_process()
@@ -242,10 +241,10 @@
         priority = 7
     w_benchmark_proc.store(space, 1, s_frame.w_self())
     w_benchmark_proc.store(space, 2, space.wrap_int(priority))
-    
+
     # Make process eligible for scheduling
     wrapper.ProcessWrapper(space, w_benchmark_proc).put_to_sleep()
-    
+
 def active_context(space):
     w_active_process = wrapper.scheduler(space).active_process()
     active_process = wrapper.ProcessWrapper(space, w_active_process)
@@ -260,7 +259,7 @@
     except error.Exit, e:
         print_error("Exited: %s" % e.msg)
         return None
-    
+
 # _____ Target and Main _____
 
 def target(driver, *args):

From noreply at buildbot.pypy.org  Fri Jul 18 14:08:27 2014
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Fri, 18 Jul 2014 14:08:27 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk storage: Fixed help string.
Message-ID: <20140718120827.288781C01D2@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: storage
Changeset: r901:37ace0c8899d
Date: 2014-07-12 13:16 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/37ace0c8899d/

Log:	Fixed help string.

diff --git a/targetimageloadingsmalltalk.py b/targetimageloadingsmalltalk.py
--- a/targetimageloadingsmalltalk.py
+++ b/targetimageloadingsmalltalk.py
@@ -12,7 +12,7 @@
 
 def _usage(argv):
     print """
-    Usage: %s  [-r|-m] [-naHu] [-jpis] [-tlLE]
+    Usage: %s  [-r|-m|-h] [-naPu] [-jpis] [-tlLE]
              - image path (default: Squeak.image)
 
           Execution mode:
@@ -35,9 +35,6 @@
             -j|--jit  - jitargs will be passed to the jit configuration.
             -p|--poll          - Actively poll for events. Try this if the image is not responding well.
             -i|--no-interrupts - Disable timer interrupt. Disables non-cooperative scheduling.
-            -s            - After num stack frames, the entire stack will be dumped to the heap.
-                                 This breaks performance, but protects agains stack overflow.
-                                 num <= 0 disables stack protection (default: %d)
             -S                 - Disable specialized storage strategies; always use generic ListStorage
             
           Logging parameters:

From noreply at buildbot.pypy.org  Fri Jul 18 14:08:28 2014
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Fri, 18 Jul 2014 14:08:28 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk storage: Added a stack_deth mechanism,
 but only if interp.trace is set.
Message-ID: <20140718120828.5FFEA1C01D2@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: storage
Changeset: r902:4c9262d36e3b
Date: 2014-07-13 15:55 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/4c9262d36e3b/

Log:	Added a stack_deth mechanism, but only if interp.trace is set. Added
	a ConstantFlag class to make CLI flags (and other boolean flags)
	constant for jit traces (promoted). A few code cleanups.

diff --git a/spyvm/interpreter.py b/spyvm/interpreter.py
--- a/spyvm/interpreter.py
+++ b/spyvm/interpreter.py
@@ -1,7 +1,7 @@
 import py
 import os
 from spyvm.shadow import ContextPartShadow, MethodContextShadow, BlockContextShadow, MethodNotFound
-from spyvm import model, constants, primitives, conftest, wrapper
+from spyvm import model, constants, primitives, conftest, wrapper, objspace
 from spyvm.tool.bitmanipulation import splitter
 
 from rpython.rlib import jit, rstackovf
@@ -36,8 +36,6 @@
 
     def __init__(self, space, image=None, image_name="",
                 trace=False, evented=True, interrupts=True):
-        import time
-
         # === Initialize immutable variables
         self.space = space
         self.image = image
@@ -56,8 +54,9 @@
         # === Initialize mutable variables
         self.interrupt_check_counter = self.interrupt_counter_size
         self.next_wakeup_tick = 0
-        self.trace = trace
-        self.trace_proxy = False
+        self.trace = objspace.ConstantFlag(trace)
+        self.trace_proxy = objspace.ConstantFlag()
+        self.stack_depth = 0
 
     def loop(self, w_active_context):
         # This is the top-level loop and is not invoked recursively.
@@ -68,7 +67,7 @@
                 self.loop_bytecodes(s_new_context)
                 raise Exception("loop_bytecodes left without raising...")
             except StackOverflow, e:
-                if self.trace:
+                if self.is_tracing():
                     print "====== StackOverflow, contexts forced to heap at: %s" % e.s_new_context.short_str()
                 s_new_context = e.s_new_context
             except Return, nlr:
@@ -81,8 +80,8 @@
                         s_new_context = s_sender
                 s_new_context.push(nlr.value)
             except ProcessSwitch, p:
-                assert not self.space.suppress_process_switch[0], "ProcessSwitch should be disabled..."
-                if self.trace:
+                assert not self.space.suppress_process_switch.is_set(), "ProcessSwitch should be disabled..."
+                if self.is_tracing():
                     print "====== Switched process from: %s" % s_new_context.short_str()
                     print "====== to: %s " % p.s_new_context.short_str()
                 s_new_context = p.s_new_context
@@ -118,16 +117,19 @@
                     s_context._activate_unwind_context(self)
                     raise nlr
 
-
     # This is a wrapper around loop_bytecodes that cleanly enters/leaves the frame
     # and handles the stack overflow protection mechanism.
     def stack_frame(self, s_frame, s_sender, may_context_switch=True):
         try:
+            if self.is_tracing():
+                self.stack_depth += 1
             if s_frame._s_sender is None and s_sender is not None:
                 s_frame.store_s_sender(s_sender, raise_error=False)
             # Now (continue to) execute the context bytecodes
             self.loop_bytecodes(s_frame, may_context_switch)
         except rstackovf.StackOverflow:
+            if self.is_tracing():
+                self.stack_depth -= 1
             rstackovf.check_stack_overflow()
             raise StackOverflow(s_frame)
 
@@ -184,7 +186,7 @@
                 wrapper.SemaphoreWrapper(self.space, semaphore).signal(s_frame)
         # We have no finalization process, so far.
         # We do not support external semaphores.
-            # In cog, the method to add such a semaphore is only called in GC.
+        # In cog, the method to add such a semaphore is only called in GC.
 
     def time_now(self):
         import time
@@ -222,8 +224,12 @@
         s_frame.push_all(list(w_arguments))
         return s_frame
 
+    def is_tracing(self):
+        return self.trace.is_set()
+        
     def padding(self, symbol=' '):
-        return symbol
+        assert self.is_tracing()
+        return self.stack_depth * symbol
 
 class ReturnFromTopLevel(Exception):
     _attrs_ = ["object"]
@@ -245,9 +251,9 @@
         self.s_new_context = s_new_context
 
 class StackOverflow(ContextSwitchException):
-    """This causes the current jit-loop to be left.
-    This is an experimental mechanism to avoid stack-overflow errors
-    on OS level, and we suspect it breaks jit performance at least sometimes."""
+    """This causes the current jit-loop to be left, thus avoiding stack overflows.
+    This breaks performance, so it should rarely happen.
+    In case of severe performance problems, execute with -t and check if this occurrs."""
 
 class ProcessSwitch(ContextSwitchException):
     """This causes the interpreter to switch the executed context."""
@@ -569,7 +575,7 @@
         self.pop() # receiver
 
         # ######################################################################
-        if interp.trace:
+        if interp.is_tracing():
             print interp.padding() + s_frame.short_str()
 
         return interp.stack_frame(s_frame, self)
@@ -586,7 +592,7 @@
         s_frame = w_method.create_frame(interp.space, receiver, w_args)
 
         # ######################################################################
-        if interp.trace:
+        if interp.is_tracing():
             print '%s %s %s: #%s' % (interp.padding('#'), special_selector, s_frame.short_str(), w_args)
             if not objectmodel.we_are_translated():
                 import pdb; pdb.set_trace()
@@ -617,7 +623,7 @@
 
     def _call_primitive(self, code, interp, argcount, w_method, w_selector):
         # ##################################################################
-        if interp.trace:
+        if interp.is_tracing():
             print "%s-> primitive %d \t(in %s, named #%s)" % (
                     interp.padding(), code, self.w_method().get_identifier_string(), w_selector.str_content())
         func = primitives.prim_holder.prim_table[code]
@@ -626,7 +632,7 @@
             # the primitive pushes the result (if any) onto the stack itself
             return func(interp, self, argcount, w_method)
         except primitives.PrimitiveFailedError, e:
-            if interp.trace:
+            if interp.is_tracing():
                 print "%s primitive %d FAILED\t (in %s, named %s)" % (
                     interp.padding(), code, w_method.safe_identifier_string(), w_selector.str_content())
             raise e
@@ -636,7 +642,7 @@
         # assert self._stack_ptr == self.tempsize()
 
         # ##################################################################
-        if interp.trace:
+        if interp.is_tracing():
             print '%s<- %s' % (interp.padding(), return_value.as_repr_string())
 
         if self.home_is_self() or local_return:
@@ -644,16 +650,16 @@
             # it will find the sender as a local, and we don't have to
             # force the reference
             s_return_to = None
-            if self.s_sender() is None:
-                # This should never happen while executing a normal image.
-                raise ReturnFromTopLevel(return_value)
+            return_from_top = self.s_sender() is None
         else:
             s_return_to = self.s_home().s_sender()
-            if s_return_to is None:
-                # This should never happen while executing a normal image.
-                raise ReturnFromTopLevel(return_value)
-
-        raise Return(s_return_to, return_value)
+            return_from_top = s_return_to is None
+        
+        if return_from_top:
+            # This should never happen while executing a normal image.
+            raise ReturnFromTopLevel(return_value)
+        else:
+            raise Return(s_return_to, return_value)
 
     # ====== Send/Return bytecodes ======
 
diff --git a/spyvm/interpreter_proxy.py b/spyvm/interpreter_proxy.py
--- a/spyvm/interpreter_proxy.py
+++ b/spyvm/interpreter_proxy.py
@@ -17,7 +17,7 @@
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib.unroll import unrolling_iterable
 
-from spyvm import error, model
+from spyvm import error, model, objspace
 
 sqInt = rffi.INT
 sqLong = rffi.LONG
@@ -50,7 +50,7 @@
         def wrapped(*c_arguments):
             assert len_unwrap_spec == len(c_arguments)
             args = ()
-            if IProxy.trace_proxy:
+            if IProxy.trace_proxy.is_set():
                 print 'Called InterpreterProxy >> %s' % func.func_name,
             assert IProxy.s_frame is not None and IProxy.space is not None and IProxy.interp is not None
             try:
@@ -63,7 +63,7 @@
                     else:
                         args += (c_arg, )
                 result = func(*args)
-                if IProxy.trace_proxy:
+                if IProxy.trace_proxy.is_set():
                     print '\t-> %s' % result
                 if result_type is oop:
                     assert isinstance(result, model.W_Object)
@@ -80,7 +80,7 @@
                 else:
                     return result
             except error.PrimitiveFailedError:
-                if IProxy.trace_proxy:
+                if IProxy.trace_proxy.is_set():
                     print '\t-> failed'
                 IProxy.failed()
                 from rpython.rlib.objectmodel import we_are_translated
@@ -999,6 +999,7 @@
         self.object_map = {}
         self.loaded_modules = {}
         self.remappable_objects = []
+        self.trace_proxy = objspace.ConstantFlag()
         self.reset()
 
     def reset(self):
@@ -1007,7 +1008,7 @@
         self.argcount = 0
         self.w_method = None
         self.fail_reason = 0
-        self.trace_proxy = False
+        self.trace_proxy.unset()
 
     def call(self, signature, interp, s_frame, argcount, w_method):
         self.initialize_from_call(signature, interp, s_frame, argcount, w_method)
@@ -1049,7 +1050,7 @@
         self.argcount = argcount
         self.w_method = w_method
         self.space = interp.space
-        self.trace_proxy = interp.trace_proxy
+        self.trace_proxy.set_to(interp.trace_proxy.is_set())
         # ensure that space.w_nil gets the first possible oop
         self.object_to_oop(self.space.w_nil)
 
diff --git a/spyvm/objspace.py b/spyvm/objspace.py
--- a/spyvm/objspace.py
+++ b/spyvm/objspace.py
@@ -6,11 +6,33 @@
 from rpython.rlib.objectmodel import instantiate, specialize
 from rpython.rlib.rarithmetic import intmask, r_uint, int_between
 
+class ConstantFlag(object):
+    """Boolean flag that can be edited, but will be promoted
+    to a constant when jitting."""
+    
+    def __init__(self, set_initially=False):
+        self.flag = [set_initially]
+    
+    def is_set(self):
+        flag = jit.promote(self.flag[0])
+        return flag
+    
+    def set(self):
+        self.flag[0] = True
+    
+    def unset(self):
+        self.flag[0] = False
+    
+    def set_to(self, flag):
+        self.flag[0] = flag
+
 class ObjSpace(object):
     def __init__(self):
-        # If this is True, then no optimizing storage strategies will be used.
+        # If this flag is set, then no optimizing storage strategies will be used.
         # Intended for performance comparisons. Breaks tests.
-        self.no_specialized_storage = [False]
+        self.no_specialized_storage = ConstantFlag()
+        # This is a hack; see compile_code() in targetimageloadingsmalltalk.py
+        self.suppress_process_switch = ConstantFlag()
         
         self.classtable = {}
         self.objtable = {}
@@ -25,9 +47,6 @@
         
         self.make_bootstrap_classes()
         self.make_bootstrap_objects()
-        
-        # This is a hack; see compile_code() in targetimageloadingsmalltalk.py
-        self.suppress_process_switch = [False]
 
     def find_executable(self, executable):
         if os.sep in executable or (os.name == "nt" and ":" in executable):
diff --git a/spyvm/plugins/vmdebugging.py b/spyvm/plugins/vmdebugging.py
--- a/spyvm/plugins/vmdebugging.py
+++ b/spyvm/plugins/vmdebugging.py
@@ -10,22 +10,22 @@
 
 @DebuggingPlugin.expose_primitive(unwrap_spec=[object])
 def trace(interp, s_frame, w_rcvr):
-    interp.trace = True
+    interp.trace.set()
     return w_rcvr
 
 @DebuggingPlugin.expose_primitive(unwrap_spec=[object])
 def untrace(interp, s_frame, w_rcvr):
-    interp.trace = False
+    interp.trace.unset()
     return w_rcvr
 
 @DebuggingPlugin.expose_primitive(unwrap_spec=[object])
 def trace_proxy(interp, s_frame, w_rcvr):
-    interp.trace_proxy = True
+    interp.trace_proxy.set()
     return w_rcvr
 
 @DebuggingPlugin.expose_primitive(unwrap_spec=[object])
 def untrace_proxy(interp, s_frame, w_rcvr):
-    interp.trace_proxy = False
+    interp.trace_proxy.unset()
     return w_rcvr
 
 @DebuggingPlugin.expose_primitive(unwrap_spec=[object])
diff --git a/spyvm/shadow.py b/spyvm/shadow.py
--- a/spyvm/shadow.py
+++ b/spyvm/shadow.py
@@ -186,7 +186,7 @@
 def empty_storage(space, w_self, size, weak=False):
     if weak:
         return WeakListStorageShadow(space, w_self, size)
-    if space.no_specialized_storage[0]:
+    if space.no_specialized_storage.is_set():
         return ListStorageShadow(space, w_self, size)
     return AllNilStorageShadow(space, w_self, size)
 
@@ -194,7 +194,7 @@
 def find_storage_for_objects(space, vars, weak=False):
     if weak:
         return WeakListStorageShadow
-    if space.no_specialized_storage[0]:
+    if space.no_specialized_storage.is_set():
         return ListStorageShadow
     specialized_strategies = 3
     all_nil_can_handle = True
diff --git a/spyvm/test/test_largeinteger.py b/spyvm/test/test_largeinteger.py
--- a/spyvm/test/test_largeinteger.py
+++ b/spyvm/test/test_largeinteger.py
@@ -8,7 +8,7 @@
     space, interp, _, _ = read_image('bootstrapped.image')
     w = space.w
     copy_to_module(locals(), __name__)
-    interp.trace = False
+    interp.trace.unset()
     space.initialize_class(space.w_String, interp)
 
 def teardown_module():
@@ -37,8 +37,8 @@
         w_selector = space.get_special_selector(selector)
     except Exception:
         w_selector = find_symbol_in_methoddict_of(selector, w(intmask(candidates[0])).getclass(space).shadow)
-
-    interp.trace=trace
+    
+    interp.trace.set_to(trace)
     for i, v in enumerate(candidates):
         x = w_l(v)
         if j is None:
@@ -50,7 +50,7 @@
                 y = w_l(j)
         z = perform_primitive(x, w_selector, y)
         assert r_uint(z.value) == r_uint(operation(v, y.value))
-    interp.trace=False
+    interp.trace.unset()
 
 def test_bitAnd():
     do_primitive("bitAnd:", operator.and_)
diff --git a/spyvm/wrapper.py b/spyvm/wrapper.py
--- a/spyvm/wrapper.py
+++ b/spyvm/wrapper.py
@@ -93,7 +93,7 @@
         active_priority = active_process.priority()
         priority = self.priority()
         if priority > active_priority:
-            if not self.space.suppress_process_switch[0]:
+            if not self.space.suppress_process_switch.is_set():
                 active_process.deactivate(s_current_frame)
                 self.activate()
         else:
@@ -104,7 +104,7 @@
 
     def suspend(self, s_current_frame):
         if self.is_active_process():
-            if not self.space.suppress_process_switch[0]:
+            if not self.space.suppress_process_switch.is_set():
                 assert self.my_list().is_nil(self.space)
                 w_process = scheduler(self.space).pop_highest_priority_process()
                 self.deactivate(s_current_frame, put_to_sleep=False)
diff --git a/targetimageloadingsmalltalk.py b/targetimageloadingsmalltalk.py
--- a/targetimageloadingsmalltalk.py
+++ b/targetimageloadingsmalltalk.py
@@ -109,7 +109,7 @@
             elif arg in ["-P", "--process"]:
                 headless = False
             elif arg in ["-S"]:
-                space.no_specialized_storage[0] = True
+                space.no_specialized_storage.set()
             elif arg in ["-u"]:
                 from spyvm.plugins.vmdebugging import stop_ui_process
                 stop_ui_process()
@@ -156,9 +156,9 @@
     # Create context to be executed
     if code or selector:
         if not have_number:
-            w_receiver = interp.space.w_nil
+            w_receiver = space.w_nil
         else:
-            w_receiver = interp.space.wrap_int(number)
+            w_receiver = space.wrap_int(number)
         if code:
             selector = compile_code(interp, w_receiver, code)
             if selector is None:
@@ -168,9 +168,9 @@
             context = s_frame
         else:
             create_process(interp, s_frame)
-            context = active_context(interp.space)
+            context = active_context(space)
     else:
-        context = active_context(interp.space)
+        context = active_context(space)
 
     w_result = execute_context(interp, context)
     print result_string(w_result)
@@ -184,7 +184,6 @@
     return w_result.as_repr_string().replace('\r', '\n')
 
 def compile_code(interp, w_receiver, code):
-    import time
     selector = "DoIt%d" % int(time.time())
     space = interp.space
     w_receiver_class = w_receiver.getclass(space)
@@ -194,7 +193,7 @@
     # registered (primitive 136 not called), so the idle process will never be left once it is entered.
     # TODO - Find a way to cleanly initialize the image, without executing the active_context of the image.
     # Instead, we want to execute our own context. Then remove this flag (and all references to it)
-    interp.space.suppress_process_switch[0] = True
+    space.suppress_process_switch.set()
     try:
         w_result = interp.perform(
             w_receiver_class,
@@ -212,7 +211,7 @@
         print_error("Exited while compiling code: %s" % e.msg)
         return None
     finally:
-            interp.space.suppress_process_switch[0] = False
+            space.suppress_process_switch.unset()
     w_receiver_class.as_class_get_shadow(space).s_methoddict().sync_method_cache()
     return selector
 

From noreply at buildbot.pypy.org  Fri Jul 18 14:08:29 2014
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Fri, 18 Jul 2014 14:08:29 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk storage: Printing better information
 in case of a doesNotUnderstand:
Message-ID: <20140718120829.7F0381C01D2@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: storage
Changeset: r903:b73849087f13
Date: 2014-07-13 16:42 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/b73849087f13/

Log:	Printing better information in case of a doesNotUnderstand:

diff --git a/spyvm/model.py b/spyvm/model.py
--- a/spyvm/model.py
+++ b/spyvm/model.py
@@ -821,9 +821,10 @@
 
     def as_string(self):
         if self.bytes is not None:
-            return "".join(self.bytes)
+            string = "".join(self.bytes)
         else:
-            return "".join([self.c_bytes[i] for i in range(self.size())])
+            string = "".join([self.c_bytes[i] for i in range(self.size())])
+        return string.replace('\r', '\n')
 
     def invariant(self):
         if not W_AbstractObjectWithClassReference.invariant(self):
diff --git a/spyvm/objspace.py b/spyvm/objspace.py
--- a/spyvm/objspace.py
+++ b/spyvm/objspace.py
@@ -33,6 +33,7 @@
         self.no_specialized_storage = ConstantFlag()
         # This is a hack; see compile_code() in targetimageloadingsmalltalk.py
         self.suppress_process_switch = ConstantFlag()
+        self.headless = ConstantFlag()
         
         self.classtable = {}
         self.objtable = {}
diff --git a/spyvm/primitives.py b/spyvm/primitives.py
--- a/spyvm/primitives.py
+++ b/spyvm/primitives.py
@@ -2,10 +2,8 @@
 import inspect
 import math
 import operator
-from spyvm import model, shadow
-from spyvm import constants, display
-from spyvm.error import PrimitiveFailedError, \
-    PrimitiveNotYetWrittenError
+from spyvm import model, shadow, error, constants, display
+from spyvm.error import PrimitiveFailedError, PrimitiveNotYetWrittenError
 from spyvm import wrapper
 
 from rpython.rlib import rarithmetic, rfloat, unroll, jit
@@ -374,14 +372,26 @@
 
 @expose_primitive(FAIL)
 def func(interp, s_frame, argcount):
-    if s_frame.w_method().lookup_selector == 'doesNotUnderstand:':
-        print ''
-        print s_frame.print_stack()
-        w_message = s_frame.peek(0)
-        print ("%s" % w_message).replace('\r', '\n')
-        print ("%s" % s_frame.peek(1)).replace('\r', '\n')
-        if isinstance(w_message, model.W_PointersObject):
-            print ('%s' % w_message.fetch_all(s_frame.space)).replace('\r', '\n')
+    if interp.space.headless.is_set() and s_frame.w_method().lookup_selector == 'doesNotUnderstand:':
+        w_msg = s_frame.peek(1)
+        if isinstance(w_msg, model.W_BytesObject):
+            print "== Error message: %s" % w_msg.as_string()
+        print "== VM Stack:%s" % s_frame.print_stack()
+        print "== Message:"
+        for w_argument in s_frame.w_arguments():
+            print w_argument.as_repr_string()
+            if isinstance(w_argument, model.W_PointersObject):
+                fields = w_argument.fetch_all(interp.space)
+                for i, w_field in enumerate(fields):
+                    print "\t%s" % w_field.as_repr_string()
+                    if i == 1 and isinstance(w_field, model.W_PointersObject):
+                        # These are the arguments to the not-undersood message
+                        for w_field_field in w_field.fetch_all(interp.space):
+                            print "\t\t%s" % w_field_field.as_repr_string()
+        w_stack = s_frame.peek(0)
+        if isinstance(w_stack, model.W_BytesObject):
+            print "== Squeak stack:\n%s" % w_stack.as_string()
+        raise error.Exit("Unhandled doesNotUnderstand:")
     raise PrimitiveFailedError()
 
 # ___________________________________________________________________________
diff --git a/spyvm/shadow.py b/spyvm/shadow.py
--- a/spyvm/shadow.py
+++ b/spyvm/shadow.py
@@ -929,6 +929,9 @@
     # ______________________________________________________________________
     # Printing
 
+    def argument_strings(self):
+        return [ w_arg.as_repr_string() for w_arg in self.w_arguments() ]
+    
     def __str__(self):
         retval = self.short_str()
         retval += "\n%s" % self.w_method().bytecode_string(markBytecode=self.pc() + 1)
@@ -1099,7 +1102,7 @@
 
     # === Printing ===
 
-    def argument_strings(self):
+    def w_arguments(self):
         return []
 
     def method_str(self):
@@ -1255,13 +1258,9 @@
 
     # === Printing ===
 
-    def argument_strings(self):
+    def w_arguments(self):
         argcount = self.w_method().argsize
-        tempsize = self.w_method().tempsize()
-        args = []
-        for i in range(argcount):
-            args.append(self.peek(tempsize - i - 1).as_repr_string())
-        return args
+        return [ self.stack_get(i) for i in range(argcount) ]
 
     def method_str(self):
         block = '[] in ' if self.is_closure_context() else ''
diff --git a/targetimageloadingsmalltalk.py b/targetimageloadingsmalltalk.py
--- a/targetimageloadingsmalltalk.py
+++ b/targetimageloadingsmalltalk.py
@@ -165,6 +165,7 @@
                 return -1 # Compilation failed, message is printed.
         s_frame = create_context(interp, w_receiver, selector, stringarg)
         if headless:
+            space.headless.set()
             context = s_frame
         else:
             create_process(interp, s_frame)

From noreply at buildbot.pypy.org  Fri Jul 18 14:08:30 2014
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Fri, 18 Jul 2014 14:08:30 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk storage: Fixed tracing and
	doesNotUnderstand outputs
Message-ID: <20140718120830.A910B1C01D2@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: storage
Changeset: r904:c5189bb59f35
Date: 2014-07-13 17:17 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/c5189bb59f35/

Log:	Fixed tracing and doesNotUnderstand outputs

diff --git a/spyvm/interpreter.py b/spyvm/interpreter.py
--- a/spyvm/interpreter.py
+++ b/spyvm/interpreter.py
@@ -128,10 +128,11 @@
             # Now (continue to) execute the context bytecodes
             self.loop_bytecodes(s_frame, may_context_switch)
         except rstackovf.StackOverflow:
+            rstackovf.check_stack_overflow()
+            raise StackOverflow(s_frame)
+        finally:
             if self.is_tracing():
                 self.stack_depth -= 1
-            rstackovf.check_stack_overflow()
-            raise StackOverflow(s_frame)
 
     def step(self, context):
         bytecode = context.fetch_next_bytecode()
@@ -229,7 +230,7 @@
         
     def padding(self, symbol=' '):
         assert self.is_tracing()
-        return self.stack_depth * symbol
+        return symbol * self.stack_depth
 
 class ReturnFromTopLevel(Exception):
     _attrs_ = ["object"]
diff --git a/spyvm/primitives.py b/spyvm/primitives.py
--- a/spyvm/primitives.py
+++ b/spyvm/primitives.py
@@ -370,24 +370,31 @@
 
 FAIL = 19
 
+def get_string(w_obj):
+    if isinstance(w_obj, model.W_BytesObject):
+        return w_obj.as_string()
+    return w_obj.as_repr_string()
+
 @expose_primitive(FAIL)
 def func(interp, s_frame, argcount):
     if interp.space.headless.is_set() and s_frame.w_method().lookup_selector == 'doesNotUnderstand:':
-        w_msg = s_frame.peek(1)
-        if isinstance(w_msg, model.W_BytesObject):
-            print "== Error message: %s" % w_msg.as_string()
+        print "== Error message: %s" % get_string(s_frame.peek(1))
+        print "== Receiver: %s" % s_frame.w_receiver().as_repr_string()
+        w_arguments = s_frame.w_arguments()
+        if len(w_arguments) >= 1:
+            w_message = w_arguments[0]
+            if isinstance(w_message, model.W_PointersObject):
+                fields = w_message.fetch_all(interp.space)
+                if len(fields) >= 1:
+                    print "== Selector: %s" % get_string(fields[0])
+                if len(fields) >= 2:
+                    w_args = fields[0]
+                    if isinstance(w_args, model.W_PointersObject):
+                        arg_strings = [ get_string(w_arg) for w_arg in w_args.fetch_all(interp.space) ]
+                        print "== Arguments: %s" % ', '.join(arg_strings)
+            else:
+                print "== Message: %s" % w_message
         print "== VM Stack:%s" % s_frame.print_stack()
-        print "== Message:"
-        for w_argument in s_frame.w_arguments():
-            print w_argument.as_repr_string()
-            if isinstance(w_argument, model.W_PointersObject):
-                fields = w_argument.fetch_all(interp.space)
-                for i, w_field in enumerate(fields):
-                    print "\t%s" % w_field.as_repr_string()
-                    if i == 1 and isinstance(w_field, model.W_PointersObject):
-                        # These are the arguments to the not-undersood message
-                        for w_field_field in w_field.fetch_all(interp.space):
-                            print "\t\t%s" % w_field_field.as_repr_string()
         w_stack = s_frame.peek(0)
         if isinstance(w_stack, model.W_BytesObject):
             print "== Squeak stack:\n%s" % w_stack.as_string()

From noreply at buildbot.pypy.org  Fri Jul 18 14:08:31 2014
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Fri, 18 Jul 2014 14:08:31 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk storage: Fixed tests and removed
	obsolete tests.
Message-ID: <20140718120831.BE2EE1C01D2@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: storage
Changeset: r905:6656aa03e7b0
Date: 2014-07-14 13:32 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/6656aa03e7b0/

Log:	Fixed tests and removed obsolete tests.

diff --git a/spyvm/test/test_interpreter.py b/spyvm/test/test_interpreter.py
--- a/spyvm/test/test_interpreter.py
+++ b/spyvm/test/test_interpreter.py
@@ -26,8 +26,11 @@
         if retval is not None:
             return retval.w_self()
     except interpreter.Return, nlr:
-        nlr.s_target_context.push(nlr.value)
-        return nlr.s_target_context.w_self()
+        new_context = nlr.s_target_context
+        if new_context is None:
+            new_context = ctxt.s_sender()
+        new_context.push(nlr.value)
+        return new_context.w_self()
 
 def assert_list(list, expected):
     for i in range(len(list)):
@@ -975,81 +978,6 @@
             2, "value:value:"]],
         test)
 
-def test_stacking_interpreter():
-    # | testBlock |
-    # testBlock := [ :aNumber |
-    #     aNumber = 0
-    #         ifTrue: [ 0 ]
-    #         ifFalse: [ (testBlock value: aNumber - 1) + aNumber ]].
-    # ^ testBlock value: 11
-    interp = TestInterpreter(space, max_stack_depth=3)
-    #create a method with the correct bytecodes and a literal
-    bytes = reduce(operator.add, map(chr, [0x8a, 0x01, 0x68, 0x10, 0x8f, 0x11,
-        0x00, 0x11, 0x10, 0x75, 0xb6, 0x9a, 0x75, 0xa4, 0x09, 0x8c, 0x00, 0x01,
-        0x10, 0x76, 0xb1, 0xca, 0x10, 0xb0, 0x7d, 0x8e, 0x00, 0x00, 0x8c, 0x00,
-        0x00, 0x20, 0xca, 0x7c]))
-    w_method = model.W_CompiledMethod(space, len(bytes))
-    w_method.islarge = 1
-    w_method.bytes = bytes
-    w_method.argsize=0
-    w_method._tempsize=1
-    w_method.setliterals([space.wrap_int(11)])
-
-    #create a frame for that method
-    w_frame = w_method.create_frame(space, space.wrap_int(0), []).w_self()
-    try:
-        interp.loop(w_frame)
-    except interpreter.ReturnFromTopLevel, e:
-        assert space.unwrap_int(e.object) == 66
-    except interpreter.StackOverflow, e:
-        assert False
-    try:
-        interp = TestInterpreter(space, image_name="", max_stack_depth=10)
-        interp._loop = True
-        interp.loop_bytecodes(w_method.create_frame(space, space.wrap_int(0), []))
-    except interpreter.StackOverflow, e:
-        assert isinstance(e.s_new_context, shadow.MethodContextShadow)
-    except interpreter.ReturnFromTopLevel, e:
-        assert False
-
-class StackTestInterpreter(TestInterpreter):
-    def stack_frame(self, s_frame, s_sender, may_interrupt=True):
-        stack_depth = self.current_stack_depth
-        for i in range(stack_depth + 1):
-            assert sys._getframe(5 + i * 7).f_code.co_name == 'loop_bytecodes'
-        assert sys._getframe(6 + stack_depth * 7).f_code.co_name == 'loop'
-        return interpreter.Interpreter.stack_frame(self, s_frame, s_sender, may_interrupt)
-
-def test_actual_stackdepth():
-    # | testBlock |
-    # testBlock := [ :aNumber |
-    #     aNumber = 0
-    #         ifTrue: [ 2 ]
-    #         ifFalse: [ (testBlock value: aNumber - 1) + aNumber ]].
-    # ^ testBlock value: 11
-    interp = StackTestInterpreter(space, max_stack_depth=10)
-    #create a method with the correct bytecodes and a literal
-    bytes = reduce(operator.add, map(chr, [0x8a, 0x01, 0x68, 0x10, 0x8f, 0x11,
-        0x00, 0x11, 0x10, 0x75, 0xb6, 0x9a, 0x77, 0xa4, 0x09, 0x8c, 0x00, 0x01,
-        0x10, 0x76, 0xb1, 0xca, 0x10, 0xb0, 0x7d, 0x8e, 0x00, 0x00, 0x8c, 0x00,
-        0x00, 0x20, 0xca, 0x7c]))
-
-    w_method = model.W_CompiledMethod(space, len(bytes))
-    w_method.islarge = 1
-    w_method.bytes = bytes
-    w_method.argsize=0
-    w_method._tempsize=1
-    w_method.setliterals([space.wrap_int(11)])
-
-    #create a frame for that method
-    w_frame = w_method.create_frame(space, space.wrap_int(0), []).w_self()
-    try:
-        interp.loop(w_frame)
-    except interpreter.ReturnFromTopLevel, e:
-        assert space.unwrap_int(e.object) == 68
-    except interpreter.StackOverflow, e:
-        assert False
-
 def test_c_stack_reset_on_sender_chain_manipulation():
     bytes = reduce(operator.add, map(chr, [0x84, 0xc0, 0x00]))
     w_frame, s_frame = new_frame(bytes)

From noreply at buildbot.pypy.org  Fri Jul 18 14:08:32 2014
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Fri, 18 Jul 2014 14:08:32 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk storage: Fixed test.
Message-ID: <20140718120832.D83981C01D2@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: storage
Changeset: r906:84e2ca8cd10e
Date: 2014-07-14 13:35 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/84e2ca8cd10e/

Log:	Fixed test.

diff --git a/spyvm/test/test_shadow.py b/spyvm/test/test_shadow.py
--- a/spyvm/test/test_shadow.py
+++ b/spyvm/test/test_shadow.py
@@ -195,11 +195,12 @@
     assert s_object.fetch(1).value == 13
 
 def test_cached_object_shadow():
-    w_o = space.wrap_list([0, 1, 2, 3, 4, 5, 6, 7])
+    l = map(space.w, [0, 1, 2, 3, 4, 5, 6, 7])
+    w_o = space.wrap_list(l)
     s_o = w_o.as_cached_object_get_shadow(space)
     version = s_o.version
     for i in range(w_o.size()):
-        assert w_o.at0(space, i) == i
+        assert w_o.at0(space, i) == l[i]
     w_o.atput0(space, 0, 8)
     assert version is not s_o.version
     assert w_o.at0(space, 0) == 8

From noreply at buildbot.pypy.org  Fri Jul 18 14:08:34 2014
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Fri, 18 Jul 2014 14:08:34 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk storage: Removed obsolete CLI flag.
Message-ID: <20140718120834.0A9E61C01D2@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: storage
Changeset: r907:0b36d560ebf1
Date: 2014-07-14 13:35 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/0b36d560ebf1/

Log:	Removed obsolete CLI flag.

diff --git a/targetimageloadingsmalltalk.py b/targetimageloadingsmalltalk.py
--- a/targetimageloadingsmalltalk.py
+++ b/targetimageloadingsmalltalk.py
@@ -12,7 +12,7 @@
 
 def _usage(argv):
     print """
-    Usage: %s  [-r|-m|-h] [-naPu] [-jpis] [-tlLE]
+    Usage: %s  [-r|-m|-h] [-naPu] [-jpiS] [-tlLE]
              - image path (default: Squeak.image)
 
           Execution mode:
@@ -104,8 +104,6 @@
                 code, idx = get_parameter(argv, idx, arg)
             elif arg in ["-i", "--no-interrupts"]:
                 interrupts = False
-            elif arg in ["-s"]:
-                max_stack_depth, idx = get_int_parameter(argv, idx, arg)
             elif arg in ["-P", "--process"]:
                 headless = False
             elif arg in ["-S"]:
@@ -172,7 +170,7 @@
             context = active_context(space)
     else:
         context = active_context(space)
-
+    
     w_result = execute_context(interp, context)
     print result_string(w_result)
     storage_logger.print_aggregated_log()

From noreply at buildbot.pypy.org  Fri Jul 18 14:08:35 2014
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Fri, 18 Jul 2014 14:08:35 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk storage: - Added a parameterized
 --trace option to control the stack depth up to which the trace is
 performed.
Message-ID: <20140718120835.3EB7E1C01D2@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: storage
Changeset: r908:cedc9509b5c7
Date: 2014-07-14 18:35 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/cedc9509b5c7/

Log:	- Added a parameterized --trace option to control the stack depth up
	to which the trace is performed.
	- Added helper to step through bytecode executions.

diff --git a/spyvm/interpreter.py b/spyvm/interpreter.py
--- a/spyvm/interpreter.py
+++ b/spyvm/interpreter.py
@@ -35,7 +35,7 @@
     )
 
     def __init__(self, space, image=None, image_name="",
-                trace=False, evented=True, interrupts=True):
+                trace_depth=-1, evented=True, interrupts=True):
         # === Initialize immutable variables
         self.space = space
         self.image = image
@@ -54,7 +54,7 @@
         # === Initialize mutable variables
         self.interrupt_check_counter = self.interrupt_counter_size
         self.next_wakeup_tick = 0
-        self.trace = objspace.ConstantFlag(trace)
+        self.trace_depth = trace_depth
         self.trace_proxy = objspace.ConstantFlag()
         self.stack_depth = 0
 
@@ -225,12 +225,36 @@
         s_frame.push_all(list(w_arguments))
         return s_frame
 
+    # ============== Methods for tracing, printing and debugging ==============
+
+    def activate_trace(self, trace_depth=0):
+        self.trace_depth = trace_depth
+        
+    def deactivate_trace(self):
+        self.trace_depth = -1
+        
     def is_tracing(self):
-        return self.trace.is_set()
+        return jit.promote(self.trace_depth) >= 0
         
-    def padding(self, symbol=' '):
-        assert self.is_tracing()
-        return symbol * self.stack_depth
+    def print_padded(self, str):
+        depth = jit.promote(self.trace_depth)
+        assert depth >= 0
+        if self.stack_depth <= depth:
+            print (' ' * self.stack_depth) + str
+    
+    def activate_debug_bytecode(self):
+        "NOT_RPYTHON"
+        def do_break(self):
+            import pdb
+            if self.break_on_bytecodes:
+                pdb.set_trace()
+        Interpreter.debug_bytecode = do_break
+        self.break_on_bytecodes = True
+    
+    def debug_bytecode(self):
+        # This is for debugging. In a pdb console, execute the following:
+        # self.activate_debug_bytecode()
+        pass
 
 class ReturnFromTopLevel(Exception):
     _attrs_ = ["object"]
@@ -287,7 +311,7 @@
                 parameters += (self.fetch_next_bytecode(), )
                 i = i + 1
             # This is a good place to step through bytecodes.
-            # import pdb; pdb.set_trace()
+            interp.debug_bytecode()
             return actual_implementation_method(self, interp, current_bytecode, *parameters)
         bytecode_implementation_wrapper.func_name = actual_implementation_method.func_name
         return bytecode_implementation_wrapper
@@ -577,7 +601,7 @@
 
         # ######################################################################
         if interp.is_tracing():
-            print interp.padding() + s_frame.short_str()
+            interp.print_padded('-> ' + s_frame.short_str())
 
         return interp.stack_frame(s_frame, self)
 
@@ -594,7 +618,7 @@
 
         # ######################################################################
         if interp.is_tracing():
-            print '%s %s %s: #%s' % (interp.padding('#'), special_selector, s_frame.short_str(), w_args)
+            interp.print_padded('-> %s %s' % (special_selector, s_frame.short_str()))
             if not objectmodel.we_are_translated():
                 import pdb; pdb.set_trace()
 
@@ -625,8 +649,8 @@
     def _call_primitive(self, code, interp, argcount, w_method, w_selector):
         # ##################################################################
         if interp.is_tracing():
-            print "%s-> primitive %d \t(in %s, named #%s)" % (
-                    interp.padding(), code, self.w_method().get_identifier_string(), w_selector.str_content())
+            interp.print_padded("-> primitive %d \t(in %s, named #%s)" % (
+                                    code, self.w_method().get_identifier_string(), w_selector.str_content()))
         func = primitives.prim_holder.prim_table[code]
         try:
             # note: argcount does not include rcvr
@@ -634,8 +658,8 @@
             return func(interp, self, argcount, w_method)
         except primitives.PrimitiveFailedError, e:
             if interp.is_tracing():
-                print "%s primitive %d FAILED\t (in %s, named %s)" % (
-                    interp.padding(), code, w_method.safe_identifier_string(), w_selector.str_content())
+                interp.print_padded("-- primitive %d FAILED\t (in %s, named %s)" % (
+                            code, w_method.safe_identifier_string(), w_selector.str_content()))
             raise e
 
     def _return(self, return_value, interp, local_return=False):
@@ -644,7 +668,7 @@
 
         # ##################################################################
         if interp.is_tracing():
-            print '%s<- %s' % (interp.padding(), return_value.as_repr_string())
+            interp.print_padded('<- ' + return_value.as_repr_string())
 
         if self.home_is_self() or local_return:
             # a local return just needs to go up the stack once. there
diff --git a/spyvm/interpreter_proxy.py b/spyvm/interpreter_proxy.py
--- a/spyvm/interpreter_proxy.py
+++ b/spyvm/interpreter_proxy.py
@@ -1016,8 +1016,8 @@
             # eventual errors are caught by the calling function (EXTERNAL_CALL)
             external_function = rffi.cast(func_bool_void,
                             self.loadFunctionFrom(signature[0], signature[1]))
-            if interp.trace:
-                print "%sCalling %s >> %s" % (interp.padding(), signature[0], signature[1])
+            if interp.is_tracing():
+                interp.print_padded("Calling %s >> %s" % (signature[0], signature[1]))
             external_function()
 
             if not self.fail_reason == 0:
diff --git a/spyvm/plugins/vmdebugging.py b/spyvm/plugins/vmdebugging.py
--- a/spyvm/plugins/vmdebugging.py
+++ b/spyvm/plugins/vmdebugging.py
@@ -10,12 +10,12 @@
 
 @DebuggingPlugin.expose_primitive(unwrap_spec=[object])
 def trace(interp, s_frame, w_rcvr):
-    interp.trace.set()
+    interp.activate_trace()
     return w_rcvr
 
 @DebuggingPlugin.expose_primitive(unwrap_spec=[object])
 def untrace(interp, s_frame, w_rcvr):
-    interp.trace.unset()
+    interp.deactivate_trace()
     return w_rcvr
 
 @DebuggingPlugin.expose_primitive(unwrap_spec=[object])
diff --git a/spyvm/test/test_largeinteger.py b/spyvm/test/test_largeinteger.py
--- a/spyvm/test/test_largeinteger.py
+++ b/spyvm/test/test_largeinteger.py
@@ -8,7 +8,7 @@
     space, interp, _, _ = read_image('bootstrapped.image')
     w = space.w
     copy_to_module(locals(), __name__)
-    interp.trace.unset()
+    interp.deactivate_trace()
     space.initialize_class(space.w_String, interp)
 
 def teardown_module():
@@ -38,7 +38,10 @@
     except Exception:
         w_selector = find_symbol_in_methoddict_of(selector, w(intmask(candidates[0])).getclass(space).shadow)
     
-    interp.trace.set_to(trace)
+    if trace:
+        interp.activate_trace()
+    else:
+        interp.deactivate_trace()
     for i, v in enumerate(candidates):
         x = w_l(v)
         if j is None:
@@ -50,7 +53,7 @@
                 y = w_l(j)
         z = perform_primitive(x, w_selector, y)
         assert r_uint(z.value) == r_uint(operation(v, y.value))
-    interp.trace.unset()
+    interp.deactivate_trace()
 
 def test_bitAnd():
     do_primitive("bitAnd:", operator.and_)
diff --git a/targetimageloadingsmalltalk.py b/targetimageloadingsmalltalk.py
--- a/targetimageloadingsmalltalk.py
+++ b/targetimageloadingsmalltalk.py
@@ -12,7 +12,7 @@
 
 def _usage(argv):
     print """
-    Usage: %s  [-r|-m|-h] [-naPu] [-jpiS] [-tlLE]
+    Usage: %s  [-r|-m|-h] [-naPu] [-jpiS] [-tTlLE]
              - image path (default: Squeak.image)
 
           Execution mode:
@@ -39,6 +39,7 @@
             
           Logging parameters:
             -t|--trace                 - Output a trace of each message, primitive, return value and process switch.
+            -T                  - Like -t, but limit the stack depth for the trace to .
             -l|--storage-log           - Output a log of storage operations.
             -L|--storage-log-aggregate - Output an aggregated storage log at the end of execution.
             -E|--storage-log-elements  - Include classnames of elements into the storage log.
@@ -75,7 +76,7 @@
     # == Other parameters
     poll = False
     interrupts = True
-    trace = False
+    trace_depth = -1
     
     space = prebuilt_space
     idx = 1
@@ -95,7 +96,11 @@
             elif arg in ["-m", "--method"]:
                 selector, idx = get_parameter(argv, idx, arg)
             elif arg in ["-t", "--trace"]:
-                trace = True
+                trace_depth = sys.maxint
+            elif arg in ["-T"]:
+                trace_depth, idx = get_int_parameter(argv, idx, arg)
+                if trace_depth < 0:
+                    raise error.Exit("Need argument >= 0 for -T.")
             elif arg in ["-p", "--poll"]:
                 poll = True
             elif arg in ["-a", "--arg"]:
@@ -146,7 +151,7 @@
     image_reader = squeakimage.reader_for_image(space, squeakimage.Stream(data=imagedata))
     image = create_image(space, image_reader)
     interp = interpreter.Interpreter(space, image, image_name=path,
-                trace=trace, evented=not poll,
+                trace_depth=trace_depth, evented=not poll,
                 interrupts=interrupts)
     space.runtime_setup(argv[0])
     print_error("") # Line break after image-loading characters

From noreply at buildbot.pypy.org  Fri Jul 18 14:08:36 2014
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Fri, 18 Jul 2014 14:08:36 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk storage: Fixed bug in CompiledMethod:
 literals were set to nil when the header was changed. Caused incorrectly
 compiled code.
Message-ID: <20140718120836.66AA11C01D2@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: storage
Changeset: r909:c063fbae6f06
Date: 2014-07-14 18:39 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/c063fbae6f06/

Log:	Fixed bug in CompiledMethod: literals were set to nil when the
	header was changed. Caused incorrectly compiled code.

diff --git a/spyvm/model.py b/spyvm/model.py
--- a/spyvm/model.py
+++ b/spyvm/model.py
@@ -1154,7 +1154,7 @@
 
     def compute_pos(self, n):
         return n * (NATIVE_DEPTH / self._depth)
-
+        
 # XXX Shouldn't compiledmethod have class reference for subclassed compiled
 # methods?
 class W_CompiledMethod(W_AbstractObjectWithIdentityHash):
@@ -1185,20 +1185,22 @@
     
     def __init__(self, space, bytecount=0, header=0):
         self.bytes = ["\x00"] * bytecount
-        self.setheader(space, header)
+        self.setheader(space, header, initializing=True)
 
     def fillin(self, space, g_self):
         # Implicitely sets the header, including self.literalsize
         for i, w_object in enumerate(g_self.get_pointers()):
-            self.literalatput0(space, i, w_object)
+            self.literalatput0(space, i, w_object, initializing=True)
         self.setbytes(g_self.get_bytes()[self.bytecodeoffset():])
 
     # === Setters ===
     
-    def setheader(self, space, header):
+    def setheader(self, space, header, initializing=False):
         _primitive, literalsize, islarge, tempsize, argsize = constants.decode_compiled_method_header(header)
-        self.literalsize = literalsize
-        self.literals = [space.w_nil] * self.literalsize
+        if initializing or self.literalsize != literalsize:
+            # Keep the literals if possible.
+            self.literalsize = literalsize
+            self.literals = [space.w_nil] * self.literalsize
         self.header = header
         self.argsize = argsize
         self._tempsize = tempsize
@@ -1314,11 +1316,11 @@
             return space.wrap_int(self.getheader())
         else:
             return self.getliteral(index0 - 1)
-
-    def literalatput0(self, space, index0, w_value):
+            
+    def literalatput0(self, space, index0, w_value, initializing=False):
         if index0 == 0:
             header = space.unwrap_int(w_value)
-            self.setheader(space, header)
+            self.setheader(space, header, initializing=initializing)
         else:
             self.setliteral(index0 - 1, w_value)
 

From noreply at buildbot.pypy.org  Fri Jul 18 14:08:37 2014
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Fri, 18 Jul 2014 14:08:37 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk storage: Improved crash report when
 running in headless mode.
Message-ID: <20140718120837.7F17F1C01D2@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: storage
Changeset: r910:9a0c4bf44ab6
Date: 2014-07-14 18:42 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/9a0c4bf44ab6/

Log:	Improved crash report when running in headless mode. Refactored
	flushCache primitive to walk the GC roots with less memory overhead.
	Not allocating huge lists containing all objects, instead walking
	recursively.

diff --git a/spyvm/interpreter.py b/spyvm/interpreter.py
--- a/spyvm/interpreter.py
+++ b/spyvm/interpreter.py
@@ -224,7 +224,7 @@
         s_frame.push(w_receiver)
         s_frame.push_all(list(w_arguments))
         return s_frame
-
+        
     # ============== Methods for tracing, printing and debugging ==============
 
     def activate_trace(self, trace_depth=0):
@@ -585,7 +585,7 @@
             w_method = receiverclassshadow.lookup(w_selector)
         except MethodNotFound:
             return self._doesNotUnderstand(w_selector, argcount, interp, receiver)
-
+        
         code = w_method.primitive()
         if code:
             if w_arguments:
@@ -615,7 +615,7 @@
         s_class = receiver.class_shadow(self.space)
         w_method = s_class.lookup(w_special_selector)
         s_frame = w_method.create_frame(interp.space, receiver, w_args)
-
+        
         # ######################################################################
         if interp.is_tracing():
             interp.print_padded('-> %s %s' % (special_selector, s_frame.short_str()))
@@ -635,13 +635,15 @@
         self.pop() # The receiver, already known.
 
         try:
+            if interp.space.headless.is_set():
+                primitives.exitFromHeadlessExecution(self, "doesNotUnderstand:", w_message)
             return self._sendSpecialSelector(interp, receiver, "doesNotUnderstand", [w_message])
         except MethodNotFound:
             from spyvm.shadow import ClassShadow
             s_class = receiver.class_shadow(self.space)
             assert isinstance(s_class, ClassShadow)
-            print "Missing doesNotUnderstand in hierarchy of %s" % s_class.getname()
-            raise
+            from spyvm import error
+            raise error.Exit("Missing doesNotUnderstand in hierarchy of %s" % s_class.getname())
 
     def _mustBeBoolean(self, interp, receiver):
         return self._sendSpecialSelector(interp, receiver, "mustBeBoolean")
@@ -665,11 +667,11 @@
     def _return(self, return_value, interp, local_return=False):
         # unfortunately, this assert is not true for some tests. TODO fix this.
         # assert self._stack_ptr == self.tempsize()
-
+        
         # ##################################################################
         if interp.is_tracing():
             interp.print_padded('<- ' + return_value.as_repr_string())
-
+        
         if self.home_is_self() or local_return:
             # a local return just needs to go up the stack once. there
             # it will find the sender as a local, and we don't have to
diff --git a/spyvm/primitives.py b/spyvm/primitives.py
--- a/spyvm/primitives.py
+++ b/spyvm/primitives.py
@@ -6,7 +6,7 @@
 from spyvm.error import PrimitiveFailedError, PrimitiveNotYetWrittenError
 from spyvm import wrapper
 
-from rpython.rlib import rarithmetic, rfloat, unroll, jit
+from rpython.rlib import rarithmetic, rfloat, unroll, jit, objectmodel
 
 def assert_class(interp, w_obj, w_class):
     if not w_obj.getclass(interp.space).is_same_object(w_class):
@@ -375,30 +375,34 @@
         return w_obj.as_string()
     return w_obj.as_repr_string()
 
+def exitFromHeadlessExecution(s_frame, selector="", w_message=None):
+    if not objectmodel.we_are_translated():
+        import pdb; pdb.set_trace()
+    print "== Receiver: %s" % s_frame.w_receiver().as_repr_string()
+    if isinstance(w_message, model.W_PointersObject):
+        fields = w_message.fetch_all(s_frame.space)
+        if len(fields) >= 1:
+            print "== Selector: %s" % get_string(fields[0])
+        if len(fields) >= 2:
+            w_args = fields[1]
+            if isinstance(w_args, model.W_PointersObject):
+                arg_strings = [ get_string(w_arg) for w_arg in w_args.fetch_all(s_frame.space) ]
+                if len(arg_strings) > 0:
+                    print "== Arguments: %s" % ', '.join(arg_strings)
+    print "== Smalltalk Stack:%s" % s_frame.print_stack()
+    if selector == "":
+        selector = s_frame.w_method().lookup_selector
+    raise error.Exit("Unhandled %s in headless mode." % selector)
+    
 @expose_primitive(FAIL)
 def func(interp, s_frame, argcount):
-    if interp.space.headless.is_set() and s_frame.w_method().lookup_selector == 'doesNotUnderstand:':
-        print "== Error message: %s" % get_string(s_frame.peek(1))
-        print "== Receiver: %s" % s_frame.w_receiver().as_repr_string()
-        w_arguments = s_frame.w_arguments()
-        if len(w_arguments) >= 1:
-            w_message = w_arguments[0]
-            if isinstance(w_message, model.W_PointersObject):
-                fields = w_message.fetch_all(interp.space)
-                if len(fields) >= 1:
-                    print "== Selector: %s" % get_string(fields[0])
-                if len(fields) >= 2:
-                    w_args = fields[0]
-                    if isinstance(w_args, model.W_PointersObject):
-                        arg_strings = [ get_string(w_arg) for w_arg in w_args.fetch_all(interp.space) ]
-                        print "== Arguments: %s" % ', '.join(arg_strings)
-            else:
-                print "== Message: %s" % w_message
-        print "== VM Stack:%s" % s_frame.print_stack()
-        w_stack = s_frame.peek(0)
-        if isinstance(w_stack, model.W_BytesObject):
-            print "== Squeak stack:\n%s" % w_stack.as_string()
-        raise error.Exit("Unhandled doesNotUnderstand:")
+    if interp.space.headless.is_set():
+        w_message = None
+        if s_frame.w_method().lookup_selector == 'doesNotUnderstand:':
+            w_arguments = s_frame.w_arguments()
+            if len(w_arguments) >= 1:
+                w_message = w_arguments[0]
+        exitFromHeadlessExecution(s_frame, w_message=w_message)
     raise PrimitiveFailedError()
 
 # ___________________________________________________________________________
@@ -725,6 +729,9 @@
 
 @expose_primitive(BE_DISPLAY, unwrap_spec=[object])
 def func(interp, s_frame, w_rcvr):
+    if interp.space.headless.is_set():
+        exitFromHeadlessExecution(s_frame)
+    
     if not isinstance(w_rcvr, model.W_PointersObject) or w_rcvr.size() < 4:
         raise PrimitiveFailedError
     # the fields required are bits (a pointer to a Bitmap), width, height, depth
@@ -857,7 +864,6 @@
 
 @expose_primitive(EXIT_TO_DEBUGGER, unwrap_spec=[object])
 def func(interp, s_frame, w_rcvr):
-    from rpython.rlib import objectmodel
     if not objectmodel.we_are_translated():
         import pdb; pdb.set_trace()
     raise PrimitiveNotYetWrittenError()
@@ -930,36 +936,38 @@
         w_class.as_class_get_shadow(interp.space).flush_method_caches()
     return w_rcvr
 
+ at objectmodel.specialize.arg(0)
+def walk_gc_references(func, gcrefs):
+    from rpython.rlib import rgc
+    for gcref in gcrefs:
+        if gcref and not rgc.get_gcflag_extra(gcref):
+            try:
+                rgc.toggle_gcflag_extra(gcref)
+                func(gcref)
+                walk_gc_references(func, rgc.get_rpy_referents(gcref))
+            finally:
+                rgc.toggle_gcflag_extra(gcref)
 
+ at objectmodel.specialize.arg(0)
+def walk_gc_objects(func):
+    from rpython.rlib import rgc
+    walk_gc_references(func, rgc.get_rpy_roots())
+
+ at objectmodel.specialize.arg(0, 1)
+def walk_gc_objects_of_type(type, func):
+    from rpython.rlib import rgc
+    def check_type(gcref):
+        w_obj = rgc.try_cast_gcref_to_instance(type, gcref)
+        if w_obj:
+            func(w_obj)
+    walk_gc_objects(check_type)
+    
 if not stm_enabled():
-    # XXX: We don't have a global symbol cache. Instead, we get all
-    # method dictionary shadows (those exists for all methodDicts that
-    # have been modified) and flush them
+    # XXX: We don't have a global symbol cache. Instead, we walk all
+    # MethodDictionaryShadow objects and flush them.
     @expose_primitive(SYMBOL_FLUSH_CACHE, unwrap_spec=[object])
     def func(interp, s_frame, w_rcvr):
-        dicts_s = []
-        from rpython.rlib import rgc
-
-        roots = [gcref for gcref in rgc.get_rpy_roots() if gcref]
-        pending = roots[:]
-        while pending:
-            gcref = pending.pop()
-            if not rgc.get_gcflag_extra(gcref):
-                rgc.toggle_gcflag_extra(gcref)
-                w_obj = rgc.try_cast_gcref_to_instance(shadow.MethodDictionaryShadow, gcref)
-                if w_obj is not None:
-                    dicts_s.append(w_obj)
-                pending.extend(rgc.get_rpy_referents(gcref))
-
-        while roots:
-            gcref = roots.pop()
-            if rgc.get_gcflag_extra(gcref):
-                rgc.toggle_gcflag_extra(gcref)
-                roots.extend(rgc.get_rpy_referents(gcref))
-
-        for s_dict in dicts_s:
-            if s_dict.invalid:
-                s_dict.sync_method_cache()
+        walk_gc_objects_of_type(shadow.MethodDictionaryShadow, lambda s_dict: s_dict.flush_method_cache())
         return w_rcvr
 
 # ___________________________________________________________________________

From noreply at buildbot.pypy.org  Fri Jul 18 14:08:38 2014
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Fri, 18 Jul 2014 14:08:38 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk storage: Commented out flushCache
 primitive. We should not need it,
 since we monitor changes to MethodDictionary objects directly. This
 primitive takes too long and prevents running the VM in interpreted mode.
Message-ID: <20140718120838.9CBA71C01D2@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: storage
Changeset: r911:63016d515218
Date: 2014-07-14 18:44 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/63016d515218/

Log:	Commented out flushCache primitive. We should not need it, since we
	monitor changes to MethodDictionary objects directly. This primitive
	takes too long and prevents running the VM in interpreted mode.

diff --git a/spyvm/primitives.py b/spyvm/primitives.py
--- a/spyvm/primitives.py
+++ b/spyvm/primitives.py
@@ -967,7 +967,9 @@
     # MethodDictionaryShadow objects and flush them.
     @expose_primitive(SYMBOL_FLUSH_CACHE, unwrap_spec=[object])
     def func(interp, s_frame, w_rcvr):
-        walk_gc_objects_of_type(shadow.MethodDictionaryShadow, lambda s_dict: s_dict.flush_method_cache())
+        # This takes a long time (at least in interpreted mode), and is not really necessary.
+        # We are monitoring changes to MethodDictionaries, so there is no need for the image to tell us.
+        #walk_gc_objects_of_type(shadow.MethodDictionaryShadow, lambda s_dict: s_dict.flush_method_cache())
         return w_rcvr
 
 # ___________________________________________________________________________

From noreply at buildbot.pypy.org  Fri Jul 18 14:08:39 2014
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Fri, 18 Jul 2014 14:08:39 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk storage: Some fixes to storage
	strategies:
Message-ID: <20140718120839.CCD4D1C01D2@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: storage
Changeset: r912:541b01ff2814
Date: 2014-07-14 20:22 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/541b01ff2814/

Log:	Some fixes to storage strategies:
	- Added size parameter to each Shadow constructor
	- Optimized transitions between certain strategies.

diff --git a/spyvm/model.py b/spyvm/model.py
--- a/spyvm/model.py
+++ b/spyvm/model.py
@@ -693,7 +693,7 @@
         old_shadow = self._get_shadow()
         shadow = old_shadow
         if not isinstance(old_shadow, TheClass):
-            shadow = TheClass(space, self)
+            shadow = TheClass(space, self, old_shadow.size())
             self.switch_shadow(shadow)
         return shadow
 
diff --git a/spyvm/plugins/bitblt.py b/spyvm/plugins/bitblt.py
--- a/spyvm/plugins/bitblt.py
+++ b/spyvm/plugins/bitblt.py
@@ -727,8 +727,8 @@
     _attrs_ = ["w_bits", "width", "height", "depth", "offsetX",
                "offsetY", "msb", "pixPerWord", "pitch", "invalid"]
 
-    def __init__(self, space, w_self):
-        AbstractCachingShadow.__init__(self, space, w_self)
+    def __init__(self, space, w_self, size):
+        AbstractCachingShadow.__init__(self, space, w_self, size)
         self.invalid = False
 
     def intOrIfNil(self, w_int, i):
diff --git a/spyvm/primitives.py b/spyvm/primitives.py
--- a/spyvm/primitives.py
+++ b/spyvm/primitives.py
@@ -1314,7 +1314,7 @@
     # The block bytecodes are stored inline: so we skip past the
     # byteodes to invoke this primitive to find them (hence +2)
     initialip = s_frame.pc() + 2
-    s_new_context = shadow.BlockContextShadow(interp.space, None, w_method_context, argcnt, initialip)
+    s_new_context = shadow.BlockContextShadow(interp.space, None, 0, w_method_context, argcnt, initialip)
     return s_new_context.w_self()
 
 @expose_primitive(VALUE, result_is_new_frame=True)
diff --git a/spyvm/shadow.py b/spyvm/shadow.py
--- a/spyvm/shadow.py
+++ b/spyvm/shadow.py
@@ -17,7 +17,7 @@
     provides_getname = False
     repr_classname = "AbstractShadow"
 
-    def __init__(self, space, w_self):
+    def __init__(self, space, w_self, size):
         self.space = space
         assert w_self is None or isinstance(w_self, model.W_PointersObject)
         self._w_self = w_self
@@ -64,7 +64,7 @@
     _attrs_ = []
     repr_classname = "AbstractStorageShadow"
     def __init__(self, space, w_self, size):
-        AbstractShadow.__init__(self, space, w_self)
+        AbstractShadow.__init__(self, space, w_self, size)
     def store(self, n0, w_val):
         if self.can_contain(w_val):
             return self.do_store(n0, w_val)
@@ -79,6 +79,15 @@
         raise NotImplementedError()
     def generalized_strategy_for(self, w_val):
         raise NotImplementedError()
+    
+    def copy_from_AllNil(self, all_nil_storage):
+        pass # Already initialized
+    def copy_from(self, other_shadow):
+        assert self.size() == other_shadow.size()
+        for i in range(self.size()):
+            w_val = other_shadow.fetch(i)
+            if not w_val.is_nil(self.space): # nil fields already initialized
+                self.store(i, w_val)
 
 class AllNilStorageShadow(AbstractStorageShadow):
     repr_classname = "AllNilStorageShadow"
@@ -130,9 +139,6 @@
             self.storage[n0] = self.nil_value
         else:
             self.storage[n0] = self.unwrap(self.space, w_val)
-            
-    def copy_from_AllNil(self, all_nil_storage):
-        pass # Already initialized
 
 # This is to avoid code duplication
 @objectmodel.specialize.arg(0)
@@ -230,13 +236,6 @@
         self.initialize_storage(size)
     def size(self):
         return len(self.storage)
-    def copy_from(self, other_shadow):
-        if self.size() != other_shadow.size():
-            self.initialize_storage(other_shadow.size())
-        for i in range(self.size()):
-            w_val = other_shadow.fetch(i)
-            if not w_val.is_nil(self.space):
-                self.store(i, w_val)
 
 class ListStorageShadow(AbstractStorageShadow):
     _attrs_ = ['storage']
@@ -273,8 +272,8 @@
     import_from_mixin(version.VersionMixin)
     version = None
 
-    def __init__(self, space, w_self):
-        ListStorageShadow.__init__(self, space, w_self, 0)
+    def __init__(self, space, w_self, size):
+        ListStorageShadow.__init__(self, space, w_self, size)
         self.changed()
 
 # ____________________________________________________________
@@ -305,9 +304,9 @@
     provides_getname = True
     repr_classname = "ClassShadow"
 
-    def __init__(self, space, w_self):
+    def __init__(self, space, w_self, size):
         self.subclass_s = {}
-        AbstractCachingShadow.__init__(self, space, w_self)
+        AbstractCachingShadow.__init__(self, space, w_self, size)
 
     def store(self, n0, w_val):
         AbstractCachingShadow.store(self, n0, w_val)
@@ -425,7 +424,7 @@
     def flush_method_caches(self):
         look_in_shadow = self
         while look_in_shadow is not None:
-            look_in_shadow.s_methoddict().sync_method_cache()
+            look_in_shadow.s_methoddict().flush_method_cache()
             look_in_shadow = look_in_shadow._s_superclass
 
     def new(self, extrasize=0):
@@ -535,7 +534,7 @@
         "NOT_RPYTHON"     # this is only for testing.
         if self._s_methoddict is None:
             w_methoddict = model.W_PointersObject(self.space, None, 2)
-            w_methoddict.store(self.space, 1, model.W_PointersObject(self.space, None, 0))
+            w_methoddict.store(self.space, constants.METHODDICT_VALUES_INDEX, model.W_PointersObject(self.space, None, 0))
             self.store_s_methoddict(w_methoddict.as_methoddict_get_shadow(self.space))
         self.s_methoddict().invalid = False
 
@@ -553,11 +552,11 @@
     _attrs_ = ['methoddict', 'invalid', 's_class']
     repr_classname = "MethodDictionaryShadow"
 
-    def __init__(self, space, w_self):
+    def __init__(self, space, w_self, size):
         self.invalid = True
         self.s_class = None
         self.methoddict = {}
-        ListStorageShadow.__init__(self, space, w_self, 0)
+        ListStorageShadow.__init__(self, space, w_self, size)
 
     def update(self):
         self.sync_method_cache()
@@ -567,7 +566,7 @@
             return None # we may be invalid if Smalltalk code did not call flushCache
         return self.methoddict.get(w_selector, None)
 
-    # Remove update call for changes to ourselves:
+    # We do not call update() after changes to ourselves:
     # Whenever a method is added, it's keyword is added to w_self, then the
     # w_compiled_method is added to our observee.
     # sync_method_cache at this point would not have the desired effect, because in
@@ -575,17 +574,30 @@
     # its contents array is filled with the value belonging to the new key.
     def store(self, n0, w_value):
         ListStorageShadow.store(self, n0, w_value)
-        self.invalid = True
-
+        if n0 == constants.METHODDICT_VALUES_INDEX:
+            self.setup_notification()
+        if n0 >= constants.METHODDICT_NAMES_INDEX:
+            self.invalid = True
+    
+    def setup_notification(self):
+        self.w_values().as_observed_get_shadow(self.space).notify(self)
+        
+    def w_values(self):
+        w_values = self.fetch(constants.METHODDICT_VALUES_INDEX)
+        assert isinstance(w_values, model.W_PointersObject)
+        return w_values
+        
+    def flush_method_cache(self):   
+        # Lazy synchronization: Only flush the cache, if we are already synchronized.
+        if self.invalid:
+            self.sync_method_cache()
+        
     def sync_method_cache(self):
         if self.size() == 0:
             return
-        w_values = self.fetch(constants.METHODDICT_VALUES_INDEX)
-        assert isinstance(w_values, model.W_PointersObject)
-        s_values = w_values.as_observed_get_shadow(self.space)
-        s_values.notify(self)
+        self.methoddict = {}
         size = self.size() - constants.METHODDICT_NAMES_INDEX
-        self.methoddict = {}
+        w_values = self.w_values()
         for i in range(size):
             w_selector = self.w_self().fetch(self.space, constants.METHODDICT_NAMES_INDEX+i)
             if not w_selector.is_nil(self.space):
@@ -614,12 +626,12 @@
     _attrs_ = ['_w_self_size']
     repr_classname = "AbstractRedirectingShadow"
 
-    def __init__(self, space, w_self):
-        AbstractShadow.__init__(self, space, w_self)
+    def __init__(self, space, w_self, size):
         if w_self is not None:
             self._w_self_size = w_self.size()
         else:
-            self._w_self_size = 0
+            self._w_self_size = size
+        AbstractShadow.__init__(self, space, w_self, self._w_self_size)
 
     def size(self):
         return self._w_self_size
@@ -641,9 +653,9 @@
     # ______________________________________________________________________
     # Initialization
 
-    def __init__(self, space, w_self):
+    def __init__(self, space, w_self, size=0):
         self._s_sender = None
-        AbstractRedirectingShadow.__init__(self, space, w_self)
+        AbstractRedirectingShadow.__init__(self, space, w_self, size)
         self.instances_w = {}
 
     def copy_field_from(self, n0, other_shadow):
@@ -981,14 +993,14 @@
 
     # === Initialization ===
 
-    def __init__(self, space, w_self=None, w_home=None, argcnt=0, initialip=0):
+    def __init__(self, space, w_self=None, size=0, w_home=None, argcnt=0, initialip=0):
         self = jit.hint(self, access_directly=True, fresh_virtualizable=True)
         creating_w_self = w_self is None
         if creating_w_self:
             s_home = w_home.as_methodcontext_get_shadow(space)
             contextsize = s_home.size() - s_home.tempsize()
             w_self = model.W_PointersObject(space, space.w_BlockContext, contextsize)
-        ContextPartShadow.__init__(self, space, w_self)
+        ContextPartShadow.__init__(self, space, w_self, size)
         if creating_w_self:
             w_self.store_shadow(self)
         self.store_expected_argument_count(argcnt)
@@ -1115,10 +1127,10 @@
     # === Initialization ===
 
     @jit.unroll_safe
-    def __init__(self, space, w_self=None, w_method=None, w_receiver=None,
+    def __init__(self, space, w_self=None, size=0, w_method=None, w_receiver=None,
                               arguments=[], closure=None, pc=0):
         self = jit.hint(self, access_directly=True, fresh_virtualizable=True)
-        ContextPartShadow.__init__(self, space, w_self)
+        ContextPartShadow.__init__(self, space, w_self, size)
         self.store_w_receiver(w_receiver)
         self.store_pc(pc)
         self.closure = closure
@@ -1280,8 +1292,8 @@
 class ObserveeShadow(ListStorageShadow):
     _attrs_ = ['dependent']
     repr_classname = "ObserveeShadow"
-    def __init__(self, space, w_self):
-        ListStorageShadow.__init__(self, space, w_self, 0)
+    def __init__(self, space, w_self, size):
+        ListStorageShadow.__init__(self, space, w_self, size)
         self.dependent = None
 
     def store(self, n0, w_value):

From noreply at buildbot.pypy.org  Fri Jul 18 14:08:41 2014
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Fri, 18 Jul 2014 14:08:41 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk storage: Added safety-net try-except
 block to print Exception messages (not printed by itself in translated
 mode).
Message-ID: <20140718120841.07FF11C01D2@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: storage
Changeset: r913:0f59f7f54a2c
Date: 2014-07-15 22:54 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/0f59f7f54a2c/

Log:	Added safety-net try-except block to print Exception messages (not
	printed by itself in translated mode).

diff --git a/targetimageloadingsmalltalk.py b/targetimageloadingsmalltalk.py
--- a/targetimageloadingsmalltalk.py
+++ b/targetimageloadingsmalltalk.py
@@ -64,6 +64,16 @@
     
 prebuilt_space = objspace.ObjSpace()
 
+def safe_entry_point(argv):
+    try:
+        return entry_point(argv)
+    except error.Exit, e:
+        print_error("Exited: %s" % e.msg)
+        return -1
+    except Exception, e:
+        print_error("Exception: %s" % str(e))
+        return -1
+
 def entry_point(argv):
     # == Main execution parameters
     path = None
@@ -146,7 +156,7 @@
     except OSError as e:
         print_error("%s -- %s (LoadError)" % (os.strerror(e.errno), path))
         return 1
-
+    
     # Load & prepare image and environment
     image_reader = squeakimage.reader_for_image(space, squeakimage.Stream(data=imagedata))
     image = create_image(space, image_reader)
@@ -164,8 +174,6 @@
             w_receiver = space.wrap_int(number)
         if code:
             selector = compile_code(interp, w_receiver, code)
-            if selector is None:
-                return -1 # Compilation failed, message is printed.
         s_frame = create_context(interp, w_receiver, selector, stringarg)
         if headless:
             space.headless.set()
@@ -198,24 +206,19 @@
     # TODO - Find a way to cleanly initialize the image, without executing the active_context of the image.
     # Instead, we want to execute our own context. Then remove this flag (and all references to it)
     space.suppress_process_switch.set()
-    try:
-        w_result = interp.perform(
-            w_receiver_class,
-            "compile:classified:notifying:",
-            w_arguments = [space.wrap_string("%s\r\n%s" % (selector, code)),
-            space.wrap_string("spy-run-code"),
-            space.w_nil]
-        )
-        
-        # TODO - is this expected in every image?
-        if not isinstance(w_result, model.W_BytesObject) or w_result.as_string() != selector:
-            print_error("Compilation failed, unexpected result: %s" % result_string(w_result))
-            return None
-    except error.Exit, e:
-        print_error("Exited while compiling code: %s" % e.msg)
-        return None
-    finally:
-            space.suppress_process_switch.unset()
+    
+    w_result = interp.perform(
+        w_receiver_class,
+        "compile:classified:notifying:",
+        w_arguments = [space.wrap_string("%s\r\n%s" % (selector, code)),
+        space.wrap_string("spy-run-code"),
+        space.w_nil]
+    )
+    # TODO - is this expected in every image?
+    if not isinstance(w_result, model.W_BytesObject) or w_result.as_string() != selector:
+        raise error.Exit("Unexpected compilation result (probably failed to compile): %s" % result_string(w_result))
+    space.suppress_process_switch.unset()
+    
     w_receiver_class.as_class_get_shadow(space).s_methoddict().sync_method_cache()
     return selector
 
@@ -253,12 +256,8 @@
     active_process.store_suspended_context(space.w_nil)
     return w_active_context.as_context_get_shadow(space)
 
-def execute_context(interp, s_frame, measure=False):
-    try:
-        return interp.interpret_toplevel(s_frame.w_self())
-    except error.Exit, e:
-        print_error("Exited: %s" % e.msg)
-        return None
+def execute_context(interp, s_frame):
+    return interp.interpret_toplevel(s_frame.w_self())
 
 # _____ Target and Main _____
 
@@ -269,11 +268,11 @@
     if hasattr(rgc, "stm_is_enabled"):
         driver.config.translation.stm = True
         driver.config.translation.thread = True
-    return entry_point, None
+    return safe_entry_point, None
 
 def jitpolicy(self):
     from rpython.jit.codewriter.policy import JitPolicy
     return JitPolicy()
 
 if __name__ == "__main__":
-    entry_point(sys.argv)
+    safe_entry_point(sys.argv)

From noreply at buildbot.pypy.org  Fri Jul 18 14:08:42 2014
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Fri, 18 Jul 2014 14:08:42 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk storage: Fixed an initialization issue
Message-ID: <20140718120842.2B4951C01D2@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: storage
Changeset: r914:a742cab321c8
Date: 2014-07-15 23:51 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/a742cab321c8/

Log:	Fixed an initialization issue

diff --git a/spyvm/model.py b/spyvm/model.py
--- a/spyvm/model.py
+++ b/spyvm/model.py
@@ -614,8 +614,8 @@
     
     def switch_shadow(self, new_shadow, w_element=None):
         old_shadow = self.assert_shadow()
+        self.store_shadow(new_shadow)
         old_shadow.copy_into(new_shadow)
-        self.store_shadow(new_shadow)
         new_shadow.attach_shadow()
         self.log_storage("Switched", old_shadow, w_element=w_element)
     

From noreply at buildbot.pypy.org  Fri Jul 18 14:08:43 2014
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Fri, 18 Jul 2014 14:08:43 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk storage: Added a name for the
	executable
Message-ID: <20140718120843.516C61C01D2@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: storage
Changeset: r915:c52509373e77
Date: 2014-07-16 00:19 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/c52509373e77/

Log:	Added a name for the executable

diff --git a/targetimageloadingsmalltalk.py b/targetimageloadingsmalltalk.py
--- a/targetimageloadingsmalltalk.py
+++ b/targetimageloadingsmalltalk.py
@@ -268,6 +268,7 @@
     if hasattr(rgc, "stm_is_enabled"):
         driver.config.translation.stm = True
         driver.config.translation.thread = True
+    driver.exe_name = "rsqueak"
     return safe_entry_point, None
 
 def jitpolicy(self):

From noreply at buildbot.pypy.org  Fri Jul 18 14:08:59 2014
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Fri, 18 Jul 2014 14:08:59 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk storage: Changes to the benchmarks. We
 really should put this on Monticello or something.
Message-ID: <20140718120859.0BD2B1C01D2@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: storage
Changeset: r916:30a053f8596e
Date: 2014-07-16 11:07 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/30a053f8596e/

Log:	Changes to the benchmarks. We really should put this on Monticello
	or something.

diff --git a/images/Squeak4.5-noBitBlt.changes b/images/Squeak4.5-noBitBlt.changes
--- a/images/Squeak4.5-noBitBlt.changes
+++ b/images/Squeak4.5-noBitBlt.changes
@@ -12348,4 +12348,276 @@
 	
 	^ 0! !
!BalloonEngine methodsFor: 'primitives-misc' stamp: 'tfel 7/2/2014 17:13' prior: 17242774!
primInitializeBuffer: buffer
 	
-	^ 0! !
!BalloonEngine methodsFor: 'profiling' stamp: 'ar 11/11/1998 21:16' prior: 17242962!
doAddCompressedShape: points segments: nSegments leftFills: leftFills rightFills: rightFills lineWidths: lineWidths lineFills: lineFills fillIndexList: fillIndexList matrix: aMatrix
	"Note: This method is for profiling the overhead of loading a compressed shape into the engine."
	^self primAddCompressedShape: points segments: nSegments leftFills: leftFills rightFills: rightFills lineWidths: lineWidths lineFills: lineFills fillIndexList: fillIndexList matrix: aMatrix! !
!BalloonEngine methodsFor: 'as yet unclassified' stamp: 'tfel 1/7/2014 18:07' prior: 47363563!
error: aString

	^self! !

"-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- "!

BalloonEngine class
	instanceVariableNames: ''!
!BalloonEngine class methodsFor: 'accessing' stamp: 'ar 10/25/1998 17:37' prior: 17243620!
debug: aBoolean
	"BalloonEngine debug: true"
	"BalloonEngine debug: false"
	Debug := aBoolean! !
!BalloonEngine class methodsFor: 'accessing' stamp: 'ar 2/2/2001 15:47' prior: 17243791!
doProfileStats: aBool
	"Note: On Macintosh systems turning on profiling can significantly
	degrade the performance of Balloon since we're using the high
	accuracy timer for measuring."
	"BalloonEngine doProfileStats: true"
	"BalloonEngine doProfileStats: false"
	
	^false! !
!BalloonEngine class methodsFor: 'accessing' stamp: 'ar 10/30/1998 23:57' prior: 17244200!
printBezierStats
	"BalloonEngine printBezierStats"
	"BalloonEngine resetBezierStats"
	Transcript 
		cr; nextPutAll:'Bezier statistics:';
		crtab; print: (BezierStats at: 1); tab; nextPutAll:' non-monoton curves splitted';
		crtab; print: (BezierStats at: 2); tab; nextPutAll:' curves splitted for numerical accuracy';
		crtab; print: (BezierStats at: 3); tab; nextPutAll:' curves splitted to avoid integer overflow';
		crtab; print: (BezierStats at: 4); tab; nextPutAll:' curves internally converted to lines';
	endEntry.! !
!BalloonEngine class methodsFor: 'accessing' stamp: 'ar 10/28/1998 23:59' prior: 17244801!
printStat: time count: n string: aString
	Transcript
		cr;
		print: time; tab;
		nextPutAll:' mSecs -- ';
		print: n; tab;
		nextPutAll:' ops -- ';
		print: ((time asFloat / (n max: 1) asFloat) roundTo: 0.01); tab;
		nextPutAll: ' avg. mSecs/op -- ';
		nextPutAll: aString.! !
!BalloonEngine class methodsFor: 'accessing' stamp: 'ar 1/12/1999 10:52' prior: 17245153!
printStats
	"BalloonEngine doProfileStats: true"
	"BalloonEngine printStats"
	"BalloonEngine resetStats"
	Transcript cr; nextPutAll:'/************** BalloonEngine statistics ****************/'.
	self printStat: (Times at: 1) count: (Counts at: 1) string: 'Initialization'.
	self printStat: (Times at: 2) count: (Counts at: 2) string: 'Finish test'.
	self printStat: (Times at: 3) count: (Counts at: 3) string: 'Fetching/Adding GET entries'.
	self printStat: (Times at: 4) count: (Counts at: 4) string: 'Adding AET entries'.
	self printStat: (Times at: 5) count: (Counts at: 5) string: 'Fetching/Computing fills'.
	self printStat: (Times at: 6) count: (Counts at: 6) string: 'Merging fills'.
	self printStat: (Times at: 7) count: (Counts at: 7) string: 'Displaying span buffer'.
	self printStat: (Times at: 8) count: (Counts at: 8) string: 'Fetching/Updating AET entries'.
	self printStat: (Times at: 9) count: (Counts at: 9) string: 'Changing AET entries'.
	Transcript cr; print: Times sum; nextPutAll:' mSecs for all operations'.
	Transcript cr; print: Counts sum; nextPutAll: ' overall operations'.
	Transcript endEntry.! !
!BalloonEngine class methodsFor: 'accessing' stamp: 'ar 10/30/1998 23:57' prior: 17246355!
resetBezierStats
	BezierStats := WordArray new: 4.! !
!BalloonEngine class methodsFor: 'accessing' stamp: 'ar 10/28/1998 23:38' prior: 17246485!
resetStats
	Times := WordArray new: 10.
	Counts := WordArray new: 10.! !
!BalloonEngine class methodsFor: 'class initialization' stamp: 'ar 11/11/1998 22:49' prior: 17246646!
initialize
	"BalloonEngine initialize"
	BufferCache := WeakArray new: 1.
	Smalltalk garbageCollect. "Make the cache old"
	CacheProtect := Semaphore forMutualExclusion.
	Times := WordArray new: 10.
	Counts := WordArray new: 10.
	BezierStats := WordArray new: 4.
	Debug ifNil:[Debug := false].! !
!BalloonEngine class methodsFor: 'private' stamp: 'ar 11/11/1998 22:50' prior: 17247016!
allocateOrRecycleBuffer: initialSize
	"Try to recycly a buffer. If this is not possibly, create a new one."
	| buffer |
	CacheProtect critical:[
		buffer := BufferCache at: 1.
		BufferCache at: 1 put: nil.
	].
	^buffer ifNil:[BalloonBuffer new: initialSize]! !
!BalloonEngine class methodsFor: 'private' stamp: 'ar 5/28/2000 22:17' prior: 17247350!
primitiveSetBitBltPlugin: pluginName
	
	^nil! !
!BalloonEngine class methodsFor: 'private' stamp: 'eem 6/11/2008 13:00' prior: 55068329!
recycleBuffer: balloonBuffer
	"Try to keep the buffer for later drawing operations."

	CacheProtect critical:[ | buffer |
		buffer := BufferCache at: 1.
		(buffer isNil or:[buffer size < balloonBuffer size] )
			ifTrue:[BufferCache at: 1 put: balloonBuffer].
	].! !

BalloonEngine initialize!

----End fileIn of C:\Dev\lang-smalltalk\images\BalloonEngine.st----!

----SNAPSHOT----{10 July 2014 . 3:48:10 pm} Squeak4.5-noBitBlt.image priorSource: 15835154!

----SNAPSHOT----{10 July 2014 . 1:49:32 pm} Squeak4.5-noBitBlt.image priorSource: 15870393!
\ No newline at end of file
+	^ 0! !
!BalloonEngine methodsFor: 'profiling' stamp: 'ar 11/11/1998 21:16' prior: 17242962!
doAddCompressedShape: points segments: nSegments leftFills: leftFills rightFills: rightFills lineWidths: lineWidths lineFills: lineFills fillIndexList: fillIndexList matrix: aMatrix
	"Note: This method is for profiling the overhead of loading a compressed shape into the engine."
	^self primAddCompressedShape: points segments: nSegments leftFills: leftFills rightFills: rightFills lineWidths: lineWidths lineFills: lineFills fillIndexList: fillIndexList matrix: aMatrix! !
!BalloonEngine methodsFor: 'as yet unclassified' stamp: 'tfel 1/7/2014 18:07' prior: 47363563!
error: aString

	^self! !

"-- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- -- "!

BalloonEngine class
	instanceVariableNames: ''!
!BalloonEngine class methodsFor: 'accessing' stamp: 'ar 10/25/1998 17:37' prior: 17243620!
debug: aBoolean
	"BalloonEngine debug: true"
	"BalloonEngine debug: false"
	Debug := aBoolean! !
!BalloonEngine class methodsFor: 'accessing' stamp: 'ar 2/2/2001 15:47' prior: 17243791!
doProfileStats: aBool
	"Note: On Macintosh systems turning on profiling can significantly
	degrade the performance of Balloon since we're using the high
	accuracy timer for measuring."
	"BalloonEngine doProfileStats: true"
	"BalloonEngine doProfileStats: false"
	
	^false! !
!BalloonEngine class methodsFor: 'accessing' stamp: 'ar 10/30/1998 23:57' prior: 17244200!
printBezierStats
	"BalloonEngine printBezierStats"
	"BalloonEngine resetBezierStats"
	Transcript 
		cr; nextPutAll:'Bezier statistics:';
		crtab; print: (BezierStats at: 1); tab; nextPutAll:' non-monoton curves splitted';
		crtab; print: (BezierStats at: 2); tab; nextPutAll:' curves splitted for numerical accuracy';
		crtab; print: (BezierStats at: 3); tab; nextPutAll:' curves splitted to avoid integer overflow';
		crtab; print: (BezierStats at: 4); tab; nextPutAll:' curves internally converted to lines';
	endEntry.! !
!BalloonEngine class methodsFor: 'accessing' stamp: 'ar 10/28/1998 23:59' prior: 17244801!
printStat: time count: n string: aString
	Transcript
		cr;
		print: time; tab;
		nextPutAll:' mSecs -- ';
		print: n; tab;
		nextPutAll:' ops -- ';
		print: ((time asFloat / (n max: 1) asFloat) roundTo: 0.01); tab;
		nextPutAll: ' avg. mSecs/op -- ';
		nextPutAll: aString.! !
!BalloonEngine class methodsFor: 'accessing' stamp: 'ar 1/12/1999 10:52' prior: 17245153!
printStats
	"BalloonEngine doProfileStats: true"
	"BalloonEngine printStats"
	"BalloonEngine resetStats"
	Transcript cr; nextPutAll:'/************** BalloonEngine statistics ****************/'.
	self printStat: (Times at: 1) count: (Counts at: 1) string: 'Initialization'.
	self printStat: (Times at: 2) count: (Counts at: 2) string: 'Finish test'.
	self printStat: (Times at: 3) count: (Counts at: 3) string: 'Fetching/Adding GET entries'.
	self printStat: (Times at: 4) count: (Counts at: 4) string: 'Adding AET entries'.
	self printStat: (Times at: 5) count: (Counts at: 5) string: 'Fetching/Computing fills'.
	self printStat: (Times at: 6) count: (Counts at: 6) string: 'Merging fills'.
	self printStat: (Times at: 7) count: (Counts at: 7) string: 'Displaying span buffer'.
	self printStat: (Times at: 8) count: (Counts at: 8) string: 'Fetching/Updating AET entries'.
	self printStat: (Times at: 9) count: (Counts at: 9) string: 'Changing AET entries'.
	Transcript cr; print: Times sum; nextPutAll:' mSecs for all operations'.
	Transcript cr; print: Counts sum; nextPutAll: ' overall operations'.
	Transcript endEntry.! !
!BalloonEngine class methodsFor: 'accessing' stamp: 'ar 10/30/1998 23:57' prior: 17246355!
resetBezierStats
	BezierStats := WordArray new: 4.! !
!BalloonEngine class methodsFor: 'accessing' stamp: 'ar 10/28/1998 23:38' prior: 17246485!
resetStats
	Times := WordArray new: 10.
	Counts := WordArray new: 10.! !
!BalloonEngine class methodsFor: 'class initialization' stamp: 'ar 11/11/1998 22:49' prior: 17246646!
initialize
	"BalloonEngine initialize"
	BufferCache := WeakArray new: 1.
	Smalltalk garbageCollect. "Make the cache old"
	CacheProtect := Semaphore forMutualExclusion.
	Times := WordArray new: 10.
	Counts := WordArray new: 10.
	BezierStats := WordArray new: 4.
	Debug ifNil:[Debug := false].! !
!BalloonEngine class methodsFor: 'private' stamp: 'ar 11/11/1998 22:50' prior: 17247016!
allocateOrRecycleBuffer: initialSize
	"Try to recycly a buffer. If this is not possibly, create a new one."
	| buffer |
	CacheProtect critical:[
		buffer := BufferCache at: 1.
		BufferCache at: 1 put: nil.
	].
	^buffer ifNil:[BalloonBuffer new: initialSize]! !
!BalloonEngine class methodsFor: 'private' stamp: 'ar 5/28/2000 22:17' prior: 17247350!
primitiveSetBitBltPlugin: pluginName
	
	^nil! !
!BalloonEngine class methodsFor: 'private' stamp: 'eem 6/11/2008 13:00' prior: 55068329!
recycleBuffer: balloonBuffer
	"Try to keep the buffer for later drawing operations."

	CacheProtect critical:[ | buffer |
		buffer := BufferCache at: 1.
		(buffer isNil or:[buffer size < balloonBuffer size] )
			ifTrue:[BufferCache at: 1 put: balloonBuffer].
	].! !

BalloonEngine initialize!

----End fileIn of C:\Dev\lang-smalltalk\images\BalloonEngine.st----!

----SNAPSHOT----{10 July 2014 . 3:48:10 pm} Squeak4.5-noBitBlt.image priorSource: 15835154!

----SNAPSHOT----{10 July 2014 . 1:49:32 pm} Squeak4.5-noBitBlt.image priorSource: 15870393!

----STARTUP----{15 July 2014 . 11:59:16 am} as C:\Dev\lang-smalltalk\images\Squeak4.5-noBitBlt.image!

!SmallInteger methodsFor: 'as yet unclassified' stamp: 'ag 7/15/2014 11:59' prior: 49367231!
runBenchmarks

	^ Benchmarks printAllResults: self benchmarkIterations! !
!SmallInteger methodsFor: 'as yet unclassified' stamp: 'ag 7/15/2014 12:00'!
printBenchmarks

	^ Benchmarks printAllResults: self benchmarkIterations! !
!SmallInteger methodsFor: 'as yet unclassified' stamp: 'ag 7/15/2014 12:00'!
printBenchmarks: substring
	
	^ Benchmarks printMatching: substring iterations: self benchmarkIterations! !
!SmallInteger methodsFor: 'as yet unclassified' stamp: 'ag 7/15/2014 12:00' prior: 49425209!
runBenchmarks

	^ Benchmarks printAll: self benchmarkIterations! !

SmallInteger removeSelector: #testMatrix!

SmallInteger removeSelector: #withArgsFoo!
!Benchmarks class methodsFor: 'benchmarks' stamp: 'ag 7/15/2014 12:01'!
printAll
	
	^ self printAll: 5! !
!Benchmarks class methodsFor: 'benchmarks' stamp: 'ag 7/15/2014 12:02'!
printAll: iterations
	
	^ self print: self allBenchmarks iterations: iterations! !
!Benchmarks class methodsFor: 'benchmarks' stamp: 'ag 7/15/2014 12:02' prior: 49426046!
printAll: iterations
	
	^ self printMatching: self allBenchmarks iterations: iterations! !
!Benchmarks class methodsFor: 'benchmarks' stamp: 'ag 7/15/2014 12:02'!
printMatching: substring iterations: iterations
	
	| benchmarks |
	benchmarks := self selectBenchmarks: substring.
	benchmarks ifEmpty: [ 
		^ String streamContents: [ :str |
				str
					nextPutAll: 'No benchmarks matched "';
					nextPutAll: substring;
					nextPutAll: '"';
					cr;
					nextPutAll: 'Available benchmarks:'.
				self allBenchmarkNames do: [ :name | str cr; nextPutAll: name ] ] ].
	^ self run: benchmarks iterations: iterations! !
!Benchmarks class methodsFor: 'benchmarks' stamp: 'ag 7/15/2014 12:03' prior: 49426380!
printMatching: substring iterations: iterations
	
	| benchmarks |
	benchmarks := self selectBenchmarksOrString: substring.
	benchmarks isString ifTrue: [ ^ benchmarks ].
	^ self run: benchmarks iterations: iterations! !
!Benchmarks class methodsFor: 'benchmarks' stamp: 'ag 7/15/2014 12:04'!
selectBenchmarksOrString: substring
	
	| benchmarks |
	benchmarks := self selectBenchmarks: substring.
	benchmarks ifEmpty: [ 
		^ String streamContents: [ :str |
				str
					nextPutAll: 'No benchmarks matched "';
					nextPutAll: substring;
					nextPutAll: '"';
					cr;
					nextPutAll: 'Available benchmarks:'.
				self allBenchmarkNames do: [ :name | str cr; nextPutAll: name ] ] ].
	^ benchmarks! !
!Benchmarks class methodsFor: 'private' stamp: 'ag 7/15/2014 12:04' prior: 49427211!
selectBenchmarksOrString: substring
	
	| benchmarks |
	benchmarks := self selectBenchmarks: substring.
	benchmarks ifEmpty: [ 
		^ String streamContents: [ :str |
				str
					nextPutAll: 'No benchmarks matched "';
					nextPutAll: substring;
					nextPutAll: '"';
					cr;
					nextPutAll: 'Available benchmarks:'.
				self allBenchmarkNames do: [ :name | str cr; nextPutAll: name ] ] ].
	^ benchmarks! !
!Benchmarks class methodsFor: 'benchmarks' stamp: 'ag 7/15/2014 12:04' prior: 49286631!
runMatching: substring iterations: iterations
	
	| benchmarks |
	benchmarks := self selectBenchmarksOrString: substring.
	benchmarks isString ifTrue: [ ^ benchmarks ].
	^ self run: benchmarks iterations: iterations! !
!Benchmarks class methodsFor: 'benchmarks' stamp: 'ag 7/15/2014 12:05' prior: 49426919!
printMatching: substring iterations: iterations
	
	| benchmarks |
	benchmarks := self selectBenchmarksOrString: substring.
	benchmarks isString ifTrue: [ ^ benchmarks ].
	^ self print: benchmarks iterations: iterations! !

{'a'. 'b'}. ','. 'a,b'!

{'a'. 'b'}. ','. 'a,b'!
!Benchmarks class methodsFor: 'private' stamp: 'ag 7/15/2014 12:12'!
print: benchmarks iterations: iterations
	
	^ String streamContents: [ :str |
		benchmarks do: [ :bench | | instance |
			instance := bench new.
			str nextPutAll: String cr, instance name.
			str nextPutAll: ':', String cr.
			(SMarkRunner getResults: instance with: iterations)
				do: [ :result | str nextPutAll: result asString]
				separatedBy: [ str nextPut: $, ] ] ]! !
!Benchmarks class methodsFor: 'private' stamp: 'ag 7/15/2014 12:12' prior: 49428843!
print: benchmarks iterations: iterations
	
	^ String streamContents: [ :str |
		benchmarks do: [ :bench | | instance |
			instance := bench new.
			str nextPutAll: String cr, instance name.
			str nextPutAll: ':', String cr.
			(SMarkRunner getResults: instance with: iterations)
				do: [ :result | str nextPutAll: result asString ]
				separatedBy: [ str nextPut: $, ] ] ]! !
!SMarkRunner class methodsFor: 'benchmarking' stamp: 'ag 7/15/2014 12:14'!
getResults: aSuite with: nIterations
+	
+	^ (self execute: aSuite with: nIterations) results! !

----QUIT----{15 July 2014 . 12:14:32 pm} Squeak4.5-noBitBlt.image priorSource: 15870393!

----STARTUP----{15 July 2014 . 12:15:33 pm} as C:\Dev\lang-smalltalk\images\Squeak4.5-noBitBlt.image!


SMarkRunner getResults: CPBBinaryTreeBenchmark new with: 5!

results := SMarkRunner getResults: CPBBinaryTreeBenchmark new with: 5!

results!

results values anyOne!

results values anyOne first!

results values anyOne first!

results values anyOne first total!

a := nil.!

results do: [ :key | a := key ]!

a!

results keysDo: [ :key | a := key ]!

results keysDo: [ :key | a := key ]!

a!

b := nil.!

results keysAndValuesDo: [ :key :value | a := key. b:= value ]!

a!

b!

String streamContents: [ :str |
		str cr ]!
!Benchmarks class methodsFor: 'private' stamp: 'ag 7/15/2014 12:21' prior: 49429305!
print: benchmarks iterations: iterations
	
	^ String streamContents: [ :str |
		benchmarks do: [ :bench |
			(SMarkRunner getResults: bench new with: iterations)
				keysAndValuesDo: [ :name :results |
					str cr; nextPutAll: name; nextPut: $:; cr.
					results
						do: [ :result | str nextPutAll: result total asString ]
						separatedBy: [ str nextPut: $, ] ] ] ]! !

----QUIT----{15 July 2014 . 12:22:06 pm} Squeak4.5-noBitBlt.image priorSource: 15875419!

----STARTUP----{15 July 2014 . 12:22:48 pm} as C:\Dev\lang-smalltalk\images\Squeak4.5-noBitBlt.image!


Smalltalk garbageCollect!
!SMarkRunner methodsFor: 'benchmarking' stamp: 'ag 7/15/2014 12:23' prior: 49235421!
performBenchmark: aSelector
+	currentBenchmark := aSelector.
+	
+	1 to: numIterations do: [:i|
+		"self timedBenchmarkExecution: aSelector."
+		Smalltalk garbageCollect.
		suite runBenchmark: aSelector.  
+	].	
+	
+	currentBenchmark := nil.
+	
+	^ results at: (suite benchmarkNameForSelector: aSelector)! !

----QUIT----{15 July 2014 . 12:24:01 pm} Squeak4.5-noBitBlt.image priorSource: 15876594!

----STARTUP----{15 July 2014 . 12:25:13 pm} as C:\Dev\lang-smalltalk\images\Squeak4.5-noBitBlt.image!


		Smalltalk garbageCollectMost.!
!SMarkRunner methodsFor: 'benchmarking' stamp: 'ag 7/15/2014 12:25' prior: 49431333!
performBenchmark: aSelector
+	currentBenchmark := aSelector.
+	
+	1 to: numIterations do: [:i|
+		"self timedBenchmarkExecution: aSelector."
+		Smalltalk garbageCollectMost.
		suite runBenchmark: aSelector.  
+	].	
+	
+	currentBenchmark := nil.
+	
+	^ results at: (suite benchmarkNameForSelector: aSelector)! !

----QUIT----{15 July 2014 . 12:25:31 pm} Squeak4.5-noBitBlt.image priorSource: 15877197!

----STARTUP----{15 July 2014 . 12:30:53 pm} as C:\Dev\lang-smalltalk\images\Squeak4.5-noBitBlt.image!

!CPBAStarBenchmark methodsFor: 'initialize-release' stamp: 'ag 7/15/2014 12:31' prior: 48592510!
benchAStar
	astar 
		graph: graph1;
		findPath;
		reset .
	astar
		graph: graph2;
		findPath;
		reset.! !
!CPBAStarBenchmark methodsFor: 'initialize-release' stamp: 'ag 7/15/2014 12:32'!
benchAStarGraph1
	astar 
		graph: graph1;
		findPath;
		reset .! !
!CPBAStarBenchmark methodsFor: 'initialize-release' stamp: 'ag 7/15/2014 12:32'!
benchAStarGraph2
	astar 
		graph: graph2;
		findPath;
		reset .! !

CPBAStarBenchmark removeSelector: #benchAStar!
!CPBBinaryTreeBenchmark methodsFor: 'testing' stamp: 'ag 7/15/2014 12:33' prior: 48678079!
benchBinaryTree
+	"starts the binary tree benchmark"
+	
+	self binarytrees: 30 .! !

----QUIT----{15 July 2014 . 12:33:52 pm} Squeak4.5-noBitBlt.image priorSource: 15877811!

----STARTUP----{15 July 2014 . 12:34:45 pm} as C:\Dev\lang-smalltalk\images\Squeak4.5-noBitBlt.image!

!CPBBinaryTreeBenchmark methodsFor: 'testing' stamp: 'ag 7/15/2014 12:34' prior: 49433076!
benchBinaryTree
+	"starts the binary tree benchmark"
+	
+	self binarytrees: 20 .! !

----SNAPSHOT----{15 July 2014 . 12:34:56 pm} Squeak4.5-noBitBlt.image priorSource: 15878724!
!CPBBinaryTreeBenchmark methodsFor: 'testing' stamp: 'ag 7/15/2014 12:36' prior: 49433442!
benchBinaryTree
+	"starts the binary tree benchmark"
+	
+	self binarytrees: 12 .! !

----QUIT----{15 July 2014 . 12:36:45 pm} Squeak4.5-noBitBlt.image priorSource: 15879090!

----STARTUP----{15 July 2014 . 12:37 pm} as C:\Dev\lang-smalltalk\images\Squeak4.5-noBitBlt.image!

!CPBBinaryTreeBenchmark methodsFor: 'testing' stamp: 'ag 7/15/2014 12:37' prior: 49433708!
benchBinaryTree
+	"starts the binary tree benchmark"
+	
+	self binarytrees: 11 .! !

----SNAPSHOT----{15 July 2014 . 12:37:16 pm} Squeak4.5-noBitBlt.image priorSource: 15879356!
!CPBBinaryTreeBenchmark methodsFor: 'testing' stamp: 'ag 7/15/2014 12:37' prior: 49434071!
benchBinaryTree
+	"starts the binary tree benchmark"
+	
+	self binarytrees: 12 .! !

----SNAPSHOT----{15 July 2014 . 12:37:42 pm} Squeak4.5-noBitBlt.image priorSource: 15879719!

#('0000000000000000' 'FFFFFFFFFFFFFFFF' '3000000000000000' '1111111111111111' '0123456789ABCDEF' '1111111111111111' '0000000000000000' 'FEDCBA9876543210' '7CA110454A1A6E57' '0131D9619DC1376E' '07A1133E4A0B2686' '3849674C2602319E' '04B915BA43FEB5B6' '0113B970FD34F2CE' '0170F175468FB5E6' '43297FAD38E373FE' '07A7137045DA2A16' '04689104C2FD3B2F' '37D06BB516CB7546' '1F08260D1AC2465E' '584023641ABA6176' '025816164629B007' '49793EBC79B3258F' '4FB05E1515AB73A7' '49E95D6D4CA229BF' '018310DC409B26D6' '1C587F1C13924FEF' '0101010101010101' '1F1F1F1F0E0E0E0E' 'E0FEE0FEF1FEF1FE' '0000000000000000' 'FFFFFFFFFFFFFFFF' '0123456789ABCDEF' 'FEDCBA9876543210') size!
!CPBBlowfishProfiling methodsFor: 'as yet unclassified' stamp: 'ag 7/15/2014 12:44' prior: 48762750!
longDecryptionTest
+	"(1 to: keys size)"
+	(27 to: keys size)
+		do: [ :each | 
+			| key clearText cipherText enc |
+			key := keys at: each.
+			clearText := clear at: each.
+			cipherText := encrypted at: each.
+			enc := CPBBlowfish decrypt: cipherText with: key ]! !
!CPBBlowfishProfiling methodsFor: 'as yet unclassified' stamp: 'ag 7/15/2014 12:45' prior: 48763107!
longEncryptionTest
+	"(1 to: keys size)"
+	(27 to: keys size)
+		do: [ :each | 
+			| key clearText cipherText enc |
+			key := keys at: each.
+			clearText := clear at: each.
+			cipherText := encrypted at: each.
+			enc := CPBBlowfish encrypt: clearText with: key ]! !

----SNAPSHOT----{15 July 2014 . 12:45:24 pm} Squeak4.5-noBitBlt.image priorSource: 15879985!
!CPBBlowfishProfiling methodsFor: 'as yet unclassified' stamp: 'ag 7/15/2014 12:46' prior: 49435642!
longEncryptionTest
+	"(1 to: keys size)"
+	(31 to: keys size)
+		do: [ :each | 
+			| key clearText cipherText enc |
+			key := keys at: each.
+			clearText := clear at: each.
+			cipherText := encrypted at: each.
+			enc := CPBBlowfish encrypt: clearText with: key ]! !
!CPBBlowfishSuite methodsFor: 'as yet unclassified' stamp: 'ag 7/15/2014 12:46'!
benchBlowfishDecryption
+	blowfish longDecryptionTest.! !
!CPBBlowfishSuite methodsFor: 'as yet unclassified' stamp: 'ag 7/15/2014 12:46'!
benchBlowfishEncryption
+	blowfish longEncryptionTest.
! !

CPBBlowfishSuite removeSelector: #benchBlowfish!
!CPBBlowfishSuite methodsFor: 'as yet unclassified' stamp: 'ag 7/15/2014 12:46' prior: 49436460!
benchBlowfishDecryption
+	blowfish longDecryptionTest: 5.! !
!CPBBlowfishSuite methodsFor: 'as yet unclassified' stamp: 'ag 7/15/2014 12:46' prior: 49436805!
benchBlowfishDecryption
+	blowfish decryptionTest: 5.! !
!CPBBlowfishSuite methodsFor: 'as yet unclassified' stamp: 'ag 7/15/2014 12:47' prior: 49436599!
benchBlowfishEncryption
+	blowfish encryptionTest: 5.
! !
!CPBBlowfishProfiling methodsFor: 'as yet unclassified' stamp: 'ag 7/15/2014 12:47'!
encryptionTest: numKeys
+	(1 to: (numKeys max: keys size))
+		do: [ :each | 
+			| key clearText cipherText enc |
+			key := keys at: each.
+			clearText := clear at: each.
+			cipherText := encrypted at: each.
+			enc := CPBBlowfish encrypt: clearText with: key ]! !
!CPBBlowfishProfiling methodsFor: 'as yet unclassified' stamp: 'ag 7/15/2014 12:48'!
decryptionTest: numKeys
+	(1 to: (numKeys max: keys size))
+		do: [ :each | 
+			| key clearText cipherText enc |
+			key := keys at: each.
+			clearText := clear at: each.
+			cipherText := encrypted at: each.
+			enc := CPBBlowfish decrypt: cipherText with: key ]! !
!CPBBlowfishProfiling methodsFor: 'as yet unclassified' stamp: 'ag 7/15/2014 12:48' prior: 49435269!
longDecryptionTest
+	(1 to: keys size)
+		do: [ :each | 
+			| key clearText cipherText enc |
+			key := keys at: each.
+			clearText := clear at: each.
+			cipherText := encrypted at: each.
+			enc := CPBBlowfish decrypt: cipherText with: key ]! !
!CPBBlowfishProfiling methodsFor: 'as yet unclassified' stamp: 'ag 7/15/2014 12:48' prior: 49436108!
longEncryptionTest
+	self decryptionTest: keys size! !
!CPBBlowfishProfiling methodsFor: 'as yet unclassified' stamp: 'ag 7/15/2014 12:48' prior: 49438329!
longEncryptionTest
+	self encryptionTest: keys size! !
!CPBBlowfishProfiling methodsFor: 'as yet unclassified' stamp: 'ag 7/15/2014 12:48' prior: 49437979!
longDecryptionTest
+	self decryptionTest: keys size! !

----SNAPSHOT----{15 July 2014 . 12:49 pm} Squeak4.5-noBitBlt.image priorSource: 15881480!
!CPBBlowfishProfiling methodsFor: 'as yet unclassified' stamp: 'ag 7/15/2014 12:53' prior: 48759322!
initialize
+	super initialize.
+	self
+		initializeClear;
+		initializeEncrypted;
+		initializeKeys.
	clear removeFirst: 5.
	encrypted removeFirst: 5.
	keys removeFirst: 5.! !
!CPBBlowfishProfiling methodsFor: 'as yet unclassified' stamp: 'ag 7/15/2014 12:53' prior: 49438888!
initialize

	| removedKeys |
+	super initialize.
+	self
+		initializeClear;
+		initializeEncrypted;
+		initializeKeys.
	removedKeys := 8.
	clear removeFirst: removedKeys.
	encrypted removeFirst: removedKeys.
	keys removeFirst: removedKeys.! !

----SNAPSHOT----{15 July 2014 . 12:53:59 pm} Squeak4.5-noBitBlt.image priorSource: 15884263!
!CPBBlowfishSuite methodsFor: 'as yet unclassified' stamp: 'ag 7/15/2014 12:55' prior: 49436963!
benchBlowfishDecryption
+	blowfish decryptionTest: 1.! !
!CPBBlowfishSuite methodsFor: 'as yet unclassified' stamp: 'ag 7/15/2014 12:55' prior: 49437117!
benchBlowfishEncryption
+	blowfish encryptionTest: 1.
! !

----SNAPSHOT----{15 July 2014 . 12:55:59 pm} Squeak4.5-noBitBlt.image priorSource: 15884974!
!CPBBlowfishProfiling methodsFor: 'as yet unclassified' stamp: 'ag 7/15/2014 12:58' prior: 49437611!
decryptionTest: numKeys
+	(1 to: (numKeys min: keys size))
+		do: [ :each | 
+			| key clearText cipherText enc |
+			key := keys at: each.
+			clearText := clear at: each.
+			cipherText := encrypted at: each.
+			enc := CPBBlowfish decrypt: cipherText with: key ]! !
!CPBBlowfishProfiling methodsFor: 'as yet unclassified' stamp: 'ag 7/15/2014 12:58' prior: 49437260!
encryptionTest: numKeys
+	(1 to: (numKeys min: keys size))
+		do: [ :each | 
+			| key clearText cipherText enc |
+			key := keys at: each.
+			clearText := clear at: each.
+			cipherText := encrypted at: each.
+			enc := CPBBlowfish encrypt: clearText with: key ]! !

----SNAPSHOT----{15 July 2014 . 12:59:56 pm} Squeak4.5-noBitBlt.image priorSource: 15885377!
!CPBBlowfishSuite methodsFor: 'as yet unclassified' stamp: 'ag 7/15/2014 13:00' prior: 49439598!
benchBlowfishDecryption
+	blowfish decryptionTest: 3.! !
!CPBBlowfishSuite methodsFor: 'as yet unclassified' stamp: 'ag 7/15/2014 13:00' prior: 49439752!
benchBlowfishEncryption
+	blowfish encryptionTest: 3.
! !

----SNAPSHOT----{15 July 2014 . 1:01:31 pm} Squeak4.5-noBitBlt.image priorSource: 15886206!

----STARTUP----{15 July 2014 . 5:30:13 pm} as C:\Dev\lang-smalltalk\images\Squeak4.5-noBitBlt.image!

!CPBPlanner class methodsFor: 'benchmarks' stamp: 'ag 7/15/2014 17:32' prior: 48911172!
standardBenchmark
+        "This the combined benchmark."
+        "Planner standardBenchmark"
+
+         self chainTest: 100.
+         self projectionTest: 100! !

----SNAPSHOT----{15 July 2014 . 5:32:20 pm} Squeak4.5-noBitBlt.image priorSource: 15886609!
!CPBPlanner class methodsFor: 'benchmarks' stamp: 'ag 7/15/2014 17:33' prior: 48908366!
chainTest: n
+        "Do chain-of-equality-constraints performance tests."
+
+        | vars editConstraint plan planner |
+        planner := CPBPlanner new.
+        vars := (1 to: n+1) collect: [ :i | CPBVariable new].
+
+        "thread a chain of equality constraints through the variables"
+        1 to: n do:
+                [ :i || v1 v2 |
+                 v1 := vars at: i.
+                 v2 := vars at: i + 1.
+                 CPBEqualityConstraint var: v1 var: v2 strength: #required].
+
+        CPBStayConstraint var: vars last strength: #strongDefault.
+        editConstraint := CPBEditConstraint var: (vars first) strength: #preferred.
+        plan := planner extractPlanFromConstraints: (Array with: editConstraint).
+        1 to: n do: [ :v |
+                vars first value: v.
+                plan execute.
+                vars last value ~= v ifTrue: [self error: 'Chain test failed!!']].
+        editConstraint destroyConstraint! !
!CPBPlanner class methodsFor: 'benchmarks' stamp: 'ag 7/15/2014 17:33' prior: 49441326!
standardBenchmark
+        "This the combined benchmark."
+        "Planner standardBenchmark"
+
+         self chainTest: 400.
+         self projectionTest: 400! !

----SNAPSHOT----{15 July 2014 . 5:33:57 pm} Squeak4.5-noBitBlt.image priorSource: 15887059!

----SNAPSHOT----{15 July 2014 . 5:34 pm} Squeak4.5-noBitBlt.image priorSource: 15888463!
!CPBPlanner class methodsFor: 'benchmarks' stamp: 'ag 7/15/2014 17:34' prior: 49442730!
standardBenchmark
+        "This the combined benchmark."
+        "Planner standardBenchmark"
+
+         self chainTest: 1000.
+         self projectionTest: 1000! !

----SNAPSHOT----{15 July 2014 . 5:34:58 pm} Squeak4.5-noBitBlt.image priorSource: 15888556!
!CPBNBodyBenchmark methodsFor: 'nbody' stamp: 'ag 7/15/2014 17:51' prior: 49054097!
benchNBody
+	"helper method to run the n body benchmark"
+	
+	self runBenchmarkFor: 2500 withStep: 0.01d0 .! !

----SNAPSHOT----{15 July 2014 . 5:51:16 pm} Squeak4.5-noBitBlt.image priorSource: 15888902!
!CPBNBodyBenchmark methodsFor: 'nbody' stamp: 'ag 7/15/2014 17:52' prior: 49443512!
benchNBody
+	"helper method to run the n body benchmark"
+	
+	self runBenchmarkFor: 3500 withStep: 0.01d0 .! !

----SNAPSHOT----{15 July 2014 . 5:52:12 pm} Squeak4.5-noBitBlt.image priorSource: 15889190!
!CPBRichardsBenchmark methodsFor: 'testing' stamp: 'ag 7/15/2014 17:57'!
executeExampleSchedule
+	| scheduler queue |
+	scheduler := CPBScheduler new .
+	scheduler addIdleTask: (CPBScheduler idIdle) withPriority: 0 withQueue: nil times: (CPBRichardsBenchmark iterations) .
+
+	queue := CPBPacket link: nil id: (CPBScheduler idWorker) kind: (CPBPacket kindWork) .
+	queue := CPBPacket link: queue id: (CPBScheduler idWorker) kind: (CPBPacket kindWork) .
+	scheduler addWorkerTask: (CPBScheduler idWorker) withPriority: 1000 withQueue: queue .
+	
+	queue := CPBPacket link: nil id: (CPBScheduler idDeviceA) kind: (CPBPacket kindDevice) .
+	queue := CPBPacket link: queue id: (CPBScheduler idDeviceA) kind: (CPBPacket kindDevice) .
+	queue := CPBPacket link: queue id: (CPBScheduler idDeviceA) kind: (CPBPacket kindDevice) .
+	scheduler addHandlerTask: (CPBScheduler idHandlerA) withPriority: 2000 withQueue: queue .
+	
+	queue := CPBPacket link: nil id: (CPBScheduler idDeviceB) kind: (CPBPacket kindDevice) .
+	queue := CPBPacket link: queue id: (CPBScheduler idDeviceB) kind: (CPBPacket kindDevice) .
+	queue := CPBPacket link: queue id: (CPBScheduler idDeviceB) kind: (CPBPacket kindDevice) .
+	scheduler addHandlerTask: (CPBScheduler idHandlerB) withPriority: 3000 withQueue: queue .	
+
+	scheduler addDeviceTask: (CPBScheduler idDeviceA) withPriority: 4000 withQueue: nil .
+	scheduler addDeviceTask: (CPBScheduler idDeviceB) withPriority: 5000 withQueue: nil .
+	
+	scheduler schedule .
+	
+	((scheduler queueCount ~~ (CPBRichardsBenchmark expectedQueueCount)) or: 
+	(scheduler holdCount ~~ (CPBRichardsBenchmark expectedHoldCount))) ifTrue: [
+		Transcript 
+			show: 'Error during execution: queueCount= ';
+			show: (scheduler queueCount);
+			show: ', holdCount=';
+			show: (scheduler holdCount);
+			show: '.';
+			cr .
+	] .
+! !
!CPBRichardsBenchmark methodsFor: 'testing' stamp: 'ag 7/15/2014 17:58' prior: 49197044!
benchRichards
	
	50 timesRepeat: [ self executeExampleSchedule ].! !
!CPBRichardsBenchmark methodsFor: 'testing' stamp: 'ag 7/15/2014 17:58' prior: 49445901!
benchRichards
	
	100 timesRepeat: [ self executeExampleSchedule ].! !

----SNAPSHOT----{15 July 2014 . 5:58:28 pm} Squeak4.5-noBitBlt.image priorSource: 15889478!
!CPBRichardsBenchmark methodsFor: 'testing' stamp: 'ag 7/15/2014 17:59' prior: 49446059!
benchRichards
	
	200 timesRepeat: [ self executeExampleSchedule ].! !

----SNAPSHOT----{15 July 2014 . 5:59:34 pm} Squeak4.5-noBitBlt.image priorSource: 15891696!
!CPBSplayTreeBenchmark methodsFor: 'as yet unclassified' stamp: 'ag 7/15/2014 18:05' prior: 49268315!
setUp	
+	splayTree := nil.
+	kSplayTreeSize := 1000. "8000"
+	kSplayTreeModifications := 80. "80."
+	kSplayTreePayloadDepth := 5.
+	keyCounter := 0.
+	
+	Transcript showln: 'entering SplayTree setup'.
+	
+	self splayTree: (CPBSplaytree new).
+	
+	1 to: self splayTreeSize do: [:i |
+		self insertNewNode.
+	]! !

----SNAPSHOT----{15 July 2014 . 6:05:51 pm} Squeak4.5-noBitBlt.image priorSource: 15891948!
!CPBSplayTreeBenchmark methodsFor: 'as yet unclassified' stamp: 'ag 7/15/2014 18:06' prior: 49446576!
setUp	
+	splayTree := nil.
+	kSplayTreeSize := 1000. "8000"
+	kSplayTreeModifications := 300. "80."
+	kSplayTreePayloadDepth := 5.
+	keyCounter := 0.
+	
+	Transcript showln: 'entering SplayTree setup'.
+	
+	self splayTree: (CPBSplaytree new).
+	
+	1 to: self splayTreeSize do: [:i |
+		self insertNewNode.
+	]! !

----SNAPSHOT----{15 July 2014 . 6:06:41 pm} Squeak4.5-noBitBlt.image priorSource: 15892442!
!CPBSplayTreeBenchmark methodsFor: 'as yet unclassified' stamp: 'ag 7/15/2014 18:07' prior: 49447070!
setUp	
+	splayTree := nil.
+	kSplayTreeSize := 1000. "8000"
+	kSplayTreeModifications := 600. "80."
+	kSplayTreePayloadDepth := 5.
+	keyCounter := 0.
+	
+	Transcript showln: 'entering SplayTree setup'.
+	
+	self splayTree: (CPBSplaytree new).
+	
+	1 to: self splayTreeSize do: [:i |
+		self insertNewNode.
+	]! !
!CPBSplayTreeBenchmark methodsFor: 'as yet unclassified' stamp: 'ag 7/15/2014 18:08' prior: 49447565!
setUp	
+	splayTree := nil.
+	kSplayTreeSize := 1000. "8000"
+	kSplayTreeModifications := 500. "80."
+	kSplayTreePayloadDepth := 10.
+	keyCounter := 0.
+	
+	Transcript showln: 'entering SplayTree setup'.
+	
+	self splayTree: (CPBSplaytree new).
+	
+	1 to: self splayTreeSize do: [:i |
+		self insertNewNode.
+	]! !

----SNAPSHOT----{15 July 2014 . 6:08:20 pm} Squeak4.5-noBitBlt.image priorSource: 15892937!
!CPBSplayTreeBenchmark methodsFor: 'as yet unclassified' stamp: 'ag 7/15/2014 18:09' prior: 49447967!
setUp	
+	splayTree := nil.
+	kSplayTreeSize := 1000. "8000"
+	kSplayTreeModifications := 500. "80."
+	kSplayTreePayloadDepth := 7.
+	keyCounter := 0.
+	
+	Transcript showln: 'entering SplayTree setup'.
+	
+	self splayTree: (CPBSplaytree new).
+	
+	1 to: self splayTreeSize do: [:i |
+		self insertNewNode.
+	]! !

----SNAPSHOT----{15 July 2014 . 6:09:41 pm} Squeak4.5-noBitBlt.image priorSource: 15893835!
!CPBSplayTreeBenchmark methodsFor: 'as yet unclassified' stamp: 'ag 7/15/2014 18:10' prior: 49448463!
setUp	
+	splayTree := nil.
+	kSplayTreeSize := 1000. "8000"
+	kSplayTreeModifications := 500. "80."
+	kSplayTreePayloadDepth := 6.
+	keyCounter := 0.
+	
+	Transcript showln: 'entering SplayTree setup'.
+	
+	self splayTree: (CPBSplaytree new).
+	
+	1 to: self splayTreeSize do: [:i |
+		self insertNewNode.
+	]! !

----SNAPSHOT----{15 July 2014 . 6:10:56 pm} Squeak4.5-noBitBlt.image priorSource: 15894330!
\ No newline at end of file
diff --git a/images/Squeak4.5-noBitBlt.image b/images/Squeak4.5-noBitBlt.image
index 175e9df2180e298a4db4f8f60698e45e1b468425..ed92c78c940799d91bb94a8ed8527076db6816c7
GIT binary patch

[cut]


From noreply at buildbot.pypy.org  Fri Jul 18 14:09:00 2014
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Fri, 18 Jul 2014 14:09:00 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk storage: Moved SenderChainManipulation
 exception to interpreter.py, as subclass of ContextSwitchException.
Message-ID: <20140718120900.46CAE1C01D2@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: storage
Changeset: r917:25b23527e013
Date: 2014-07-18 13:34 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/25b23527e013/

Log:	Moved SenderChainManipulation exception to interpreter.py, as
	subclass of ContextSwitchException. Catching SenderChainManipulation
	explicitely in toplevel loop. Added "sender is not new_sender"
	condition to raising SenderChainManipulation. Fixed test.

diff --git a/spyvm/error.py b/spyvm/error.py
--- a/spyvm/error.py
+++ b/spyvm/error.py
@@ -30,7 +30,3 @@
     _attrs_ = ["msg"]
     def __init__(self, msg):
         self.msg = msg
-
-class SenderChainManipulation(Exception):
-    def __init__(self, manipulated_context):
-        self.s_context = manipulated_context
diff --git a/spyvm/interpreter.py b/spyvm/interpreter.py
--- a/spyvm/interpreter.py
+++ b/spyvm/interpreter.py
@@ -70,6 +70,10 @@
                 if self.is_tracing():
                     print "====== StackOverflow, contexts forced to heap at: %s" % e.s_new_context.short_str()
                 s_new_context = e.s_new_context
+            except SenderChainManipulation, e:
+                if self.is_tracing():
+                    print "====== SenderChainManipulation, contexts forced to heap at: %s" % e.s_new_context.short_str()
+                s_new_context = e.s_new_context
             except Return, nlr:
                 assert nlr.s_target_context or nlr.is_local
                 s_new_context = s_sender
@@ -198,13 +202,13 @@
 
     def interpret_toplevel(self, w_frame):
         try:
+            self.interrupt_check_counter = self.interrupt_counter_size
             self.loop(w_frame)
         except ReturnFromTopLevel, e:
             return e.object
 
     def perform(self, w_receiver, selector="", w_selector=None, w_arguments=[]):
         s_frame = self.create_toplevel_context(w_receiver, selector, w_selector, w_arguments)
-        self.interrupt_check_counter = self.interrupt_counter_size
         return self.interpret_toplevel(s_frame.w_self())
 
     def create_toplevel_context(self, w_receiver, selector="", w_selector=None, w_arguments=[]):
@@ -276,13 +280,18 @@
         self.s_new_context = s_new_context
 
 class StackOverflow(ContextSwitchException):
-    """This causes the current jit-loop to be left, thus avoiding stack overflows.
+    """This causes the current jit-loop to be left, dumping all virtualized objects to the heap.
     This breaks performance, so it should rarely happen.
     In case of severe performance problems, execute with -t and check if this occurrs."""
 
 class ProcessSwitch(ContextSwitchException):
-    """This causes the interpreter to switch the executed context."""
+    """This causes the interpreter to switch the executed context.
+    Triggered when switching the process."""
 
+class SenderChainManipulation(ContextSwitchException):
+    """Manipulation of the sender chain can invalidate the jitted C stack.
+    We have to dump all virtual objects and rebuild the stack.
+    We try to raise this as rarely as possible and as late as possible."""
 
 import rpython.rlib.unroll
 if hasattr(unroll, "unrolling_zero"):
@@ -753,15 +762,16 @@
             association = wrapper.AssociationWrapper(self.space, w_association)
             self.push(association.value())
         elif opType == 5:
+            # TODO - the following two special cases should not be necessary
             try:
                 self.w_receiver().store(self.space, third, self.top())
-            except error.SenderChainManipulation, e:
-                raise StackOverflow(self)
+            except SenderChainManipulation, e:
+                raise SenderChainManipulation(self)
         elif opType == 6:
             try:
                 self.w_receiver().store(self.space, third, self.pop())
-            except error.SenderChainManipulation, e:
-                raise StackOverflow(self)
+            except SenderChainManipulation, e:
+                raise SenderChainManipulation(self)
         elif opType == 7:
             w_association = self.w_method().getliteral(third)
             association = wrapper.AssociationWrapper(self.space, w_association)
diff --git a/spyvm/shadow.py b/spyvm/shadow.py
--- a/spyvm/shadow.py
+++ b/spyvm/shadow.py
@@ -659,10 +659,11 @@
         self.instances_w = {}
 
     def copy_field_from(self, n0, other_shadow):
+        from spyvm.interpreter import SenderChainManipulation
         try:
             AbstractRedirectingShadow.copy_field_from(self, n0, other_shadow)
-        except error.SenderChainManipulation, e:
-            assert e.s_context == self
+        except SenderChainManipulation, e:
+            assert e.s_new_context == self
 
     def copy_from(self, other_shadow):
         # Some fields have to be initialized before the rest, to ensure correct initialization.
@@ -725,9 +726,11 @@
     # === Sender ===
 
     def store_s_sender(self, s_sender, raise_error=True):
-        self._s_sender = s_sender
-        if raise_error:
-            raise error.SenderChainManipulation(self)
+        if s_sender is not self._s_sender:
+            self._s_sender = s_sender
+            if raise_error:
+                from spyvm.interpreter import SenderChainManipulation
+                raise SenderChainManipulation(self)
 
     def w_sender(self):
         sender = self.s_sender()
diff --git a/spyvm/test/test_interpreter.py b/spyvm/test/test_interpreter.py
--- a/spyvm/test/test_interpreter.py
+++ b/spyvm/test/test_interpreter.py
@@ -983,4 +983,4 @@
     w_frame, s_frame = new_frame(bytes)
     s_frame.store_w_receiver(w_frame)
     s_frame.push(w_frame)
-    py.test.raises(interpreter.StackOverflow, step_in_interp, s_frame)
+    py.test.raises(interpreter.SenderChainManipulation, step_in_interp, s_frame)

From noreply at buildbot.pypy.org  Fri Jul 18 14:09:01 2014
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Fri, 18 Jul 2014 14:09:01 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk storage: Printing stack trace in
	non-translated mode
Message-ID: <20140718120901.794501C01D2@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: storage
Changeset: r918:2a1d5616e87f
Date: 2014-07-18 13:38 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/2a1d5616e87f/

Log:	Printing stack trace in non-translated mode

diff --git a/targetimageloadingsmalltalk.py b/targetimageloadingsmalltalk.py
--- a/targetimageloadingsmalltalk.py
+++ b/targetimageloadingsmalltalk.py
@@ -3,7 +3,7 @@
 import os
 
 from rpython.rlib.streamio import open_file_as_stream
-from rpython.rlib import jit, rpath
+from rpython.rlib import jit, rpath, objectmodel
 
 from spyvm import model, interpreter, squeakimage, objspace, wrapper,\
     error, shadow, storage_logger, constants
@@ -72,6 +72,9 @@
         return -1
     except Exception, e:
         print_error("Exception: %s" % str(e))
+        if not objectmodel.we_are_translated():
+            import traceback
+            traceback.print_exc()
         return -1
 
 def entry_point(argv):

From noreply at buildbot.pypy.org  Fri Jul 18 14:09:02 2014
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Fri, 18 Jul 2014 14:09:02 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk storage: Merged.
Message-ID: <20140718120902.C6CFC1C01D2@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: storage
Changeset: r919:8c1727512fd2
Date: 2014-07-18 13:50 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/8c1727512fd2/

Log:	Merged. Removed configurable trace depth (unnecessary).

diff --git a/spyvm/interpreter.py b/spyvm/interpreter.py
--- a/spyvm/interpreter.py
+++ b/spyvm/interpreter.py
@@ -35,7 +35,7 @@
     )
 
     def __init__(self, space, image=None, image_name="",
-                trace_depth=-1, evented=True, interrupts=True):
+                trace=False, evented=True, interrupts=True):
         # === Initialize immutable variables
         self.space = space
         self.image = image
@@ -54,7 +54,7 @@
         # === Initialize mutable variables
         self.interrupt_check_counter = self.interrupt_counter_size
         self.next_wakeup_tick = 0
-        self.trace_depth = trace_depth
+        self.trace = trace
         self.trace_proxy = objspace.ConstantFlag()
         self.stack_depth = 0
 
@@ -230,21 +230,13 @@
         return s_frame
         
     # ============== Methods for tracing, printing and debugging ==============
-
-    def activate_trace(self, trace_depth=0):
-        self.trace_depth = trace_depth
-        
-    def deactivate_trace(self):
-        self.trace_depth = -1
-        
+    
     def is_tracing(self):
-        return jit.promote(self.trace_depth) >= 0
-        
+        return jit.promote(self.trace)
+    
     def print_padded(self, str):
-        depth = jit.promote(self.trace_depth)
-        assert depth >= 0
-        if self.stack_depth <= depth:
-            print (' ' * self.stack_depth) + str
+        assert self.is_tracing()
+        print (' ' * self.stack_depth) + str
     
     def activate_debug_bytecode(self):
         "NOT_RPYTHON"
diff --git a/spyvm/plugins/vmdebugging.py b/spyvm/plugins/vmdebugging.py
--- a/spyvm/plugins/vmdebugging.py
+++ b/spyvm/plugins/vmdebugging.py
@@ -10,12 +10,12 @@
 
 @DebuggingPlugin.expose_primitive(unwrap_spec=[object])
 def trace(interp, s_frame, w_rcvr):
-    interp.activate_trace()
+    interp.trace = True
     return w_rcvr
 
 @DebuggingPlugin.expose_primitive(unwrap_spec=[object])
 def untrace(interp, s_frame, w_rcvr):
-    interp.deactivate_trace()
+    interp.trace = False
     return w_rcvr
 
 @DebuggingPlugin.expose_primitive(unwrap_spec=[object])
diff --git a/spyvm/test/test_largeinteger.py b/spyvm/test/test_largeinteger.py
--- a/spyvm/test/test_largeinteger.py
+++ b/spyvm/test/test_largeinteger.py
@@ -8,7 +8,7 @@
     space, interp, _, _ = read_image('bootstrapped.image')
     w = space.w
     copy_to_module(locals(), __name__)
-    interp.deactivate_trace()
+    interp.trace = False
     space.initialize_class(space.w_String, interp)
 
 def teardown_module():
@@ -38,10 +38,7 @@
     except Exception:
         w_selector = find_symbol_in_methoddict_of(selector, w(intmask(candidates[0])).getclass(space).shadow)
     
-    if trace:
-        interp.activate_trace()
-    else:
-        interp.deactivate_trace()
+    interp.trace = trace
     for i, v in enumerate(candidates):
         x = w_l(v)
         if j is None:
@@ -53,7 +50,7 @@
                 y = w_l(j)
         z = perform_primitive(x, w_selector, y)
         assert r_uint(z.value) == r_uint(operation(v, y.value))
-    interp.deactivate_trace()
+    interp.trace = False
 
 def test_bitAnd():
     do_primitive("bitAnd:", operator.and_)
diff --git a/targetimageloadingsmalltalk.py b/targetimageloadingsmalltalk.py
--- a/targetimageloadingsmalltalk.py
+++ b/targetimageloadingsmalltalk.py
@@ -12,7 +12,7 @@
 
 def _usage(argv):
     print """
-    Usage: %s  [-r|-m|-h] [-naPu] [-jpiS] [-tTlLE]
+    Usage: %s  [-r|-m|-h] [-naPu] [-jpiS] [-tlLE]
              - image path (default: Squeak.image)
 
           Execution mode:
@@ -39,7 +39,6 @@
             
           Logging parameters:
             -t|--trace                 - Output a trace of each message, primitive, return value and process switch.
-            -T                  - Like -t, but limit the stack depth for the trace to .
             -l|--storage-log           - Output a log of storage operations.
             -L|--storage-log-aggregate - Output an aggregated storage log at the end of execution.
             -E|--storage-log-elements  - Include classnames of elements into the storage log.
@@ -89,7 +88,7 @@
     # == Other parameters
     poll = False
     interrupts = True
-    trace_depth = -1
+    trace = False
     
     space = prebuilt_space
     idx = 1
@@ -109,11 +108,7 @@
             elif arg in ["-m", "--method"]:
                 selector, idx = get_parameter(argv, idx, arg)
             elif arg in ["-t", "--trace"]:
-                trace_depth = sys.maxint
-            elif arg in ["-T"]:
-                trace_depth, idx = get_int_parameter(argv, idx, arg)
-                if trace_depth < 0:
-                    raise error.Exit("Need argument >= 0 for -T.")
+                trace = True
             elif arg in ["-p", "--poll"]:
                 poll = True
             elif arg in ["-a", "--arg"]:
@@ -164,7 +159,7 @@
     image_reader = squeakimage.reader_for_image(space, squeakimage.Stream(data=imagedata))
     image = create_image(space, image_reader)
     interp = interpreter.Interpreter(space, image, image_name=path,
-                trace_depth=trace_depth, evented=not poll,
+                trace=trace, evented=not poll,
                 interrupts=interrupts)
     space.runtime_setup(argv[0])
     print_error("") # Line break after image-loading characters

From noreply at buildbot.pypy.org  Sat Jul 19 13:57:30 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Sat, 19 Jul 2014 13:57:30 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: Copy and adapt the talk from
 pycon-italy-2014 (thanks antocuni)
Message-ID: <20140719115730.1810B1D2317@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: extradoc
Changeset: r5362:3dd14da26db0
Date: 2014-07-19 13:57 +0200
http://bitbucket.org/pypy/extradoc/changeset/3dd14da26db0/

Log:	Copy and adapt the talk from pycon-italy-2014 (thanks antocuni)

diff --git a/talk/pycon-italy-2014/Makefile b/talk/ep2014/status/Makefile
copy from talk/pycon-italy-2014/Makefile
copy to talk/ep2014/status/Makefile
--- a/talk/pycon-italy-2014/Makefile
+++ b/talk/ep2014/status/Makefile
@@ -1,10 +1,10 @@
 # you can find rst2beamer.py here:
-# http://codespeak.net/svn/user/antocuni/bin/rst2beamer.py
+# https://bitbucket.org/antocuni/env/raw/default/bin/rst2beamer.py
 
 # WARNING: to work, it needs this patch for docutils
 # https://sourceforge.net/tracker/?func=detail&atid=422032&aid=1459707&group_id=38414
 
-talk.pdf: talk.rst author.latex title.latex stylesheet.latex
+talk.pdf: talk.rst author.latex stylesheet.latex
 	python `which rst2beamer.py` --stylesheet=stylesheet.latex --documentoptions=14pt talk.rst talk.latex || exit
 	#/home/antocuni/.virtualenvs/rst2beamer/bin/python `which rst2beamer.py` --stylesheet=stylesheet.latex --documentoptions=14pt talk.rst talk.latex || exit
 	sed 's/\\date{}/\\input{author.latex}/' -i talk.latex || exit
diff --git a/talk/pycon-italy-2014/author.latex b/talk/ep2014/status/author.latex
copy from talk/pycon-italy-2014/author.latex
copy to talk/ep2014/status/author.latex
--- a/talk/pycon-italy-2014/author.latex
+++ b/talk/ep2014/status/author.latex
@@ -1,8 +1,10 @@
 \definecolor{rrblitbackground}{rgb}{0.0, 0.0, 0.0}
 
-\title[PyPy Status]{PyPy Status\\\small{(no, PyPy is not dead)}}
-\author[antocuni]
-{Antonio Cuni}
+\title[PyPy Status Talk]{PyPy Status Talk\\\small{(no no, PyPy is not dead)}}
+\author[arigo, rguillebert]
+{Armin Rigo, Romain Guillebert\\
+based on a PyCon Italia talk by Antonio Cuni\\
+\includegraphics[width=80px]{../../img/py-web-new.png}}
 
-\institute{PyCon Cinque}
-\date{May 24, 2014}
+\institute{EuroPython}
+\date{July 22, 2014}
diff --git a/talk/pycon-italy-2014/beamerdefs.txt b/talk/ep2014/status/beamerdefs.txt
copy from talk/pycon-italy-2014/beamerdefs.txt
copy to talk/ep2014/status/beamerdefs.txt
diff --git a/talk/ep2014/status/speed.png b/talk/ep2014/status/speed.png
new file mode 100644
index 0000000000000000000000000000000000000000..33fe20ac9d81ddbd3ced48f52f9717693dc15518
GIT binary patch

[cut]

diff --git a/talk/pycon-italy-2014/stylesheet.latex b/talk/ep2014/status/stylesheet.latex
copy from talk/pycon-italy-2014/stylesheet.latex
copy to talk/ep2014/status/stylesheet.latex
diff --git a/talk/ep2014/status/talk.pdf b/talk/ep2014/status/talk.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..f9532b263b7bf1aa53ba0d807a875af9991c6a39
GIT binary patch

[cut]

diff --git a/talk/pycon-italy-2014/talk.pdf.info b/talk/ep2014/status/talk.pdf.info
copy from talk/pycon-italy-2014/talk.pdf.info
copy to talk/ep2014/status/talk.pdf.info
diff --git a/talk/pycon-italy-2014/talk.rst b/talk/ep2014/status/talk.rst
copy from talk/pycon-italy-2014/talk.rst
copy to talk/ep2014/status/talk.rst
--- a/talk/pycon-italy-2014/talk.rst
+++ b/talk/ep2014/status/talk.rst
@@ -4,18 +4,6 @@
 PyPy Status
 ================================
 
-About me
----------
-
-- PyPy core dev
-
-- ``pdb++``, ``fancycompleter``, ...
-
-- Consultant, trainer
-
-- http://antocuni.eu
-
-
 PyPy is not dead
 ----------------
 
@@ -114,6 +102,8 @@
 
   * ~7x faster than standard PHP
 
+  * comparable speed as HHVM
+
   * http://hippyvm.com/
 
 
@@ -121,18 +111,27 @@
 Fundraising campaign
 ---------------------
 
-- py3k: 50'852 $ of 105'000 $ (48.4%)
+- py3k: 52'000 $ of 105'000 $ (50%)
 
-- numpy: 48'121 $ of 60'000 $ (80.2%)
+- numpy: 48'000 $ of 60'000 $ (80%)
 
 - STM, 1st call: 25'000 $
 
-- STM, 2nd call: 2'097 $ of 80'000 $ (2.6%)
-
-  * more on STM later
+- STM, 2nd call: 3'000 $ of 80'000 $ (4%)
 
 - thank to all donors!
 
+Commercial support
+------------------
+
+- We offer commercial support for PyPy
+
+- Consultancy and training
+
+- Performance issues for open- or closed-source programs, porting,
+  improving support in parts of the Python or non-Python interpreters,
+  etc.
+
 Current status
 ---------------
 
@@ -155,11 +154,11 @@
 - numpy: in-progress (more later)
 
 
-Speed: 6.3x faster than CPython
+Speed: 6.5x faster than CPython
 --------------------------------
 
 .. image:: speed.png
-   :scale: 47%
+   :scale: 44%
 
 
 ARM
@@ -252,92 +251,19 @@
 
   * by Armin Rigo and Remi Meier
 
-STM semantics
--------------
-
-- N threads
-
-- Each thread split into atomic blocks
-
-- Sequential execution in some arbitrary order
-
-- In practice:
-
-- Parallel execution, conflicts solved by STM
-
-
-Unit of execution (1)
----------------------
-
-- Atomic blocks == 1 Python bytecode
-
-- Threads are executed in arbitrary order, but bytecodes are atomic
-
-- ==> Same semantics as GIL
-
-- "and this will solve the GIL problem" (A. Rigo, EuroPython 2011 lighting talk)
-
-Unit of execution (2)
+Current status for STM
 ----------------------
 
-- Larger atomic blocks
-
-- ``with atomic:``
-
-- Much easier to use than explicit locks
-
-- Can be hidden by libraries to provide even higher level paradigms
-
-  * e.g.: Twisted apps made parallel out of the box
-
-Race conditions
----------------
-
-- They don't magically disappear
-
-- With explicit locks
-
-  * ==> BOOM
-
-  * you fix bugs by preventing race conditions
-
-- With atomic blocks
-
-  * ==> Rollaback
-
-  * Performance penalty
-
-  * You optimize by preventing race conditions
-
-- Fast&broken vs. Slower&correct
-
-
-Implementation
----------------
-
-- Conflicts detection, commit and rollaback is costly
-
-- Original goal (2011): 2x-5x slower than PyPy without STM
-
-  * But parallelizable!
-
-|pause|
-
-- Current goal (2014): 25% slower than PyPy without STM
-
-- Yes, that's 10x less overhead than original goal
-
-- mmap black magic
-
-Current status
----------------
-
 - Preliminary versions of pypy-jit-stm available
 
-- The JIT overhead is still a bit too high
+- The overhead is still a bit too high and hard to precict
 
 - Lots of polishing needed
 
+- More fundamentally, how to best use it is still unknown
+
+- See talk tomorrow
+
 
 Contacts, Q&A
 --------------
@@ -346,13 +272,4 @@
 
 - http://morepypy.blogspot.com/
 
-- twitter: @antocuni
-
-- Available for consultancy & training:
-
-  * http://antocuni.eu
-
-  * info at antocuni.eu
-
 - Any question?
-

From noreply at buildbot.pypy.org  Sat Jul 19 14:13:38 2014
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Sat, 19 Jul 2014 14:13:38 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk storage: Refactoring,
	reduced code duplication.
Message-ID: <20140719121338.9FDEE1C104D@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: storage
Changeset: r920:2dd2b3555772
Date: 2014-07-18 14:21 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/2dd2b3555772/

Log:	Refactoring, reduced code duplication.

diff --git a/spyvm/interpreter.py b/spyvm/interpreter.py
--- a/spyvm/interpreter.py
+++ b/spyvm/interpreter.py
@@ -66,13 +66,9 @@
             try:
                 self.loop_bytecodes(s_new_context)
                 raise Exception("loop_bytecodes left without raising...")
-            except StackOverflow, e:
+            except ContextSwitchException, e:
                 if self.is_tracing():
-                    print "====== StackOverflow, contexts forced to heap at: %s" % e.s_new_context.short_str()
-                s_new_context = e.s_new_context
-            except SenderChainManipulation, e:
-                if self.is_tracing():
-                    print "====== SenderChainManipulation, contexts forced to heap at: %s" % e.s_new_context.short_str()
+                    e.print_trace(s_new_context)
                 s_new_context = e.s_new_context
             except Return, nlr:
                 assert nlr.s_target_context or nlr.is_local
@@ -83,13 +79,7 @@
                         s_new_context._activate_unwind_context(self)
                         s_new_context = s_sender
                 s_new_context.push(nlr.value)
-            except ProcessSwitch, p:
-                assert not self.space.suppress_process_switch.is_set(), "ProcessSwitch should be disabled..."
-                if self.is_tracing():
-                    print "====== Switched process from: %s" % s_new_context.short_str()
-                    print "====== to: %s " % p.s_new_context.short_str()
-                s_new_context = p.s_new_context
-
+    
     def loop_bytecodes(self, s_context, may_context_switch=True):
         old_pc = 0
         if not jit.we_are_jitted() and may_context_switch:
@@ -267,23 +257,34 @@
 class ContextSwitchException(Exception):
     """General Exception that causes the interpreter to leave
     the current context."""
+    
     _attrs_ = ["s_new_context"]
+    type = "ContextSwitch"
     def __init__(self, s_new_context):
         self.s_new_context = s_new_context
-
+    
+    def print_trace(self, old_context):
+        print "====== %s, contexts forced to heap at: %s" % (self.type, self.s_new_context.short_str())
+        
 class StackOverflow(ContextSwitchException):
     """This causes the current jit-loop to be left, dumping all virtualized objects to the heap.
     This breaks performance, so it should rarely happen.
     In case of severe performance problems, execute with -t and check if this occurrs."""
-
+    type = "Stack Overflow"
+    
 class ProcessSwitch(ContextSwitchException):
     """This causes the interpreter to switch the executed context.
     Triggered when switching the process."""
-
+    
+    def print_trace(self, old_context):
+        print "====== Switched process from: %s" % old_context.short_str()
+        print "====== to: %s " % self.s_new_context.short_str()
+    
 class SenderChainManipulation(ContextSwitchException):
     """Manipulation of the sender chain can invalidate the jitted C stack.
     We have to dump all virtual objects and rebuild the stack.
     We try to raise this as rarely as possible and as late as possible."""
+    type = "Sender Manipulation"
 
 import rpython.rlib.unroll
 if hasattr(unroll, "unrolling_zero"):

From noreply at buildbot.pypy.org  Sat Jul 19 14:13:40 2014
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Sat, 19 Jul 2014 14:13:40 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk storage-context-state: Delaying
 SenderChainManipulation as much as possible.
Message-ID: <20140719121340.006541C104D@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: storage-context-state
Changeset: r921:76c81645836d
Date: 2014-07-18 15:29 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/76c81645836d/

Log:	Delaying SenderChainManipulation as much as possible. Added a
	'state' field to Context objects, can be Inactive, Active or Dirty.
	Setting the sender of an Active context makes it Dirty. When a Dirty
	context is left, SenderChainManipulation will be raised, forcing all
	remaining contexts from the stack to the heap.

diff --git a/spyvm/interpreter.py b/spyvm/interpreter.py
--- a/spyvm/interpreter.py
+++ b/spyvm/interpreter.py
@@ -1,6 +1,6 @@
 import py
 import os
-from spyvm.shadow import ContextPartShadow, MethodContextShadow, BlockContextShadow, MethodNotFound
+from spyvm.shadow import ContextPartShadow, MethodContextShadow, BlockContextShadow, MethodNotFound, ActiveContext, InactiveContext, DirtyContext
 from spyvm import model, constants, primitives, conftest, wrapper, objspace
 from spyvm.tool.bitmanipulation import splitter
 
@@ -118,8 +118,10 @@
             if self.is_tracing():
                 self.stack_depth += 1
             if s_frame._s_sender is None and s_sender is not None:
-                s_frame.store_s_sender(s_sender, raise_error=False)
+                s_frame.store_s_sender(s_sender)
             # Now (continue to) execute the context bytecodes
+            assert s_frame.state is InactiveContext
+            s_frame.state = ActiveContext
             self.loop_bytecodes(s_frame, may_context_switch)
         except rstackovf.StackOverflow:
             rstackovf.check_stack_overflow()
@@ -127,7 +129,11 @@
         finally:
             if self.is_tracing():
                 self.stack_depth -= 1
-
+            dirty_frame = s_frame.state is DirtyContext
+            s_frame.state = InactiveContext
+            if dirty_frame:
+                raise SenderChainManipulation(s_frame)
+    
     def step(self, context):
         bytecode = context.fetch_next_bytecode()
         for entry in UNROLLING_BYTECODE_RANGES:
@@ -755,16 +761,9 @@
             association = wrapper.AssociationWrapper(self.space, w_association)
             self.push(association.value())
         elif opType == 5:
-            # TODO - the following two special cases should not be necessary
-            try:
-                self.w_receiver().store(self.space, third, self.top())
-            except SenderChainManipulation, e:
-                raise SenderChainManipulation(self)
+            self.w_receiver().store(self.space, third, self.top())
         elif opType == 6:
-            try:
-                self.w_receiver().store(self.space, third, self.pop())
-            except SenderChainManipulation, e:
-                raise SenderChainManipulation(self)
+            self.w_receiver().store(self.space, third, self.pop())
         elif opType == 7:
             w_association = self.w_method().getliteral(third)
             association = wrapper.AssociationWrapper(self.space, w_association)
diff --git a/spyvm/shadow.py b/spyvm/shadow.py
--- a/spyvm/shadow.py
+++ b/spyvm/shadow.py
@@ -636,18 +636,29 @@
     def size(self):
         return self._w_self_size
 
+class ContextState(object):
+    def __init__(self, name):
+        self.name = name
+    def __str__(self):
+        return self.name
+    def __repr__(self):
+        return self.name
+InactiveContext = ContextState("InactiveContext")
+ActiveContext = ContextState("ActiveContext")
+DirtyContext = ContextState("DirtyContext")
+
 class ContextPartShadow(AbstractRedirectingShadow):
 
     __metaclass__ = extendabletype
     _attrs_ = ['_s_sender',
             '_pc', '_temps_and_stack',
-            '_stack_ptr', 'instances_w']
+            '_stack_ptr', 'instances_w', 'state']
     repr_classname = "ContextPartShadow"
 
     _virtualizable_ = [
         '_s_sender',
         "_pc", "_temps_and_stack[*]", "_stack_ptr",
-        "_w_self", "_w_self_size"
+        "_w_self", "_w_self_size", 'state'
     ]
 
     # ______________________________________________________________________
@@ -657,13 +668,7 @@
         self._s_sender = None
         AbstractRedirectingShadow.__init__(self, space, w_self, size)
         self.instances_w = {}
-
-    def copy_field_from(self, n0, other_shadow):
-        from spyvm.interpreter import SenderChainManipulation
-        try:
-            AbstractRedirectingShadow.copy_field_from(self, n0, other_shadow)
-        except SenderChainManipulation, e:
-            assert e.s_new_context == self
+        self.state = InactiveContext
 
     def copy_from(self, other_shadow):
         # Some fields have to be initialized before the rest, to ensure correct initialization.
@@ -705,7 +710,7 @@
         if n0 == constants.CTXPART_SENDER_INDEX:
             assert isinstance(w_value, model.W_PointersObject)
             if w_value.is_nil(self.space):
-                self.store_s_sender(None, raise_error=False)
+                self.store_s_sender(None)
             else:
                 self.store_s_sender(w_value.as_context_get_shadow(self.space))
             return
@@ -725,12 +730,12 @@
 
     # === Sender ===
 
-    def store_s_sender(self, s_sender, raise_error=True):
+    def store_s_sender(self, s_sender):
         if s_sender is not self._s_sender:
             self._s_sender = s_sender
-            if raise_error:
-                from spyvm.interpreter import SenderChainManipulation
-                raise SenderChainManipulation(self)
+            # If new sender is None, we are just being marked as returned.
+            if s_sender is not None and self.state is ActiveContext:
+                self.state = DirtyContext
 
     def w_sender(self):
         sender = self.s_sender()
@@ -822,7 +827,7 @@
 
     def mark_returned(self):
         self.store_pc(-1)
-        self.store_s_sender(None, raise_error=False)
+        self.store_s_sender(None)
 
     def is_returned(self):
         return self.pc() == -1 and self.w_sender().is_nil(self.space)

From noreply at buildbot.pypy.org  Sat Jul 19 14:13:41 2014
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Sat, 19 Jul 2014 14:13:41 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk storage-context-state: Fixed tests,
	added tests.
Message-ID: <20140719121341.2E00F1C104D@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: storage-context-state
Changeset: r922:2d1854d40231
Date: 2014-07-18 15:29 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/2d1854d40231/

Log:	Fixed tests, added tests.

diff --git a/spyvm/test/test_interpreter.py b/spyvm/test/test_interpreter.py
--- a/spyvm/test/test_interpreter.py
+++ b/spyvm/test/test_interpreter.py
@@ -978,9 +978,32 @@
             2, "value:value:"]],
         test)
 
-def test_c_stack_reset_on_sender_chain_manipulation():
-    bytes = reduce(operator.add, map(chr, [0x84, 0xc0, 0x00]))
+def test_frame_dirty_if_active():
+    bytes = reduce(operator.add, map(chr, [0x84, 0xc0, 0x00])) + returnReceiverBytecode
     w_frame, s_frame = new_frame(bytes)
     s_frame.store_w_receiver(w_frame)
     s_frame.push(w_frame)
-    py.test.raises(interpreter.SenderChainManipulation, step_in_interp, s_frame)
+    s_frame.state = shadow.ActiveContext
+    step_in_interp(s_frame)
+    assert s_frame.state is shadow.DirtyContext
+
+def test_frame_not_dirty_if_inactive():
+    bytes = reduce(operator.add, map(chr, [0x84, 0xc0, 0x00])) + returnReceiverBytecode
+    w_frame, s_frame = new_frame(bytes)
+    w_other_frame, s_other_frame = new_frame("")
+    s_frame.store_w_receiver(w_other_frame)
+    s_frame.push(w_frame)
+    s_frame.state = shadow.ActiveContext
+    step_in_interp(s_frame)
+    assert s_frame.state is shadow.ActiveContext
+    assert s_other_frame.state is shadow.InactiveContext
+    
+def test_raise_SenderManipulation_on_dirty_frame():
+    w_frame, s_frame = new_frame(returnReceiverBytecode)
+    s_frame.state = shadow.DirtyContext
+    def run_frame():
+        #import pdb; pdb.set_trace()
+        interp._loop = True
+        interp.stack_frame(s_frame, None)
+    py.test.raises(interpreter.SenderChainManipulation, run_frame)
+    
\ No newline at end of file
diff --git a/spyvm/test/test_zin_squeak_4_5_image.py b/spyvm/test/test_zin_squeak_4_5_image.py
--- a/spyvm/test/test_zin_squeak_4_5_image.py
+++ b/spyvm/test/test_zin_squeak_4_5_image.py
@@ -43,7 +43,7 @@
     # create a frame for our newly crafted method with a valid sender (to avoid raising returnFromTop to early)
     s_initial_frame = create_method(chr(0x7c)).create_frame(space, w(0), [])
     s_frame = w_method.create_frame(space, w(0))
-    s_frame.store_s_sender(s_initial_frame, raise_error=False)
+    s_frame.store_s_sender(s_initial_frame)
     
     try:
         interp.loop(s_frame.w_self())
@@ -70,7 +70,7 @@
     # create a frame for our newly crafted method with a valid sender (to avoid raising returnFromTop to early)
     s_initial_frame = create_method(chr(0x7c)).create_frame(space, w(0))
     s_frame = w_method.create_frame(space, w(0))
-    s_frame.store_s_sender(s_initial_frame, raise_error=False)
+    s_frame.store_s_sender(s_initial_frame)
     
     try:
         interp.loop(s_frame.w_self())
diff --git a/spyvm/test/util.py b/spyvm/test/util.py
--- a/spyvm/test/util.py
+++ b/spyvm/test/util.py
@@ -85,7 +85,7 @@
         if not self._loop:
             # this test is done to not loop in test, but rather step just once where wanted
             # Unfortunately, we have to mimick some of the original behaviour.
-            s_new_frame.store_s_sender(s_sender, raise_error=False)
+            s_new_frame.store_s_sender(s_sender)
             return s_new_frame
         return interpreter.Interpreter.stack_frame(self, s_new_frame, s_sender, may_context_switch)
 

From noreply at buildbot.pypy.org  Sat Jul 19 14:13:42 2014
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Sat, 19 Jul 2014 14:13:42 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk storage-context-state: Refactored
 Return-mechanism to make the context state and late
 sender-chain-manipulation refactoring work.
Message-ID: <20140719121342.6434F1C104D@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: storage-context-state
Changeset: r923:a27bd7d3d458
Date: 2014-07-19 13:15 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/a27bd7d3d458/

Log:	Refactored Return-mechanism to make the context state and late
	sender-chain-manipulation refactoring work. Added LocalReturn class.
	stack_frame() handles dispatching of Return types, always invoked
	together with loop_bytecodes now.

diff --git a/spyvm/interpreter.py b/spyvm/interpreter.py
--- a/spyvm/interpreter.py
+++ b/spyvm/interpreter.py
@@ -60,25 +60,58 @@
 
     def loop(self, w_active_context):
         # This is the top-level loop and is not invoked recursively.
-        s_new_context = w_active_context.as_context_get_shadow(self.space)
+        s_context = w_active_context.as_context_get_shadow(self.space)
         while True:
-            s_sender = s_new_context.s_sender()
+            s_sender = s_context.s_sender()
             try:
-                self.loop_bytecodes(s_new_context)
+                self.stack_frame(s_context, None)
                 raise Exception("loop_bytecodes left without raising...")
             except ContextSwitchException, e:
                 if self.is_tracing():
-                    e.print_trace(s_new_context)
-                s_new_context = e.s_new_context
-            except Return, nlr:
-                assert nlr.s_target_context or nlr.is_local
-                s_new_context = s_sender
-                if not nlr.is_local:
-                    while s_new_context is not nlr.s_target_context:
-                        s_sender = s_new_context.s_sender()
-                        s_new_context._activate_unwind_context(self)
-                        s_new_context = s_sender
-                s_new_context.push(nlr.value)
+                    e.print_trace()
+                s_context = e.s_new_context
+            except LocalReturn, ret:
+                s_context = self.unwind_context_chain(s_sender, s_sender, ret.value)
+            except Return, ret:
+                s_context = self.unwind_context_chain(s_sender, ret.s_target_context, ret.value)
+            except NonVirtualReturn, ret:
+                if self.is_tracing():
+                    ret.print_trace()
+                s_context = self.unwind_context_chain(ret.s_current_context, ret.s_target_context, ret.value)
+    
+    # This is a wrapper around loop_bytecodes that cleanly enters/leaves the frame,
+    # handles the stack overflow protection mechanism and handles/dispatches Returns.
+    def stack_frame(self, s_frame, s_sender, may_context_switch=True):
+        try:
+            if self.is_tracing():
+                self.stack_depth += 1
+            if s_frame._s_sender is None and s_sender is not None:
+                s_frame.store_s_sender(s_sender)
+            # Now (continue to) execute the context bytecodes
+            assert s_frame.state is InactiveContext
+            s_frame.state = ActiveContext
+            self.loop_bytecodes(s_frame, may_context_switch)
+        except rstackovf.StackOverflow:
+            rstackovf.check_stack_overflow()
+            raise StackOverflow(s_frame)
+        except Return, e:
+            # Do not catch NonVirtualReturn. If there are multiple dirty contexts
+            # on the stack, the inner-most one will count.
+            if s_frame.state is DirtyContext:
+                s_sender = s_frame.s_sender()
+                s_frame._activate_unwind_context(self)
+                target_context = s_sender if e.is_local else e.s_target_context
+                raise NonVirtualReturn(target_context, e.value, s_sender)
+            else:
+                s_frame._activate_unwind_context(self)
+                if e.s_target_context is s_sender or e.is_local:
+                    raise LocalReturn(e.value)
+                else:
+                    raise e
+        finally:
+            if self.is_tracing():
+                self.stack_depth -= 1
+            s_frame.state = InactiveContext
     
     def loop_bytecodes(self, s_context, may_context_switch=True):
         old_pc = 0
@@ -100,39 +133,22 @@
                 s_context=s_context)
             try:
                 self.step(s_context)
-            except Return, nlr:
-                if nlr.s_target_context is s_context or nlr.is_local:
-                    s_context.push(nlr.value)
-                else:
-                    if nlr.s_target_context is None:
-                        # This is the case where we are returning to our sender.
-                        # Mark the return as local, so our sender will take it
-                        nlr.is_local = True
-                    s_context._activate_unwind_context(self)
-                    raise nlr
-
-    # This is a wrapper around loop_bytecodes that cleanly enters/leaves the frame
-    # and handles the stack overflow protection mechanism.
-    def stack_frame(self, s_frame, s_sender, may_context_switch=True):
-        try:
-            if self.is_tracing():
-                self.stack_depth += 1
-            if s_frame._s_sender is None and s_sender is not None:
-                s_frame.store_s_sender(s_sender)
-            # Now (continue to) execute the context bytecodes
-            assert s_frame.state is InactiveContext
-            s_frame.state = ActiveContext
-            self.loop_bytecodes(s_frame, may_context_switch)
-        except rstackovf.StackOverflow:
-            rstackovf.check_stack_overflow()
-            raise StackOverflow(s_frame)
-        finally:
-            if self.is_tracing():
-                self.stack_depth -= 1
-            dirty_frame = s_frame.state is DirtyContext
-            s_frame.state = InactiveContext
-            if dirty_frame:
-                raise SenderChainManipulation(s_frame)
+            except LocalReturn, ret:
+                s_context.push(ret.value)
+    
+    def unwind_context_chain(self, start_context, target_context, return_value):
+        if start_context is None:
+            # This is the toplevel frame. Execution ended.
+            raise ReturnFromTopLevel(return_value)
+        assert target_context
+        context = start_context
+        while context is not target_context:
+            assert context, "Sender chain ended without finding return-context."
+            s_sender = context.s_sender()
+            context._activate_unwind_context(self)
+            context = s_sender
+        context.push(return_value)
+        return context
     
     def step(self, context):
         bytecode = context.fetch_next_bytecode()
@@ -253,44 +269,60 @@
     def __init__(self, object):
         self.object = object
 
-class Return(Exception):
-    _attrs_ = ["value", "s_target_context", "is_local"]
+class LocalReturn(Exception):
+    _attrs_ = ["value"]
+    def __init__(self, value):
+        self.value = value
+
+class AbstractReturn(Exception):
+    _attrs_ = ["value", "s_target_context"]
     def __init__(self, s_target_context, w_result):
         self.value = w_result
         self.s_target_context = s_target_context
-        self.is_local = False
+
+class Return(AbstractReturn):
+    """This is the basic Return, handled on the C-stack,
+    without forcing all contexts to the heap."""
+    _attrs_ = ["is_local"]
+    def __init__(self, s_target_context, w_result, is_local):
+        AbstractReturn.__init__(self, s_target_context, w_result)
+        self.is_local = is_local
+    
+class NonVirtualReturn(AbstractReturn):
+    """This Return will be passed through the entire C-stack built from stack_frame()
+    invokations. Used when the sender-chain has been manipulated.
+    s_current_context is the context where the outermost loop()
+    will start unrolling the context-chain and looking for s_target_context."""
+    
+    _attrs_ = ["s_current_context"]
+    def __init__(self, s_target_context, w_result, s_current_context):
+        AbstractReturn.__init__(self, s_target_context, w_result)
+        self.s_current_context = s_current_context
+    
+    def print_trace(self):
+        print "====== Sender Chain Manipulation, contexts forced to heap at: %s" % self.s_current_context.short_str()
 
 class ContextSwitchException(Exception):
     """General Exception that causes the interpreter to leave
     the current context."""
-    
     _attrs_ = ["s_new_context"]
-    type = "ContextSwitch"
     def __init__(self, s_new_context):
         self.s_new_context = s_new_context
-    
-    def print_trace(self, old_context):
-        print "====== %s, contexts forced to heap at: %s" % (self.type, self.s_new_context.short_str())
-        
+
 class StackOverflow(ContextSwitchException):
     """This causes the current jit-loop to be left, dumping all virtualized objects to the heap.
     This breaks performance, so it should rarely happen.
     In case of severe performance problems, execute with -t and check if this occurrs."""
-    type = "Stack Overflow"
     
+    def print_trace(self):
+        print "====== Stack Overflow, contexts forced to heap at: %s" % self.s_new_context.short_str()
+
 class ProcessSwitch(ContextSwitchException):
     """This causes the interpreter to switch the executed context.
     Triggered when switching the process."""
     
-    def print_trace(self, old_context):
-        print "====== Switched process from: %s" % old_context.short_str()
-        print "====== to: %s " % self.s_new_context.short_str()
-    
-class SenderChainManipulation(ContextSwitchException):
-    """Manipulation of the sender chain can invalidate the jitted C stack.
-    We have to dump all virtual objects and rebuild the stack.
-    We try to raise this as rarely as possible and as late as possible."""
-    type = "Sender Manipulation"
+    def print_trace(self):
+        print "====== Switched Process to: %s" % self.s_new_context.short_str()
 
 import rpython.rlib.unroll
 if hasattr(unroll, "unrolling_zero"):
@@ -306,7 +338,6 @@
             return unrolling_int(int.__rsub__(self, other))
     unrolling_zero = unrolling_int(0)
 
-
 # This is a decorator for bytecode implementation methods.
 # parameter_bytes=N means N additional bytes are fetched as parameters.
 def bytecode_implementation(parameter_bytes=0):
@@ -685,16 +716,12 @@
             # it will find the sender as a local, and we don't have to
             # force the reference
             s_return_to = None
-            return_from_top = self.s_sender() is None
+            is_local = True
         else:
             s_return_to = self.s_home().s_sender()
-            return_from_top = s_return_to is None
+            is_local = False
         
-        if return_from_top:
-            # This should never happen while executing a normal image.
-            raise ReturnFromTopLevel(return_value)
-        else:
-            raise Return(s_return_to, return_value)
+        raise Return(s_return_to, return_value, is_local)
 
     # ====== Send/Return bytecodes ======
 

From noreply at buildbot.pypy.org  Sat Jul 19 14:13:43 2014
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Sat, 19 Jul 2014 14:13:43 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk storage-context-state: Fixed ensure:
 mechanism. Fixed test.
Message-ID: <20140719121343.94F481C104D@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: storage-context-state
Changeset: r924:f5b3945a1fdc
Date: 2014-07-19 13:48 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/f5b3945a1fdc/

Log:	Fixed ensure: mechanism. Fixed test.

diff --git a/spyvm/interpreter.py b/spyvm/interpreter.py
--- a/spyvm/interpreter.py
+++ b/spyvm/interpreter.py
@@ -819,10 +819,8 @@
             self.push(self.gettemp(0)) # push the first argument
             try:
                 self.bytecodePrimValue(interp, 0)
-            except Return, nlr:
-                assert nlr.s_target_context or nlr.is_local
-                if self is not nlr.s_target_context and not nlr.is_local:
-                    raise nlr
+            except LocalReturn, ret:
+                pass # Local return value of ensure: block is ignored
             finally:
                 self.mark_returned()
 
diff --git a/spyvm/test/test_interpreter.py b/spyvm/test/test_interpreter.py
--- a/spyvm/test/test_interpreter.py
+++ b/spyvm/test/test_interpreter.py
@@ -979,7 +979,7 @@
         test)
 
 def test_frame_dirty_if_active():
-    bytes = reduce(operator.add, map(chr, [0x84, 0xc0, 0x00])) + returnReceiverBytecode
+    bytes = reduce(operator.add, map(chr, [0x84, 0xc0, 0x00]))
     w_frame, s_frame = new_frame(bytes)
     s_frame.store_w_receiver(w_frame)
     s_frame.push(w_frame)
@@ -988,7 +988,7 @@
     assert s_frame.state is shadow.DirtyContext
 
 def test_frame_not_dirty_if_inactive():
-    bytes = reduce(operator.add, map(chr, [0x84, 0xc0, 0x00])) + returnReceiverBytecode
+    bytes = reduce(operator.add, map(chr, [0x84, 0xc0, 0x00]))
     w_frame, s_frame = new_frame(bytes)
     w_other_frame, s_other_frame = new_frame("")
     s_frame.store_w_receiver(w_other_frame)
@@ -998,12 +998,14 @@
     assert s_frame.state is shadow.ActiveContext
     assert s_other_frame.state is shadow.InactiveContext
     
-def test_raise_SenderManipulation_on_dirty_frame():
-    w_frame, s_frame = new_frame(returnReceiverBytecode)
-    s_frame.state = shadow.DirtyContext
-    def run_frame():
-        #import pdb; pdb.set_trace()
-        interp._loop = True
+def test_raise_NonVirtualReturn_on_dirty_frame():
+    bytes = reduce(operator.add, map(chr, [0x84, 0xc0, 0x00])) + returnTopFromMethodBytecode
+    w_frame, s_frame = new_frame(bytes)
+    s_frame.store_w_receiver(w_frame)
+    s_frame.push(w_frame)
+    
+    interp._loop = True
+    def do_test():
         interp.stack_frame(s_frame, None)
-    py.test.raises(interpreter.SenderChainManipulation, run_frame)
+    py.test.raises(interpreter.NonVirtualReturn, do_test)
     
\ No newline at end of file

From noreply at buildbot.pypy.org  Sat Jul 19 14:41:58 2014
From: noreply at buildbot.pypy.org (waedt)
Date: Sat, 19 Jul 2014 14:41:58 +0200 (CEST)
Subject: [pypy-commit] pypy utf8-unicode2: Fix _multibytecodec
Message-ID: <20140719124158.EF4F71C0DCA@cobra.cs.uni-duesseldorf.de>

Author: Tyler Wade 
Branch: utf8-unicode2
Changeset: r72468:e70f582fd5dc
Date: 2014-07-17 01:43 -0500
http://bitbucket.org/pypy/pypy/changeset/e70f582fd5dc/

Log:	Fix _multibytecodec

diff --git a/pypy/interpreter/utf8.py b/pypy/interpreter/utf8.py
--- a/pypy/interpreter/utf8.py
+++ b/pypy/interpreter/utf8.py
@@ -2,9 +2,8 @@
 from rpython.rlib.objectmodel import specialize
 from rpython.rlib.runicode import utf8_code_length
 from rpython.rlib.unicodedata import unicodedb_5_2_0 as unicodedb
-from rpython.rlib.rarithmetic import r_uint
-from rpython.rtyper.lltypesystem import rffi
-from rpython.rtyper.lltypesystem import lltype
+from rpython.rlib.rarithmetic import r_uint, intmask
+from rpython.rtyper.lltypesystem import rffi, lltype
 
 wchar_rint = rffi.r_uint
 WCHAR_INTP = rffi.UINTP
@@ -464,7 +463,7 @@
             if rffi.sizeof(rffi.WCHAR_T) == 2:
                 if 0xD800 <= c <= 0xDBFF:
                     i += 1
-                    c2 = int(array[i])
+                    c2 = intmask(array[i])
                     if c2 == 0:
                         builder.append(c)
                         break
@@ -485,7 +484,7 @@
         builder = Utf8Builder()
         i = 0;
         while i < size:
-            c = int(array[i])
+            c = intmask(array[i])
             if c == 0:
                 break
 
@@ -513,7 +512,7 @@
         builder = Utf8Builder()
         i = 0;
         while i < size:
-            c = int(array[i])
+            c = intmask(array[i])
 
             if rffi.sizeof(rffi.WCHAR_T) == 2:
                 if i != size - 1 and 0xD800 <= c <= 0xDBFF:
diff --git a/pypy/module/_multibytecodec/c_codecs.py b/pypy/module/_multibytecodec/c_codecs.py
--- a/pypy/module/_multibytecodec/c_codecs.py
+++ b/pypy/module/_multibytecodec/c_codecs.py
@@ -1,8 +1,9 @@
 import py
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
+from pypy.interpreter.utf8 import Utf8Str
 
-UNICODE_REPLACEMENT_CHARACTER = u'\uFFFD'
+UNICODE_REPLACEMENT_CHARACTER = Utf8Str.from_unicode(u'\uFFFD')
 
 
 class EncodeDecodeError(Exception):
@@ -139,7 +140,7 @@
                                     errorcb, namecb, stringdata)
         src = pypy_cjk_dec_outbuf(decodebuf)
         length = pypy_cjk_dec_outlen(decodebuf)
-        return rffi.wcharpsize2unicode(src, length)
+        return Utf8Str.from_wcharpsize(src, length)
     #
     finally:
         rffi.free_nonmovingbuffer(stringdata, inbuf)
@@ -164,18 +165,18 @@
     if errors == "strict":
         raise EncodeDecodeError(start, end, reason)
     elif errors == "ignore":
-        replace = u""
+        replace = Utf8Str("")
     elif errors == "replace":
         replace = UNICODE_REPLACEMENT_CHARACTER
     else:
         assert errorcb
         replace, end = errorcb(errors, namecb, reason,
                                stringdata, start, end)
-    inbuf = rffi.get_nonmoving_unicodebuffer(replace)
+    inbuf = replace.copy_to_wcharp()
     try:
         r = pypy_cjk_dec_replace_on_error(decodebuf, inbuf, len(replace), end)
     finally:
-        rffi.free_nonmoving_unicodebuffer(replace, inbuf)
+        rffi.free_wcharp(inbuf)
     if r == MBERR_NOMEMORY:
         raise MemoryError
 
@@ -222,7 +223,7 @@
 def encodeex(encodebuf, unicodedata, errors="strict", errorcb=None,
              namecb=None, ignore_error=0):
     inleft = len(unicodedata)
-    inbuf = rffi.get_nonmoving_unicodebuffer(unicodedata)
+    inbuf = unicodedata.copy_to_wcharp()
     try:
         if pypy_cjk_enc_init(encodebuf, inbuf, inleft) < 0:
             raise MemoryError
@@ -247,7 +248,7 @@
         return rffi.charpsize2str(src, length)
     #
     finally:
-        rffi.free_nonmoving_unicodebuffer(unicodedata, inbuf)
+        rffi.free_wcharp(inbuf)
 
 def multibytecodec_encerror(encodebuf, e, errors,
                             errorcb, namecb, unicodedata):
@@ -273,7 +274,7 @@
     elif errors == "replace":
         codec = pypy_cjk_enc_getcodec(encodebuf)
         try:
-            replace = encode(codec, u"?")
+            replace = encode(codec, Utf8Str("?"))
         except EncodeDecodeError:
             replace = "?"
     else:
diff --git a/pypy/module/_multibytecodec/test/test_c_codecs.py b/pypy/module/_multibytecodec/test/test_c_codecs.py
--- a/pypy/module/_multibytecodec/test/test_c_codecs.py
+++ b/pypy/module/_multibytecodec/test/test_c_codecs.py
@@ -1,4 +1,5 @@
 import py
+from pypy.interpreter.utf8 import Utf8Str
 from pypy.module._multibytecodec.c_codecs import getcodec, codecs
 from pypy.module._multibytecodec.c_codecs import decode, encode
 from pypy.module._multibytecodec.c_codecs import EncodeDecodeError
@@ -95,37 +96,38 @@
 
 def test_encode_hz():
     c = getcodec("hz")
-    s = encode(c, u'foobar')
+    s = encode(c, Utf8Str('foobar'))
     assert s == 'foobar' and type(s) is str
-    s = encode(c, u'\u5f95\u6cef')
+    s = encode(c, Utf8Str.from_unicode(u'\u5f95\u6cef'))
     assert s == '~{abc}~}'
 
 def test_encode_hz_error():
     # error
     c = getcodec("hz")
-    e = py.test.raises(EncodeDecodeError, encode, c, u'abc\u1234def').value
+    e = py.test.raises(EncodeDecodeError, encode, c,
+                       Utf8Str.from_unicode(u'abc\u1234def')).value
     assert e.start == 3
     assert e.end == 4
     assert e.reason == "illegal multibyte sequence"
 
 def test_encode_hz_ignore():
     c = getcodec("hz")
-    s = encode(c, u'abc\u1234def', 'ignore')
+    s = encode(c, Utf8Str.from_unicode(u'abc\u1234def'), 'ignore')
     assert s == 'abcdef'
 
 def test_encode_hz_replace():
     c = getcodec("hz")
-    s = encode(c, u'abc\u1234def', 'replace')
+    s = encode(c, Utf8Str.from_unicode(u'abc\u1234def'), 'replace')
     assert s == 'abc?def'
 
 def test_encode_jisx0208():
     c = getcodec('iso2022_jp')
-    s = encode(c, u'\u83ca\u5730\u6642\u592b')
+    s = encode(c, Utf8Str.from_unicode(u'\u83ca\u5730\u6642\u592b'))
     assert s == '\x1b$B5FCO;~IW\x1b(B' and type(s) is str
 
 def test_encode_custom_error_handler_bytes():
     c = getcodec("hz")
     def errorhandler(errors, enc, msg, t, startingpos, endingpos):
         return None, '\xc3', endingpos
-    s = encode(c, u'abc\u1234def', 'foo', errorhandler)
+    s = encode(c, Utf8Str.from_unicode(u'abc\u1234def'), 'foo', errorhandler)
     assert '\xc3' in s
diff --git a/pypy/module/sys/vm.py b/pypy/module/sys/vm.py
--- a/pypy/module/sys/vm.py
+++ b/pypy/module/sys/vm.py
@@ -3,11 +3,11 @@
 """
 
 from rpython.rlib import jit
-from rpython.rlib.runicode import MAXUNICODE
 
 from pypy.interpreter import gateway
 from pypy.interpreter.error import OperationError
 from pypy.interpreter.gateway import unwrap_spec
+from pypy.interpreter.utf8_codecs import MAXUNICODE
 
 
 # ____________________________________________________________

From noreply at buildbot.pypy.org  Sat Jul 19 14:42:00 2014
From: noreply at buildbot.pypy.org (waedt)
Date: Sat, 19 Jul 2014 14:42:00 +0200 (CEST)
Subject: [pypy-commit] pypy utf8-unicode2: Fix _cffi_backend
Message-ID: <20140719124200.534BF1C0DCA@cobra.cs.uni-duesseldorf.de>

Author: Tyler Wade 
Branch: utf8-unicode2
Changeset: r72469:fbbabe9aebd1
Date: 2014-07-17 05:23 -0500
http://bitbucket.org/pypy/pypy/changeset/fbbabe9aebd1/

Log:	Fix _cffi_backend

diff --git a/pypy/interpreter/test/test_utf8.py b/pypy/interpreter/test/test_utf8.py
--- a/pypy/interpreter/test/test_utf8.py
+++ b/pypy/interpreter/test/test_utf8.py
@@ -195,18 +195,21 @@
     assert s.rsplit(' ', 2) == u.rsplit(' ', 2)
     assert s.rsplit('\n') == [s]
 
-def test_copy_to_wcharp():
+def test_copy_to_new_wcharp():
     s = build_utf8str()
     if sys.maxunicode < 0x10000 and rffi.sizeof(rffi.WCHAR_T) == 4:
         # The last character requires a surrogate pair on narrow builds and
         # so won't be converted correctly by rffi.wcharp2unicode
         s = s[:-1]
 
-    wcharp = s.copy_to_wcharp()
+    wcharp = s.copy_to_new_wcharp()
     u = rffi.wcharp2unicode(wcharp)
     rffi.free_wcharp(wcharp)
     assert s == u
 
+    with s.scoped_wcharp_copy():
+        assert s == u
+
 def test_from_wcharp():
     def check(u):
         wcharp = rffi.unicode2wcharp(u)
diff --git a/pypy/interpreter/utf8.py b/pypy/interpreter/utf8.py
--- a/pypy/interpreter/utf8.py
+++ b/pypy/interpreter/utf8.py
@@ -422,7 +422,7 @@
             byte_pos -= 1
         return byte_pos
 
-    def copy_to_wcharp(self, track_allocation=True):
+    def copy_to_new_wcharp(self, track_allocation=True):
         length = len(self) + 1
         if rffi.sizeof(rffi.WCHAR_T) == 2:
             for c in self.codepoint_iter():
@@ -431,24 +431,34 @@
 
         array = lltype.malloc(WCHAR_INTP.TO, length, flavor='raw',
                               track_allocation=track_allocation)
+
+        self.copy_to_wcharp(array, 0, length)
+        array[length - 1] = wchar_rint(0)
+
+        array = rffi.cast(rffi.CWCHARP, array)
+        return array
+
+    def copy_to_wcharp(self, dst, dststart, length):
         from pypy.interpreter.utf8_codecs import create_surrogate_pair
 
         i = 0;
         for c in self.codepoint_iter():
+            if i == length:
+                break
+
             if rffi.sizeof(rffi.WCHAR_T) == 2:
                 c1, c2 = create_surrogate_pair(c)
-                array[i] = wchar_rint(c1)
+                dst[i + dststart] = wchar_rint(c1)
                 if c2:
                     i += 1
-                    array[i] = wchar_rint(c2)
+                    dst[i + dststart] = wchar_rint(c2)
             else:
-                array[i] = wchar_rint(c)
+                dst[i + dststart] = wchar_rint(c)
 
             i += 1
 
-        array[i] = wchar_rint(0)
-        array = rffi.cast(rffi.CWCHARP, array)
-        return array
+    def scoped_wcharp_copy(self):
+        return WCharContextManager(self)
 
     @staticmethod
     def from_wcharp(wcharp):
@@ -600,6 +610,15 @@
     def build(self):
         return Utf8Str(self._builder.build(), self._is_ascii)
 
+class WCharContextManager(object):
+    def __init__(self, str):
+        self.str = str
+    def __enter__(self):
+        self.data = self.str.copy_to_new_wcharp()
+        return self.data
+    def __exit__(self, *args):
+        rffi.free_wcharp(self.data)
+
 # _______________________________________________
 
 # iter.current is the current (ie the last returned) element
diff --git a/pypy/module/__pypy__/interp_builders.py b/pypy/module/__pypy__/interp_builders.py
--- a/pypy/module/__pypy__/interp_builders.py
+++ b/pypy/module/__pypy__/interp_builders.py
@@ -2,7 +2,8 @@
 from pypy.interpreter.error import OperationError
 from pypy.interpreter.gateway import interp2app, unwrap_spec
 from pypy.interpreter.typedef import TypeDef
-from rpython.rlib.rstring import UnicodeBuilder, StringBuilder
+from pypy.interpreter.utf8 import Utf8Builder
+from rpython.rlib.rstring import StringBuilder
 from rpython.tool.sourcetools import func_with_new_name
 
 
@@ -62,4 +63,4 @@
     return W_Builder
 
 W_StringBuilder = create_builder("StringBuilder", str, StringBuilder)
-W_UnicodeBuilder = create_builder("UnicodeBuilder", unicode, UnicodeBuilder)
+W_UnicodeBuilder = create_builder("UnicodeBuilder", unicode, Utf8Builder)
diff --git a/pypy/module/_cffi_backend/ctypeprim.py b/pypy/module/_cffi_backend/ctypeprim.py
--- a/pypy/module/_cffi_backend/ctypeprim.py
+++ b/pypy/module/_cffi_backend/ctypeprim.py
@@ -9,7 +9,9 @@
 from rpython.rlib import jit
 from rpython.rtyper.lltypesystem import lltype, rffi
 
+from pypy.interpreter import utf8
 from pypy.interpreter.error import oefmt
+from pypy.interpreter.utf8 import Utf8Str, utf8ord
 from pypy.module._cffi_backend import cdataobj, misc
 from pypy.module._cffi_backend.ctypeobj import W_CType
 
@@ -46,7 +48,7 @@
             raise oefmt(space.w_TypeError,
                         "cannot cast unicode string of length %d to ctype '%s'",
                         len(s), self.name)
-        return ord(s[0])
+        return utf8ord(s)
 
     def cast(self, w_ob):
         from pypy.module._cffi_backend import ctypeptr
@@ -128,12 +130,12 @@
     _attrs_ = []
 
     def cast_to_int(self, cdata):
-        unichardata = rffi.cast(rffi.CWCHARP, cdata)
-        return self.space.wrap(ord(unichardata[0]))
+        unichardata = rffi.cast(utf8.WCHAR_INTP, cdata)
+        return self.space.wrap(intmask(unichardata[0]))
 
     def convert_to_object(self, cdata):
         unichardata = rffi.cast(rffi.CWCHARP, cdata)
-        s = rffi.wcharpsize2unicode(unichardata, 1)
+        s = Utf8Str.from_wcharpsize(unichardata, 1)
         return self.space.wrap(s)
 
     def string(self, cdataobj, maxlen):
@@ -154,7 +156,7 @@
 
     def convert_from_object(self, cdata, w_ob):
         value = self._convert_to_unichar(w_ob)
-        rffi.cast(rffi.CWCHARP, cdata)[0] = value
+        rffi.cast(utf8.WCHAR_INTP, cdata)[0] = utf8.wchar_rint(utf8ord(value))
 
 
 class W_CTypePrimitiveSigned(W_CTypePrimitive):
diff --git a/pypy/module/_cffi_backend/ctypeptr.py b/pypy/module/_cffi_backend/ctypeptr.py
--- a/pypy/module/_cffi_backend/ctypeptr.py
+++ b/pypy/module/_cffi_backend/ctypeptr.py
@@ -9,6 +9,8 @@
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rtyper.lltypesystem.rstr import copy_string_to_raw, copy_unicode_to_raw
 
+from pypy.interpreter import utf8
+from pypy.interpreter.utf8 import Utf8Str
 from pypy.interpreter.error import OperationError, oefmt, wrap_oserror
 from pypy.module._cffi_backend import cdataobj, misc, ctypeprim, ctypevoid
 from pypy.module._cffi_backend.ctypeobj import W_CType
@@ -98,10 +100,11 @@
                 raise oefmt(space.w_IndexError,
                             "initializer unicode string is too long for '%s' "
                             "(got %d characters)", self.name, n)
-            unichardata = rffi.cast(rffi.CWCHARP, cdata)
-            copy_unicode_to_raw(llunicode(s), unichardata, 0, n)
+
+            unichardata = rffi.cast(utf8.WCHAR_INTP, cdata)
+            s.copy_to_wcharp(unichardata, 0, n)
             if n != self.length:
-                unichardata[n] = u'\x00'
+                unichardata[n] = utf8.wchar_rint(0)
         else:
             raise self._convert_error("list or tuple", w_ob)
 
@@ -131,9 +134,9 @@
             if self.is_unichar_ptr_or_array():
                 cdata = rffi.cast(rffi.CWCHARP, cdata)
                 if length < 0:
-                    u = rffi.wcharp2unicode(cdata)
+                    u = Utf8Str.from_wcharp(cdata)
                 else:
-                    u = rffi.wcharp2unicoden(cdata, length)
+                    u = Utf8Str.from_wcharpn(cdata, length)
                 keepalive_until_here(cdataobj)
                 return space.wrap(u)
         #
diff --git a/pypy/module/_multibytecodec/c_codecs.py b/pypy/module/_multibytecodec/c_codecs.py
--- a/pypy/module/_multibytecodec/c_codecs.py
+++ b/pypy/module/_multibytecodec/c_codecs.py
@@ -172,7 +172,7 @@
         assert errorcb
         replace, end = errorcb(errors, namecb, reason,
                                stringdata, start, end)
-    inbuf = replace.copy_to_wcharp()
+    inbuf = replace.copy_to_new_wcharp()
     try:
         r = pypy_cjk_dec_replace_on_error(decodebuf, inbuf, len(replace), end)
     finally:
@@ -223,7 +223,7 @@
 def encodeex(encodebuf, unicodedata, errors="strict", errorcb=None,
              namecb=None, ignore_error=0):
     inleft = len(unicodedata)
-    inbuf = unicodedata.copy_to_wcharp()
+    inbuf = unicodedata.copy_to_new_wcharp()
     try:
         if pypy_cjk_enc_init(encodebuf, inbuf, inleft) < 0:
             raise MemoryError
diff --git a/pypy/module/_rawffi/alt/interp_funcptr.py b/pypy/module/_rawffi/alt/interp_funcptr.py
--- a/pypy/module/_rawffi/alt/interp_funcptr.py
+++ b/pypy/module/_rawffi/alt/interp_funcptr.py
@@ -168,7 +168,7 @@
         self.argchain.arg(addr)
 
     def handle_unichar_p(self, w_ffitype, w_obj, unicodeval):
-        buf = unicodeval.copy_to_wcharp()
+        buf = unicodeval.copy_to_new_wcharp()
         self.w_func.to_free.append(rffi.cast(rffi.VOIDP, buf))
         addr = rffi.cast(rffi.ULONG, buf)
         self.argchain.arg(addr)
diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py
--- a/pypy/module/cpyext/unicodeobject.py
+++ b/pypy/module/cpyext/unicodeobject.py
@@ -210,7 +210,7 @@
         # Copy unicode buffer
         w_unicode = from_ref(space, ref)
         u = space.unicode_w(w_unicode)
-        ref_unicode.c_buffer = u.copy_to_wcharp()
+        ref_unicode.c_buffer = u.copy_to_new_wcharp()
     return ref_unicode.c_buffer
 
 @cpython_api([PyObject], rffi.CWCHARP)

From noreply at buildbot.pypy.org  Sat Jul 19 14:42:01 2014
From: noreply at buildbot.pypy.org (waedt)
Date: Sat, 19 Jul 2014 14:42:01 +0200 (CEST)
Subject: [pypy-commit] pypy utf8-unicode2: Fix _io
Message-ID: <20140719124201.ACC021C0DCA@cobra.cs.uni-duesseldorf.de>

Author: Tyler Wade 
Branch: utf8-unicode2
Changeset: r72470:ed2146bad83c
Date: 2014-07-17 23:18 -0500
http://bitbucket.org/pypy/pypy/changeset/ed2146bad83c/

Log:	Fix _io

diff --git a/pypy/module/_io/interp_stringio.py b/pypy/module/_io/interp_stringio.py
--- a/pypy/module/_io/interp_stringio.py
+++ b/pypy/module/_io/interp_stringio.py
@@ -1,6 +1,7 @@
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.typedef import (
     TypeDef, generic_new_descr, GetSetProperty)
+from pypy.interpreter.utf8 import Utf8Str, utf8ord
 from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
 from pypy.module._io.interp_textio import W_TextIOBase, W_IncrementalNewlineDecoder
 from pypy.module._io.interp_iobase import convert_size
@@ -26,8 +27,8 @@
         else:
             newline = space.unicode_w(w_newline)
 
-        if (newline is not None and newline != u"" and newline != u"\n" and
-            newline != u"\r" and newline != u"\r\n"):
+        if (newline is not None and len(newline) != 0 and
+            newline not in (Utf8Str('\n'), Utf8Str('\r\n'), Utf8Str('\r'))):
             # Not using oefmt() because I don't know how to ues it
             # with unicode
             raise OperationError(space.w_ValueError,
@@ -37,9 +38,9 @@
             )
         if newline is not None:
             self.readnl = newline
-        self.readuniversal = newline is None or newline == u""
+        self.readuniversal = newline is None or len(newline) == 0
         self.readtranslate = newline is None
-        if newline and newline[0] == u"\r":
+        if newline and utf8ord(newline) == ord("\r"):
             self.writenl = newline
         if self.readuniversal:
             self.w_decoder = space.call_function(
@@ -112,7 +113,7 @@
         if len(self.buf) > newlength:
             self.buf = self.buf[:newlength]
         if len(self.buf) < newlength:
-            self.buf.extend([u'\0'] * (newlength - len(self.buf)))
+            self.buf.extend([Utf8Str('\0')] * (newlength - len(self.buf)))
 
     def write(self, string):
         length = len(string)
@@ -156,21 +157,21 @@
         start = self.pos
         available = len(self.buf) - start
         if available <= 0:
-            return space.wrap(u"")
+            return space.wrap(Utf8Str(""))
         if size >= 0 and size <= available:
             end = start + size
         else:
             end = len(self.buf)
         assert 0 <= start <= end
         self.pos = end
-        return space.wrap(u''.join(self.buf[start:end]))
+        return space.wrap(Utf8Str('').join(self.buf[start:end]))
 
     def readline_w(self, space, w_limit=None):
         self._check_closed(space)
         limit = convert_size(space, w_limit)
 
         if self.pos >= len(self.buf):
-            return space.wrap(u"")
+            return space.wrap(Utf8Str(""))
 
         start = self.pos
         if limit < 0 or limit > len(self.buf) - self.pos:
@@ -181,7 +182,7 @@
 
         endpos, consumed = self._find_line_ending(
             # XXX: super inefficient, makes a copy of the entire contents.
-            u"".join(self.buf),
+            Utf8Str("").join(self.buf),
             start,
             end
         )
@@ -191,7 +192,7 @@
             endpos = end
         assert endpos >= 0
         self.pos = endpos
-        return space.wrap(u"".join(self.buf[start:endpos]))
+        return space.wrap(Utf8Str("").join(self.buf[start:endpos]))
 
     @unwrap_spec(pos=int, mode=int)
     def seek_w(self, space, pos, mode=0):
@@ -234,7 +235,7 @@
 
     def getvalue_w(self, space):
         self._check_closed(space)
-        return space.wrap(u''.join(self.buf))
+        return space.wrap(Utf8Str('').join(self.buf))
 
     def readable_w(self, space):
         self._check_closed(space)
diff --git a/pypy/module/_io/interp_textio.py b/pypy/module/_io/interp_textio.py
--- a/pypy/module/_io/interp_textio.py
+++ b/pypy/module/_io/interp_textio.py
@@ -6,11 +6,11 @@
 from pypy.interpreter.typedef import (
     GetSetProperty, TypeDef, generic_new_descr, interp_attrproperty,
     interp_attrproperty_w)
+from pypy.interpreter.utf8 import Utf8Str, Utf8Builder, utf8ord
 from pypy.module._codecs import interp_codecs
 from pypy.module._io.interp_iobase import W_IOBase, convert_size, trap_eintr
 from rpython.rlib.rarithmetic import intmask, r_uint, r_ulonglong
 from rpython.rlib.rbigint import rbigint
-from rpython.rlib.rstring import UnicodeBuilder
 
 
 STATE_ZERO, STATE_OK, STATE_DETACHED = range(3)
@@ -29,17 +29,17 @@
 
     def __init__(self, space):
         self.w_newlines_dict = {
-            SEEN_CR: space.wrap(u"\r"),
-            SEEN_LF: space.wrap(u"\n"),
-            SEEN_CRLF: space.wrap(u"\r\n"),
+            SEEN_CR: space.wrap(Utf8Str("\r")),
+            SEEN_LF: space.wrap(Utf8Str("\n")),
+            SEEN_CRLF: space.wrap(Utf8Str("\r\n")),
             SEEN_CR | SEEN_LF: space.newtuple(
-                [space.wrap(u"\r"), space.wrap(u"\n")]),
+                [space.wrap(Utf8Str("\r")), space.wrap(Utf8Str("\n"))]),
             SEEN_CR | SEEN_CRLF: space.newtuple(
-                [space.wrap(u"\r"), space.wrap(u"\r\n")]),
+                [space.wrap(Utf8Str("\r")), space.wrap(Utf8Str("\r\n"))]),
             SEEN_LF | SEEN_CRLF: space.newtuple(
-                [space.wrap(u"\n"), space.wrap(u"\r\n")]),
+                [space.wrap(Utf8Str("\n")), space.wrap(Utf8Str("\r\n"))]),
             SEEN_CR | SEEN_LF | SEEN_CRLF: space.newtuple(
-                [space.wrap(u"\r"), space.wrap(u"\n"), space.wrap(u"\r\n")]),
+                [space.wrap(Utf8Str("\r")), space.wrap(Utf8Str("\n")), space.wrap(Utf8Str("\r\n"))]),
             }
 
     @unwrap_spec(translate=int)
@@ -76,7 +76,7 @@
         output = space.unicode_w(w_output)
         output_len = len(output)
         if self.pendingcr and (final or output_len):
-            output = u'\r' + output
+            output = Utf8Str('\r') + output
             self.pendingcr = False
             output_len += 1
 
@@ -85,13 +85,13 @@
         if not final and output_len > 0:
             last = output_len - 1
             assert last >= 0
-            if output[last] == u'\r':
+            if output[last] == Utf8Str('\r'):
                 output = output[:last]
                 self.pendingcr = True
                 output_len -= 1
 
         if output_len == 0:
-            return space.wrap(u"")
+            return space.wrap(Utf8Str(""))
 
         # Record which newlines are read and do newline translation if
         # desired, all in one pass.
@@ -101,12 +101,12 @@
         # for the \r
         only_lf = False
         if seennl == SEEN_LF or seennl == 0:
-            only_lf = (output.find(u'\r') < 0)
+            only_lf = (output.find(Utf8Str('\r')) < 0)
 
         if only_lf:
             # If not already seen, quick scan for a possible "\n" character.
             # (there's nothing else to be done, even when in translation mode)
-            if seennl == 0 and output.find(u'\n') >= 0:
+            if seennl == 0 and output.find('\n') >= 0:
                 seennl |= SEEN_LF
                 # Finished: we have scanned for newlines, and none of them
                 # need translating.
@@ -115,32 +115,32 @@
             while i < output_len:
                 if seennl == SEEN_ALL:
                     break
-                c = output[i]
+                c = utf8ord(output, i)
                 i += 1
-                if c == u'\n':
+                if c == ord('\n'):
                     seennl |= SEEN_LF
-                elif c == u'\r':
-                    if i < output_len and output[i] == u'\n':
+                elif c == ord('\r'):
+                    if i < output_len and utf8ord(output, i) == ord('\n'):
                         seennl |= SEEN_CRLF
                         i += 1
                     else:
                         seennl |= SEEN_CR
-        elif output.find(u'\r') >= 0:
+        elif output.find('\r') >= 0:
             # Translate!
-            builder = UnicodeBuilder(output_len)
+            builder = Utf8Builder(output_len)
             i = 0
             while i < output_len:
-                c = output[i]
+                c = utf8ord(output, i)
                 i += 1
-                if c == u'\n':
+                if c == ord('\n'):
                     seennl |= SEEN_LF
-                elif c == u'\r':
-                    if i < output_len and output[i] == u'\n':
+                elif c == ord('\r'):
+                    if i < output_len and utf8ord(output, i) == ord('\n'):
                         seennl |= SEEN_CRLF
                         i += 1
                     else:
                         seennl |= SEEN_CR
-                    builder.append(u'\n')
+                    builder.append('\n')
                     continue
                 builder.append(c)
             output = builder.build()
@@ -217,7 +217,7 @@
         if self.readtranslate:
 
             # Newlines are already translated, only search for \n
-            pos = line.find(u'\n', start, end)
+            pos = line.find('\n', start, end)
             if pos >= 0:
                 return pos - start + 1, 0
             else:
@@ -229,16 +229,16 @@
             while True:
                 # Fast path for non-control chars. The loop always ends
                 # since the Py_UNICODE storage is NUL-terminated.
-                while i < size and line[start + i] > '\r':
+                while i < size and utf8ord(line, start + i) > ord('\r'):
                     i += 1
                 if i >= size:
                     return -1, size
-                ch = line[start + i]
+                ch = utf8ord(line, start + i)
                 i += 1
-                if ch == '\n':
+                if ch == ord('\n'):
                     return i, 0
-                if ch == '\r':
-                    if line[start + i] == '\n':
+                if ch == ord('\r'):
+                    if utf8ord(line, start + i) == ord('\n'):
                         return i + 1, 0
                     else:
                         return i, 0
@@ -371,7 +371,8 @@
             newline = None
         else:
             newline = space.unicode_w(w_newline)
-        if newline and newline not in (u'\n', u'\r\n', u'\r'):
+        if newline and newline not in (Utf8Str('\n'), Utf8Str('\r\n'),
+                                       Utf8Str('\r')):
             r = space.str_w(space.repr(w_newline))
             raise OperationError(space.w_ValueError, space.wrap(
                 "illegal newline value: %s" % (r,)))
@@ -382,13 +383,13 @@
         self.readtranslate = newline is None
         self.readnl = newline
 
-        self.writetranslate = (newline != u'')
+        self.writetranslate = (newline != Utf8Str(''))
         if not self.readuniversal:
             self.writenl = self.readnl
-            if self.writenl == u'\n':
+            if self.writenl == Utf8Str('\n'):
                 self.writenl = None
         elif _WINDOWS:
-            self.writenl = u"\r\n"
+            self.writenl = Utf8Str("\r\n")
         else:
             self.writenl = None
 
@@ -508,7 +509,7 @@
 
     def _get_decoded_chars(self, size):
         if self.decoded_chars is None:
-            return u""
+            return Utf8Str("")
 
         available = len(self.decoded_chars) - self.decoded_chars_used
         if size < 0 or size > available:
@@ -603,7 +604,7 @@
             return w_final
 
         remaining = size
-        builder = UnicodeBuilder(size)
+        builder = Utf8Builder(size)
 
         # Keep reading chunks until we have n characters to return
         while True:
@@ -710,12 +711,12 @@
         if chunks:
             if line:
                 chunks.append(line)
-            line = u''.join(chunks)
+            line = Utf8Str('').join(chunks)
 
         if line:
             return space.wrap(line)
         else:
-            return space.wrap(u'')
+            return space.wrap(Utf8Str(''))
 
     # _____________________________________________________________
     # write methods
@@ -736,15 +737,16 @@
 
         haslf = False
         if (self.writetranslate and self.writenl) or self.line_buffering:
-            if text.find(u'\n') >= 0:
+            if text.find('\n') >= 0:
                 haslf = True
         if haslf and self.writetranslate and self.writenl:
-            w_text = space.call_method(w_text, "replace", space.wrap(u'\n'),
+            w_text = space.call_method(w_text, "replace",
+                                       space.wrap(Utf8Str('\n')),
                                        space.wrap(self.writenl))
             text = space.unicode_w(w_text)
 
         needflush = False
-        if self.line_buffering and (haslf or text.find(u'\r') >= 0):
+        if self.line_buffering and (haslf or text.find('\r') >= 0):
             needflush = True
 
         # XXX What if we were just reading?

From noreply at buildbot.pypy.org  Sat Jul 19 14:42:03 2014
From: noreply at buildbot.pypy.org (waedt)
Date: Sat, 19 Jul 2014 14:42:03 +0200 (CEST)
Subject: [pypy-commit] pypy utf8-unicode2: Fix unicodedb
Message-ID: <20140719124203.0D6391C0DCA@cobra.cs.uni-duesseldorf.de>

Author: Tyler Wade 
Branch: utf8-unicode2
Changeset: r72471:6ae77b6a146a
Date: 2014-07-19 07:40 -0500
http://bitbucket.org/pypy/pypy/changeset/6ae77b6a146a/

Log:	Fix unicodedb

diff --git a/pypy/module/unicodedata/interp_ucd.py b/pypy/module/unicodedata/interp_ucd.py
--- a/pypy/module/unicodedata/interp_ucd.py
+++ b/pypy/module/unicodedata/interp_ucd.py
@@ -6,11 +6,11 @@
 from pypy.interpreter.baseobjspace import W_Root
 from pypy.interpreter.error import OperationError
 from pypy.interpreter.typedef import TypeDef, interp_attrproperty
+from pypy.interpreter.utf8 import utf8chr
 from rpython.rlib.rarithmetic import r_longlong
 from rpython.rlib.objectmodel import we_are_translated
-from rpython.rlib.runicode import MAXUNICODE
 from rpython.rlib.unicodedata import unicodedb_5_2_0, unicodedb_3_2_0
-from rpython.rlib.runicode import code_to_unichr, ord_accepts_surrogate
+from rpython.rlib.runicode import ord_accepts_surrogate
 import sys
 
 
@@ -30,47 +30,15 @@
 # unicode code point.
 
 
-if MAXUNICODE > 0xFFFF:
-    # Target is wide build
-    def unichr_to_code_w(space, w_unichr):
-        if not space.isinstance_w(w_unichr, space.w_unicode):
+def unichr_to_code_w(space, w_unichr):
+    if not space.isinstance_w(w_unichr, space.w_unicode):
+        raise OperationError(space.w_TypeError, space.wrap(
+            'argument 1 must be unicode'))
+
+        if not space.len_w(w_unichr) == 1:
             raise OperationError(space.w_TypeError, space.wrap(
-                'argument 1 must be unicode'))
-
-        if not we_are_translated() and sys.maxunicode == 0xFFFF:
-            # Host CPython is narrow build, accept surrogates
-            try:
-                return ord_accepts_surrogate(space.unicode_w(w_unichr))
-            except TypeError:
-                raise OperationError(space.w_TypeError, space.wrap(
                     'need a single Unicode character as parameter'))
-        else:
-            if not space.len_w(w_unichr) == 1:
-                raise OperationError(space.w_TypeError, space.wrap(
-                    'need a single Unicode character as parameter'))
-            return space.int_w(space.ord(w_unichr))
-
-else:
-    # Target is narrow build
-    def unichr_to_code_w(space, w_unichr):
-        if not space.isinstance_w(w_unichr, space.w_unicode):
-            raise OperationError(space.w_TypeError, space.wrap(
-                'argument 1 must be unicode'))
-
-        if not we_are_translated() and sys.maxunicode > 0xFFFF:
-            # Host CPython is wide build, forbid surrogates
-            if not space.len_w(w_unichr) == 1:
-                raise OperationError(space.w_TypeError, space.wrap(
-                    'need a single Unicode character as parameter'))
-            return space.int_w(space.ord(w_unichr))
-
-        else:
-            # Accept surrogates
-            try:
-                return ord_accepts_surrogate(space.unicode_w(w_unichr))
-            except TypeError:
-                raise OperationError(space.w_TypeError, space.wrap(
-                    'need a single Unicode character as parameter'))
+    return space.int_w(space.ord(w_unichr))
 
 
 class UCD(W_Root):
@@ -108,7 +76,7 @@
         except KeyError:
             msg = space.mod(space.wrap("undefined character name '%s'"), space.wrap(name))
             raise OperationError(space.w_KeyError, msg)
-        return space.wrap(code_to_unichr(code))
+        return space.wrap(utf8chr(code))
 
     def name(self, space, w_unichr, w_default=None):
         code = unichr_to_code_w(space, w_unichr)

From noreply at buildbot.pypy.org  Sat Jul 19 14:42:04 2014
From: noreply at buildbot.pypy.org (waedt)
Date: Sat, 19 Jul 2014 14:42:04 +0200 (CEST)
Subject: [pypy-commit] pypy utf8-unicode2: Remove some stray unicode literals
Message-ID: <20140719124204.566071C0DCA@cobra.cs.uni-duesseldorf.de>

Author: Tyler Wade 
Branch: utf8-unicode2
Changeset: r72472:821f21a34f0a
Date: 2014-07-19 07:41 -0500
http://bitbucket.org/pypy/pypy/changeset/821f21a34f0a/

Log:	Remove some stray unicode literals

diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -1551,7 +1551,7 @@
         "Like unicode_w, but rejects strings with NUL bytes."
         from rpython.rlib import rstring
         result = w_obj.unicode_w(self)
-        if u'\x00' in result:
+        if Utf8Str('\x00') in result:
             raise OperationError(self.w_TypeError, self.wrap(
                     'argument must be a unicode string without NUL characters'))
         return rstring.assert_str0(result)
diff --git a/pypy/interpreter/utf8_codecs.py b/pypy/interpreter/utf8_codecs.py
--- a/pypy/interpreter/utf8_codecs.py
+++ b/pypy/interpreter/utf8_codecs.py
@@ -727,7 +727,7 @@
     else:
         bo = 1
     if size == 0:
-        return u'', 0, bo
+        return Utf8Str(''), 0, bo
     if bo == -1:
         # force little endian
         ihi = 1
@@ -911,7 +911,7 @@
     else:
         bo = 1
     if size == 0:
-        return u'', 0, bo
+        return Utf8Str(''), 0, bo
     if bo == -1:
         # force little endian
         iorder = [0, 1, 2, 3]
@@ -1285,7 +1285,7 @@
     if errorhandler is None:
         errorhandler = default_unicode_error_decode
     if size == 0:
-        return u'', 0
+        return Utf8Str(''), 0
 
     pos = 0
     result = Utf8Builder(size)

From noreply at buildbot.pypy.org  Sat Jul 19 20:36:24 2014
From: noreply at buildbot.pypy.org (pjenvey)
Date: Sat, 19 Jul 2014 20:36:24 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: fix bytes result
Message-ID: <20140719183624.D95961C0DCA@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey 
Branch: py3k
Changeset: r72473:e2fd970b748f
Date: 2014-07-18 16:48 -0700
http://bitbucket.org/pypy/pypy/changeset/e2fd970b748f/

Log:	fix bytes result

diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -244,7 +244,7 @@
         """
         size = self.len
         if size == 0:
-            return space.wrap('')
+            return space.wrapbytes('')
         cbuf = self._charbuf_start()
         s = rffi.charpsize2str(cbuf, size * self.itemsize)
         self._charbuf_stop()

From noreply at buildbot.pypy.org  Sat Jul 19 20:36:26 2014
From: noreply at buildbot.pypy.org (pjenvey)
Date: Sat, 19 Jul 2014 20:36:26 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: issue1556: cache json object keys (cpython
 issue7451) in _pypyjson and
Message-ID: <20140719183626.4B7D61C0DCA@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey 
Branch: py3k
Changeset: r72474:ed9ae55d00c8
Date: 2014-07-18 16:48 -0700
http://bitbucket.org/pypy/pypy/changeset/ed9ae55d00c8/

Log:	issue1556: cache json object keys (cpython issue7451) in _pypyjson
	and re-enable it now that it fully passes test_json

diff --git a/lib-python/3/json/__init__.py b/lib-python/3/json/__init__.py
--- a/lib-python/3/json/__init__.py
+++ b/lib-python/3/json/__init__.py
@@ -107,6 +107,12 @@
 
 __author__ = 'Bob Ippolito '
 
+try:
+    # PyPy speedup, the interface is different than CPython's _json
+    import _pypyjson
+except ImportError:
+    _pypyjson = None
+
 from .decoder import JSONDecoder
 from .encoder import JSONEncoder
 
@@ -316,7 +322,7 @@
     if (cls is None and object_hook is None and
             parse_int is None and parse_float is None and
             parse_constant is None and object_pairs_hook is None and not kw):
-        return _default_decoder.decode(s)
+        return _pypyjson.loads(s) if _pypyjson else _default_decoder.decode(s)
     if cls is None:
         cls = JSONDecoder
     if object_hook is not None:
diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py
--- a/pypy/module/_pypyjson/interp_decoder.py
+++ b/pypy/module/_pypyjson/interp_decoder.py
@@ -56,6 +56,7 @@
         self.end_ptr = lltype.malloc(rffi.CCHARPP.TO, 1, flavor='raw')
         self.pos = 0
         self.last_type = TYPE_UNKNOWN
+        self.memo = {}
 
     def close(self):
         rffi.free_charp(self.ll_chars)
@@ -261,6 +262,8 @@
             w_name = self.decode_any(i)
             if self.last_type != TYPE_STRING:
                 self._raise("Key name must be string for object starting at char %d", start)
+            w_name = self.memo.setdefault(self.space.unicode_w(w_name), w_name)
+
             i = self.skip_whitespace(self.pos)
             ch = self.ll_chars[i]
             if ch != ':':
diff --git a/pypy/module/_pypyjson/test/test__pypyjson.py b/pypy/module/_pypyjson/test/test__pypyjson.py
--- a/pypy/module/_pypyjson/test/test__pypyjson.py
+++ b/pypy/module/_pypyjson/test/test__pypyjson.py
@@ -187,4 +187,12 @@
         import _pypyjson
         # http://json.org/JSON_checker/test/fail25.json
         s = '["\ttab\tcharacter\tin\tstring\t"]'
-        raises(ValueError, "_pypyjson.loads(s)")
\ No newline at end of file
+        raises(ValueError, "_pypyjson.loads(s)")
+
+    def test_keys_reuse(self):
+        import _pypyjson
+        s = '[{"a_key": 1, "b_\xe9": 2}, {"a_key": 3, "b_\xe9": 4}]'
+        rval = _pypyjson.loads(s)
+        (a, b), (c, d) = sorted(rval[0]), sorted(rval[1])
+        assert a is c
+        assert b is d

From noreply at buildbot.pypy.org  Sat Jul 19 20:36:27 2014
From: noreply at buildbot.pypy.org (pjenvey)
Date: Sat, 19 Jul 2014 20:36:27 +0200 (CEST)
Subject: [pypy-commit] pypy py3k: merge default
Message-ID: <20140719183627.AD3671C0DCA@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey 
Branch: py3k
Changeset: r72475:8064c75eb975
Date: 2014-07-19 11:35 -0700
http://bitbucket.org/pypy/pypy/changeset/8064c75eb975/

Log:	merge default

diff --git a/pypy/interpreter/test/test_generator.py b/pypy/interpreter/test/test_generator.py
--- a/pypy/interpreter/test/test_generator.py
+++ b/pypy/interpreter/test/test_generator.py
@@ -307,13 +307,13 @@
     w_co = space.appexec([], '''():
         def g(x):
             yield x + 5
-        return g.func_code
+        return g.__code__
     ''')
     assert should_not_inline(w_co) == False
     w_co = space.appexec([], '''():
         def g(x):
             yield x + 5
             yield x + 6
-        return g.func_code
+        return g.__code__
     ''')
     assert should_not_inline(w_co) == True

From noreply at buildbot.pypy.org  Mon Jul 21 10:30:59 2014
From: noreply at buildbot.pypy.org (timfel)
Date: Mon, 21 Jul 2014 10:30:59 +0200 (CEST)
Subject: [pypy-commit] pypy default: fix compilation with Visual Studio 2013
Message-ID: <20140721083059.28F461D2A6B@cobra.cs.uni-duesseldorf.de>

Author: Tim Felgentreff 
Branch: 
Changeset: r72476:dbdcc057562b
Date: 2014-07-21 10:24 +0200
http://bitbucket.org/pypy/pypy/changeset/dbdcc057562b/

Log:	fix compilation with Visual Studio 2013

diff --git a/rpython/translator/platform/windows.py b/rpython/translator/platform/windows.py
--- a/rpython/translator/platform/windows.py
+++ b/rpython/translator/platform/windows.py
@@ -375,21 +375,28 @@
         for rule in rules:
             m.rule(*rule)
 
+        if len(headers_to_precompile)>0 and self.version >= 80:
+            # at least from VS2013 onwards we need to include PCH
+            # objects in the final link command
+            linkobjs = 'stdafx.obj @<<\n$(OBJECTS)\n<<'
+        else:
+            linkobjs = '@<<\n$(OBJECTS)\n<<'
+
         if self.version < 80:
             m.rule('$(TARGET)', '$(OBJECTS)',
                     [ '$(CC_LINK) /nologo $(LDFLAGS) $(LDFLAGSEXTRA) /out:$@' +\
-                      ' $(LIBDIRS) $(LIBS) @<<\n$(OBJECTS)\n<<',
+                      ' $(LIBDIRS) $(LIBS) ' + linkobjs,
                    ])
         else:
             m.rule('$(TARGET)', '$(OBJECTS)',
                     [ '$(CC_LINK) /nologo $(LDFLAGS) $(LDFLAGSEXTRA)' + \
                       ' $(LINKFILES) /out:$@ $(LIBDIRS) $(LIBS) /MANIFEST' + \
-                      ' /MANIFESTFILE:$*.manifest @<<\n$(OBJECTS)\n<<',
+                      ' /MANIFESTFILE:$*.manifest ' + linkobjs,
                     'mt.exe -nologo -manifest $*.manifest -outputresource:$@;1',
                     ])
         m.rule('debugmode_$(TARGET)', '$(OBJECTS)',
                 [ '$(CC_LINK) /nologo /DEBUG $(LDFLAGS) $(LDFLAGSEXTRA)' + \
-                  ' $(LINKFILES) /out:$@ $(LIBDIRS) $(LIBS) @<<\n$(OBJECTS)\n<<',
+                  ' $(LINKFILES) /out:$@ $(LIBDIRS) $(LIBS) ' + linkobjs,
                 ])
 
         if shared:

From noreply at buildbot.pypy.org  Mon Jul 21 11:10:02 2014
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Mon, 21 Jul 2014 11:10:02 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk storage-gcrefs: Extracted walking of
 gc references in separate module.
Message-ID: <20140721091002.6FC8A1C3273@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: storage-gcrefs
Changeset: r925:8476f6a16d21
Date: 2014-07-21 11:10 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/8476f6a16d21/

Log:	Extracted walking of gc references in separate module. Implemented
	walking the heap correctly without collecting a huge list of all
	objects. someInstance primitive still fails in interpreted mode, due
	to MemoryError in get_rpy_roots(). Don't see a way to fix except to
	go 64 bit. Translated version segfaults.

diff --git a/spyvm/gcrefs.py b/spyvm/gcrefs.py
new file mode 100644
--- /dev/null
+++ b/spyvm/gcrefs.py
@@ -0,0 +1,67 @@
+
+from rpython.rlib import rgc, objectmodel, jit
+
+# ======== Internal functions ========
+
+def flag(gcref):
+    return rgc.get_gcflag_extra(gcref)
+
+def toggle_flag(gcref):
+    rgc.toggle_gcflag_extra(gcref)
+
+def references(gcref):
+    return rgc.get_rpy_referents(gcref)
+
+def gc_roots():
+    return rgc.get_rpy_roots()
+
+def _clear_all_flags(gcrefs):
+    for gcref in gcrefs:
+        if gcref and flag(gcref):
+            toggle_flag(gcref)
+            _clear_all_flags(references(gcref))
+
+def _walk_gc_references(func, extra_parameter, collect_into, gcrefs):
+    for gcref in gcrefs:
+        if gcref and not flag(gcref):
+            toggle_flag(gcref)
+            result = func(gcref, extra_parameter)
+            if result is not None:
+                collect_into.append(result)
+            _walk_gc_references(func, extra_parameter, collect_into, references(gcref))
+    return collect_into
+
+# ======== API of this module ========
+# The extra_parameter is here to avoid creating closures in the function parameters,
+# and still be able to pass some context into the functions. It should always be a short tuple,
+# so that rpython can autmatically specialize these functions. If it fails to do so, annotate
+# all functions with extra_parameter with @objectmodel.specialize.argtype(2).
+
+def try_cast(type, gcref):
+    return rgc.try_cast_gcref_to_instance(type, gcref)
+
+ at jit.dont_look_inside
+def walk_gc_references(func, extra_parameter = None):
+    roots = gc_roots()
+    result = _walk_gc_references(func, extra_parameter, [], roots)
+    _clear_all_flags(roots)
+    _clear_all_flags(gc_roots()) # Just in case
+    return result
+
+def walk_gc_references_of_type(type, func, extra_parameter = None):
+    def check_type(gcref, extra):
+        type, func, extra_parameter = extra
+        w_obj = try_cast(type, gcref)
+        if w_obj:
+            func(w_obj, extra_parameter)
+        return None
+    walk_gc_references(check_type, (type, func, extra_parameter))
+
+def collect_gc_references_of_type(type, filter_func = lambda obj, extra: True, extra_parameter = None):
+    def check_type(gcref, extra):
+        type, filter_func, extra_parameter = extra
+        w_obj = try_cast(type, gcref)
+        if w_obj and filter_func(w_obj, extra_parameter):
+            return w_obj
+        return None
+    return walk_gc_references(check_type, (type, filter_func, extra_parameter))
diff --git a/spyvm/primitives.py b/spyvm/primitives.py
--- a/spyvm/primitives.py
+++ b/spyvm/primitives.py
@@ -4,7 +4,7 @@
 import operator
 from spyvm import model, shadow, error, constants, display
 from spyvm.error import PrimitiveFailedError, PrimitiveNotYetWrittenError
-from spyvm import wrapper
+from spyvm import wrapper, gcrefs
 
 from rpython.rlib import rarithmetic, rfloat, unroll, jit, objectmodel
 
@@ -540,43 +540,25 @@
     w_frame.store(interp.space, constants.CTXPART_STACKP_INDEX, interp.space.wrap_int(stackp))
     return w_frame
 
-
 def stm_enabled():
     """NOT RPYTHON"""
     from rpython.rlib import rgc
     return hasattr(rgc, "stm_is_enabled") and rgc.stm_is_enabled()
+
 if stm_enabled():
     def get_instances_array(space, s_frame, w_class):
         return []
 else:
     def get_instances_array(space, s_frame, w_class):
-        # This primitive returns some instance of the class on the stack.
-        # Not sure quite how to do this; maintain a weak list of all
-        # existing instances or something?
-        match_w = s_frame.instances_array(w_class)
-        if match_w is None:
-            match_w = []
-            from rpython.rlib import rgc
-
-            roots = [gcref for gcref in rgc.get_rpy_roots() if gcref]
-            pending = roots[:]
-            while pending:
-                gcref = pending.pop()
-                if not rgc.get_gcflag_extra(gcref):
-                    rgc.toggle_gcflag_extra(gcref)
-                    w_obj = rgc.try_cast_gcref_to_instance(model.W_Object, gcref)
-                    if (w_obj is not None and w_obj.has_class()
-                        and w_obj.getclass(space) is w_class):
-                        match_w.append(w_obj)
-                    pending.extend(rgc.get_rpy_referents(gcref))
-
-            while roots:
-                gcref = roots.pop()
-                if rgc.get_gcflag_extra(gcref):
-                    rgc.toggle_gcflag_extra(gcref)
-                    roots.extend(rgc.get_rpy_referents(gcref))
-            s_frame.store_instances_array(w_class, match_w)
-        return match_w
+        # TODO find a better way then always pre-collecting the entire list of objects.
+        instances = s_frame.instances_array(w_class)
+        if instances is None:
+            def filter_w_obj(w_obj, extra):
+                w_class, space = extra
+                return w_obj.has_class() and w_obj.getclass(space) is w_class
+            instances = gcrefs.collect_gc_references_of_type(model.W_Object, filter_w_obj, (w_class, space))
+            s_frame.store_instances_array(w_class, instances)
+        return instances
 
 @expose_primitive(SOME_INSTANCE, unwrap_spec=[object])
 def func(interp, s_frame, w_class):
@@ -935,32 +917,6 @@
         w_class = assert_pointers(w_class)
         w_class.as_class_get_shadow(interp.space).flush_method_caches()
     return w_rcvr
-
- at objectmodel.specialize.arg(0)
-def walk_gc_references(func, gcrefs):
-    from rpython.rlib import rgc
-    for gcref in gcrefs:
-        if gcref and not rgc.get_gcflag_extra(gcref):
-            try:
-                rgc.toggle_gcflag_extra(gcref)
-                func(gcref)
-                walk_gc_references(func, rgc.get_rpy_referents(gcref))
-            finally:
-                rgc.toggle_gcflag_extra(gcref)
-
- at objectmodel.specialize.arg(0)
-def walk_gc_objects(func):
-    from rpython.rlib import rgc
-    walk_gc_references(func, rgc.get_rpy_roots())
-
- at objectmodel.specialize.arg(0, 1)
-def walk_gc_objects_of_type(type, func):
-    from rpython.rlib import rgc
-    def check_type(gcref):
-        w_obj = rgc.try_cast_gcref_to_instance(type, gcref)
-        if w_obj:
-            func(w_obj)
-    walk_gc_objects(check_type)
     
 if not stm_enabled():
     # XXX: We don't have a global symbol cache. Instead, we walk all
@@ -969,7 +925,7 @@
     def func(interp, s_frame, w_rcvr):
         # This takes a long time (at least in interpreted mode), and is not really necessary.
         # We are monitoring changes to MethodDictionaries, so there is no need for the image to tell us.
-        #walk_gc_objects_of_type(shadow.MethodDictionaryShadow, lambda s_dict: s_dict.flush_method_cache())
+        # gcrefs.walk_gc_references_of_type(shadow.MethodDictionaryShadow, lambda s_dict: s_dict.flush_method_cache())
         return w_rcvr
 
 # ___________________________________________________________________________

From noreply at buildbot.pypy.org  Mon Jul 21 17:48:53 2014
From: noreply at buildbot.pypy.org (Raemi)
Date: Mon, 21 Jul 2014 17:48:53 +0200 (CEST)
Subject: [pypy-commit] benchmarks default: some quicksort changes
Message-ID: <20140721154853.9D6F91C06A7@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: 
Changeset: r267:76d14fcfb50a
Date: 2014-07-21 17:48 +0200
http://bitbucket.org/pypy/benchmarks/changeset/76d14fcfb50a/

Log:	some quicksort changes

diff --git a/multithread/quick_sort/quick_sort.py b/multithread/quick_sort/quick_sort.py
--- a/multithread/quick_sort/quick_sort.py
+++ b/multithread/quick_sort/quick_sort.py
@@ -41,6 +41,7 @@
     qsort(xs, l, l0 + n - l)
 
 
+
 def qsort_f(xs, l0, n, level):
     if n < 2:
         return []
@@ -50,25 +51,35 @@
     r = l + n - 1
     while l <= r:
         with atomic:
-            if xs[l] < pivot:
+            xl = xs[l]
+            if xl < pivot:
                 l += 1
                 continue
-            if xs[r] > pivot:
+            xr = xs[r]
+            if xr > pivot:
                 r -= 1
                 continue
-            xs[l], xs[r] = xs[r], xs[l]
-        l += 1
-        r -= 1
+            xs[l], xs[r] = xr, xl
+            l += 1
+            r -= 1
 
     fs = []
     #right_amount = 1000 > n // 2 > 505
-    right_amount = level == 4
-    if right_amount:
-        fs.append(Future(qsort_f, xs, l0, r - l0 + 1, level+1))
-        fs.append(Future(qsort_f, xs, l, l0 + n - l, level+1))
+    do_futures = level == 4
+    largs = (xs, l0, r - l0 + 1, level+1)
+    rargs = (xs, l, l0 + n - l, level+1)
+    if do_futures:
+        fs.append(Future(qsort_f, *largs))
+        fs.append(Future(qsort_f, *rargs))
     else:
-        fs.extend(qsort_f(xs, l0, r - l0 + 1, level+1))
-        fs.extend(qsort_f(xs, l, l0 + n - l, level+1))
+        if level > 4 and n < 100:
+            with atomic:
+                fs.extend(qsort_f(*largs))
+            with atomic:
+                fs.extend(qsort_f(*rargs))
+        else:
+            fs.extend(qsort_f(*largs))
+            fs.extend(qsort_f(*rargs))
     #print_abort_info(0.0000001)
 
     return fs
@@ -79,26 +90,35 @@
         f = fs.pop()
         fs.extend(f())
 
-def run(threads=2, n=100000):
+def run(threads=2, n=20000):
     threads = int(threads)
     n = int(n)
 
     set_thread_pool(ThreadPool(threads))
 
+    to_sort = range(n)
+    t = 0
+    for i in range(20):
+        with atomic:
+            random.seed(i)
+            random.shuffle(to_sort)
+            s = deque(to_sort)
+        # qsort(s, 0, len(s))
+        hint_commit_soon()
 
-    to_sort = range(n)
-    random.seed(121)
-    random.shuffle(to_sort)
-    s = deque(to_sort)
-    # qsort(s, 0, len(s))
-
-    fs = qsort_f(s, 0, len(s), 0)
-    wait_for_futures(fs)
-
+        t -= time.time()
+        # start as future, otherwise we get more threads
+        # than we want (+1 for the main thread)
+        fs = Future(qsort_f, s, 0, len(s), 0)
+        wait_for_futures(fs())
+        #assert sorted(to_sort) == list(s)
+        t += time.time()
 
     # shutdown current pool
     set_thread_pool(None)
 
+    return t
+
 
 
 if __name__ == "__main__":

From noreply at buildbot.pypy.org  Mon Jul 21 17:48:55 2014
From: noreply at buildbot.pypy.org (Raemi)
Date: Mon, 21 Jul 2014 17:48:55 +0200 (CEST)
Subject: [pypy-commit] benchmarks default: the primes counter
Message-ID: <20140721154855.22D1A1C06A7@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: 
Changeset: r268:bd9befb5e179
Date: 2014-07-21 17:48 +0200
http://bitbucket.org/pypy/benchmarks/changeset/bd9befb5e179/

Log:	the primes counter

diff --git a/multithread/primes/primes.py b/multithread/primes/primes.py
new file mode 100644
--- /dev/null
+++ b/multithread/primes/primes.py
@@ -0,0 +1,78 @@
+# -*- coding: utf-8 -*-
+
+# from https://github.com/Tinche/stm-playground
+
+
+import sys
+import time, random
+from common.abstract_threading import (
+    atomic, Future, set_thread_pool, ThreadPool,
+    hint_commit_soon, print_abort_info)
+
+from itertools import izip, chain, repeat
+
+from Queue import Queue
+from pyprimes import isprime
+import threading
+
+def check_prime(num):
+    return isprime(num), num
+
+
+def grouper(n, iterable, padvalue=None):
+    "grouper(3, 'abcdefg', 'x') --> ('a','b','c'), ('d','e','f'), ('g','x','x')"
+    return izip(*[chain(iterable, repeat(padvalue, n-1))]*n)
+
+
+poison_pill = object()
+
+def worker(tasks, results):
+    while True:
+        batch = tasks.get()
+        if batch is poison_pill:
+            tasks.task_done()
+            return
+
+        result = []
+        for task in batch:
+            with atomic:
+                result.append(check_prime(task))
+        results.put(result)
+
+        tasks.task_done()
+
+
+
+def run(threads=2, n=2000000):
+    threads = int(threads)
+    n = int(n)
+
+    LIMIT = n
+    BATCH_SIZE = 1000
+
+    tasks = Queue()
+    results = Queue()
+    print("Starting...")
+
+    with atomic:
+        for batch in grouper(BATCH_SIZE, xrange(LIMIT), 1):
+            tasks.put(list(batch))
+        for _ in xrange(threads):
+            tasks.put(poison_pill)
+
+    for _ in xrange(threads):
+        t = threading.Thread(target=worker, args=(tasks, results))
+        t.start()
+    tasks.join()
+
+    count = 0
+    while not results.empty():
+        batch_results = results.get()
+        count += sum(1 for res in batch_results if res[0])
+
+    return count
+
+
+
+if __name__ == "__main__":
+    run()
diff --git a/multithread/primes/pyprimes.py b/multithread/primes/pyprimes.py
new file mode 100644
--- /dev/null
+++ b/multithread/primes/pyprimes.py
@@ -0,0 +1,1193 @@
+#!/usr/bin/env python
+
+##  Module pyprimes.py
+##
+##  Copyright (c) 2012 Steven D'Aprano.
+##
+##  Permission is hereby granted, free of charge, to any person obtaining
+##  a copy of this software and associated documentation files (the
+##  "Software"), to deal in the Software without restriction, including
+##  without limitation the rights to use, copy, modify, merge, publish,
+##  distribute, sublicense, and/or sell copies of the Software, and to
+##  permit persons to whom the Software is furnished to do so, subject to
+##  the following conditions:
+##
+##  The above copyright notice and this permission notice shall be
+##  included in all copies or substantial portions of the Software.
+##
+##  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+##  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+##  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+##  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+##  CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+##  TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+##  SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+
+"""Generate and test for small primes using a variety of algorithms
+implemented in pure Python.
+
+This module includes functions for generating prime numbers, primality
+testing, and factorising numbers into prime factors. Prime numbers are
+positive integers with no factors other than themselves and 1.
+
+
+Generating prime numbers
+========================
+
+To generate an unending stream of prime numbers, use the ``primes()``
+generator function:
+
+    primes():
+        Yield prime numbers 2, 3, 5, 7, 11, ...
+
+
+    >>> p = primes()
+    >>> [next(p) for _ in range(10)]
+    [2, 3, 5, 7, 11, 13, 17, 19, 23, 29]
+
+
+To efficiently generate pairs of (isprime(i), i) for integers i, use the
+generator functions ``checked_ints()`` and ``checked_oddints()``:
+
+    checked_ints()
+        Yield pairs of (isprime(i), i) for i=0,1,2,3,4,5...
+
+    checked_oddints()
+        Yield pairs of (isprime(i), i) for odd i=1,3,5,7...
+
+
+    >>> it = checked_ints()
+    >>> [next(it) for _ in range(5)]
+    [(False, 0), (False, 1), (True, 2), (True, 3), (False, 4)]
+
+
+Other convenience functions wrapping ``primes()`` are:
+
+    ------------------  ----------------------------------------------------
+    Function            Description
+    ------------------  ----------------------------------------------------
+    nprimes(n)          Yield the first n primes, then stop.
+    nth_prime(n)        Return the nth prime number.
+    prime_count(x)      Return the number of primes less than or equal to x.
+    primes_below(x)     Yield the primes less than or equal to x.
+    primes_above(x)     Yield the primes strictly greater than x.
+    primesum(n)         Return the sum of the first n primes.
+    primesums()         Yield the partial sums of the prime numbers.
+    ------------------  ----------------------------------------------------
+
+
+Primality testing
+=================
+
+These functions test whether numbers are prime or not. Primality tests fall
+into two categories: exact tests, and probabilistic tests.
+
+Exact tests are guaranteed to give the correct result, but may be slow,
+particularly for large arguments. Probabilistic tests do not guarantee
+correctness, but may be much faster for large arguments.
+
+To test whether an integer is prime, use the ``isprime`` function:
+
+    isprime(n)
+        Return True if n is prime, otherwise return False.
+
+
+    >>> isprime(101)
+    True
+    >>> isprime(102)
+    False
+
+
+Exact primality tests are:
+
+    isprime_naive(n)
+        Naive and slow trial division test for n being prime.
+
+    isprime_division(n)
+        A less naive trial division test for n being prime.
+
+    isprime_regex(n)
+        Uses a regex to test if n is a prime number.
+
+        .. NOTE:: ``isprime_regex`` should be considered a novelty
+           rather than a serious test, as it is very slow.
+
+
+Probabilistic tests do not guarantee correctness, but can be faster for
+large arguments. There are two probabilistic tests:
+
+    fermat(n [, base])
+        Fermat primality test, returns True if n is a weak probable
+        prime to the given base, otherwise False.
+
+    miller_rabin(n [, base])
+        Miller-Rabin primality test, returns True if n is a strong
+        probable prime to the given base, otherwise False.
+
+
+Both guarantee no false negatives: if either function returns False, the
+number being tested is certainly composite. However, both are subject to false
+positives: if they return True, the number is only possibly prime.
+
+
+    >>> fermat(12400013)  # composite 23*443*1217
+    False
+    >>> miller_rabin(14008971)  # composite 3*947*4931
+    False
+
+
+Prime factorisation
+===================
+
+These functions return or yield the prime factors of an integer.
+
+    factors(n)
+        Return a list of the prime factors of n.
+
+    factorise(n)
+        Yield tuples (factor, count) for n.
+
+
+The ``factors(n)`` function lists repeated factors:
+
+
+    >>> factors(37*37*109)
+    [37, 37, 109]
+
+
+The ``factorise(n)`` generator yields a 2-tuple for each unique factor, giving
+the factor itself and the number of times it is repeated:
+
+    >>> list(factorise(37*37*109))
+    [(37, 2), (109, 1)]
+
+
+Alternative and toy prime number generators
+===========================================
+
+These functions are alternative methods of generating prime numbers. Unless
+otherwise stated, they generate prime numbers lazily on demand. These are
+supplied for educational purposes and are generally slower or less efficient
+than the preferred ``primes()`` generator.
+
+    --------------  --------------------------------------------------------
+    Function        Description
+    --------------  --------------------------------------------------------
+    croft()         Yield prime numbers using the Croft Spiral sieve.
+    erat(n)         Return primes up to n by the sieve of Eratosthenes.
+    sieve()         Yield primes using the sieve of Eratosthenes.
+    cookbook()      Yield primes using "Python Cookbook" algorithm.
+    wheel()         Yield primes by wheel factorization.
+    --------------  --------------------------------------------------------
+
+    .. TIP:: In the current implementation, the fastest of these
+       generators is aliased as ``primes()``.
+
+
+"""
+
+
+from __future__ import division
+
+
+import functools
+import itertools
+import random
+
+from re import match as _re_match
+
+
+# Module metadata.
+__version__ = "0.1.2a"
+__date__ = "2012-08-25"
+__author__ = "Steven D'Aprano"
+__author_email__ = "steve+python at pearwood.info"
+
+__all__ = ['primes', 'checked_ints', 'checked_oddints', 'nprimes',
+           'primes_above', 'primes_below', 'nth_prime', 'prime_count',
+           'primesum', 'primesums', 'warn_probably', 'isprime', 'factors',
+           'factorise',
+           ]
+
+
+# ============================
+# Python 2.x/3.x compatibility
+# ============================
+
+# This module should support 2.5+, including Python 3.
+
+try:
+    next
+except NameError:
+    # No next() builtin, so we're probably running Python 2.5.
+    # Use a simplified version (without support for default).
+    def next(iterator):
+        return iterator.next()
+
+try:
+    range = xrange
+except NameError:
+    # No xrange built-in, so we're almost certainly running Python3
+    # and range is already a lazy iterator.
+    assert type(range(3)) is not list
+
+try:
+    from itertools import ifilter as filter, izip as zip
+except ImportError:
+    # Python 3, where filter and zip are already lazy.
+    assert type(filter(None, [1, 2])) is not list
+    assert type(zip("ab", [1, 2])) is not list
+
+try:
+    from itertools import compress
+except ImportError:
+    # Must be Python 2.x, so we need to roll our own.
+    def compress(data, selectors):
+        """compress('ABCDEF', [1,0,1,0,1,1]) --> A C E F"""
+        return (d for d, s in zip(data, selectors) if s)
+
+try:
+    from math import isfinite
+except ImportError:
+    # Python 2.6 or older.
+    try:
+        from math import isnan, isinf
+    except ImportError:
+        # Python 2.5. Quick and dirty substitutes.
+        def isnan(x):
+            return x != x
+        def isinf(x):
+            return x - x != 0
+    def isfinite(x):
+        return not (isnan(x) or isinf(x))
+
+
+# =====================
+# Helpers and utilities
+# =====================
+
+def _validate_int(obj):
+    """Raise an exception if obj is not an integer."""
+    m = int(obj + 0)  # May raise TypeError, or OverflowError.
+    if obj != m:
+        raise ValueError('expected an integer but got %r' % obj)
+
+
+def _validate_num(obj):
+    """Raise an exception if obj is not a finite real number."""
+    m = obj + 0  # May raise TypeError.
+    if not isfinite(m):
+        raise ValueError('expected a finite real number but got %r' % obj)
+
+
+def _base_to_bases(base, n):
+    if isinstance(base, tuple):
+        bases = base
+    else:
+        bases = (base,)
+    for b in bases:
+        _validate_int(b)
+        if not 1 <= b < n:
+            # Note that b=1 is a degenerate case which is always a prime
+            # witness for both the Fermat and Miller-Rabin tests. I mention
+            # this for completeness, not because we need to do anything
+            # about it.
+            raise ValueError('base %d out of range 1...%d' % (b, n-1))
+    return bases
+
+
+# =======================
+# Prime number generators
+# =======================
+
+# The preferred generator to use is ``primes()``, which will be set to the
+# "best" of these generators. (If you disagree with my judgement of best,
+# feel free to use the generator of your choice.)
+
+
+def erat(n):
+    """Return a list of primes up to and including n.
+
+    This is a fixed-size version of the Sieve of Eratosthenes, using an
+    adaptation of the traditional algorithm.
+
+    >>> erat(30)
+    [2, 3, 5, 7, 11, 13, 17, 19, 23, 29]
+    >>> erat(10000) == list(primes_below(10000))
+    True
+
+    """
+    _validate_int(n)
+    # Generate a fixed array of integers.
+    arr = list(range(n+1))  # A list is faster than an array
+    # Cross out 0 and 1 since they aren't prime.
+    arr[0] = arr[1] = None
+    i = 2
+    while i*i <= n:
+        # Cross out all the multiples of i starting from i**2.
+        for p in range(i*i, n+1, i):
+            arr[p] = None
+        # Advance to the next number not crossed off.
+        i += 1
+        while i <= n and arr[i] is None:
+            i += 1
+    return list(filter(None, arr))
+
+
+def sieve():
+    """Yield prime integers using the Sieve of Eratosthenes.
+
+    This algorithm is modified to generate the primes lazily rather than the
+    traditional version which operates on a fixed size array of integers.
+    """
+    # This is based on a paper by Melissa E. O'Neill, with an implementation
+    # given by Gerald Britton:
+    # http://mail.python.org/pipermail/python-list/2009-January/1188529.html
+    innersieve = sieve()
+    prevsq = 1
+    table  = {}
+    i = 2
+    while True:
+        # Note: this explicit test is slightly faster than using
+        # prime = table.pop(i, None) and testing for None.
+        if i in table:
+            prime = table[i]
+            del table[i]
+            nxt = i + prime
+            while nxt in table:
+                nxt += prime
+            table[nxt] = prime
+        else:
+            yield i
+            if i > prevsq:
+                j = next(innersieve)
+                prevsq = j**2
+                table[prevsq] = j
+        i += 1
+
+
+def cookbook():
+    """Yield prime integers lazily using the Sieve of Eratosthenes.
+
+    Another version of the algorithm, based on the Python Cookbook,
+    2nd Edition, recipe 18.10, variant erat2.
+    """
+    # http://onlamp.com/pub/a/python/excerpt/pythonckbk_chap1/index1.html?page=2
+    table = {}
+    yield 2
+    # Iterate over [3, 5, 7, 9, ...]. The following is equivalent to, but
+    # faster than, (2*i+1 for i in itertools.count(1))
+    for q in itertools.islice(itertools.count(3), 0, None, 2):
+        # Note: this explicit test is marginally faster than using
+        # table.pop(i, None) and testing for None.
+        if q in table:
+            p = table[q]; del table[q]  # Faster than pop.
+            x = p + q
+            while x in table or not (x & 1):
+                x += p
+            table[x] = p
+        else:
+            table[q*q] = q
+            yield q
+
+
+def croft():
+    """Yield prime integers using the Croft Spiral sieve.
+
+    This is a variant of wheel factorisation modulo 30.
+    """
+    # Implementation is based on erat3 from here:
+    #   http://stackoverflow.com/q/2211990
+    # and this website:
+    #   http://www.primesdemystified.com/
+    # Memory usage increases roughly linearly with the number of primes seen.
+    # dict ``roots`` stores an entry x:p for every prime p.
+    for p in (2, 3, 5):
+        yield p
+    roots = {9: 3, 25: 5}  # Map d**2 -> d.
+    primeroots = frozenset((1, 7, 11, 13, 17, 19, 23, 29))
+    selectors = (1, 0, 1, 1, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0)
+    for q in compress(
+            # Iterate over prime candidates 7, 9, 11, 13, ...
+            itertools.islice(itertools.count(7), 0, None, 2),
+            # Mask out those that can't possibly be prime.
+            itertools.cycle(selectors)
+            ):
+        # Using dict membership testing instead of pop gives a
+        # 5-10% speedup over the first three million primes.
+        if q in roots:
+            p = roots[q]
+            del roots[q]
+            x = q + 2*p
+            while x in roots or (x % 30) not in primeroots:
+                x += 2*p
+            roots[x] = p
+        else:
+            roots[q*q] = q
+            yield q
+
+
+def wheel():
+    """Generate prime numbers using wheel factorisation modulo 210."""
+    for i in (2, 3, 5, 7, 11):
+        yield i
+    # The following constants are taken from the paper by O'Neill.
+    spokes = (2, 4, 2, 4, 6, 2, 6, 4, 2, 4, 6, 6, 2, 6, 4, 2, 6, 4, 6,
+        8, 4, 2, 4, 2, 4, 8, 6, 4, 6, 2, 4, 6, 2, 6, 6, 4, 2, 4, 6, 2,
+        6, 4, 2, 4, 2, 10, 2, 10)
+    assert len(spokes) == 48
+    # This removes about 77% of the composites that we would otherwise
+    # need to divide by.
+    found = [(11, 121)]  # Smallest prime we care about, and its square.
+    for incr in itertools.cycle(spokes):
+        i += incr
+        for p, p2 in found:
+            if p2 > i:  # i must be a prime.
+                found.append((i, i*i))
+                yield i
+                break
+            elif i % p == 0:  # i must be composite.
+                break
+        else:  # This should never happen.
+            raise RuntimeError("internal error: ran out of prime divisors")
+
+
+# This is the preferred way of generating prime numbers. Set this to the
+# fastest/best generator.
+primes = croft
+
+
+# === Algorithms to avoid ===
+
+class Awful:
+    """Awful and naive prime functions namespace.
+
+    A collection of prime-related algorithms which are supplied for
+    educational purposes, as toys, curios, or as terrible warnings on
+    what **not** to do.
+
+    None of these methods have acceptable performance; they are barely
+    tolerable even for the first 100 primes.
+    """
+
+    # === Prime number generators ===
+
+    @staticmethod
+    def naive_primes1():
+        """Generate prime numbers naively, and REALLY slowly.
+
+        >>> p = Awful.naive_primes1()
+        >>> [next(p) for _ in range(10)]
+        [2, 3, 5, 7, 11, 13, 17, 19, 23, 29]
+
+        This is about as awful as a straight-forward algorithm to generate
+        primes can get without deliberate pessimation. This algorithm does
+        not make even the most trivial optimizations:
+
+        - it tests all numbers as potential primes, whether odd or even,
+          instead of skipping even numbers apart from 2;
+        - it checks for primality by dividing against every number less
+          than the candidate prime itself, instead of stopping at the
+          square root of the candidate;
+        - it fails to bail out early when it finds a factor, instead
+          pointlessly keeps testing.
+
+        The result is that this is horribly slow.
+        """
+        i = 2
+        yield i
+        while True:
+            i += 1
+            composite = False
+            for p in range(2, i):
+                if i%p == 0:
+                    composite = True
+            if not composite:  # It must be a prime.
+                yield i
+
+    @staticmethod
+    def naive_primes2():
+        """Generate prime numbers naively, and very slowly.
+
+        >>> p = Awful.naive_primes2()
+        >>> [next(p) for _ in range(10)]
+        [2, 3, 5, 7, 11, 13, 17, 19, 23, 29]
+
+        This is a little better than ``naive_primes1``, but still horribly
+        slow. It makes a single optimization by using a short-circuit test
+        for primality testing: as soon as a factor is found, the candidate
+        is rejected immediately.
+        """
+        i = 2
+        yield i
+        while True:
+            i += 1
+            if all(i%p != 0 for p in range(2, i)):
+                yield i
+
+    @staticmethod
+    def naive_primes3():
+        """Generate prime numbers naively, and very slowly.
+
+        >>> p = Awful.naive_primes3()
+        >>> [next(p) for _ in range(10)]
+        [2, 3, 5, 7, 11, 13, 17, 19, 23, 29]
+
+        This is an incremental improvement over ``naive_primes2`` by only
+        testing odd numbers as potential primes and factors.
+        """
+        yield 2
+        i = 3
+        yield i
+        while True:
+            i += 2
+            if all(i%p != 0 for p in range(3, i, 2)):
+                yield i
+
+    @staticmethod
+    def trial_division():
+        """Generate prime numbers using a simple trial division algorithm.
+
+        >>> p = Awful.trial_division()
+        >>> [next(p) for _ in range(10)]
+        [2, 3, 5, 7, 11, 13, 17, 19, 23, 29]
+
+        This is the first non-naive algorithm. Due to its simplicity, it may
+        perform acceptably for the first hundred or so primes, if your needs
+        are not very demanding. However, it does not scale well for large
+        numbers of primes.
+
+        This uses three optimizations:
+
+        - only test odd numbers for primality;
+        - only check against the prime factors already seen;
+        - stop checking at the square root of the number being tested.
+
+        With these three optimizations, we get asymptotic behaviour of
+        O(N*sqrt(N)/(log N)**2) where N is the number of primes found.
+
+        Despite these , this is still unacceptably slow, especially
+        as the list of memorised primes grows.
+        """
+        yield 2
+        primes = [2]
+        i = 3
+        while True:
+            it = itertools.takewhile(lambda p, i=i: p*p <= i, primes)
+            if all(i%p != 0 for p in it):
+                primes.append(i)
+                yield i
+            i += 2
+
+    @staticmethod
+    def turner():
+        """Generate prime numbers very slowly using Euler's sieve.
+
+        >>> p = Awful.turner()
+        >>> [next(p) for _ in range(10)]
+        [2, 3, 5, 7, 11, 13, 17, 19, 23, 29]
+
+        The function is named for David Turner, who developed this implementation
+        in a paper in 1975. Due to its simplicity, it has become very popular,
+        particularly in Haskell circles where it is usually implemented as some
+        variation of::
+
+            primes = sieve [2..]
+            sieve (p : xs) = p : sieve [x | x <- xs, x `mod` p > 0]
+
+        This algorithm is sometimes wrongly described as the Sieve of
+        Eratosthenes, but it is not, it is a version of Euler's Sieve.
+
+        Although simple, it is extremely slow and inefficient, with
+        asymptotic behaviour of O(N**2/(log N)**2) which is even worse than
+        trial division, and only marginally better than ``naive_primes1``.
+        O'Neill calls this the "Sleight on Eratosthenes".
+        """
+        # References:
+        #   http://en.wikipedia.org/wiki/Sieve_of_Eratosthenes
+        #   http://en.literateprograms.org/Sieve_of_Eratosthenes_(Haskell)
+        #   http://www.cs.hmc.edu/~oneill/papers/Sieve-JFP.pdf
+        #   http://www.haskell.org/haskellwiki/Prime_numbers
+        nums = itertools.count(2)
+        while True:
+            prime = next(nums)
+            yield prime
+            nums = filter(lambda v, p=prime: (v % p) != 0, nums)
+
+    # === Prime number testing ===
+
+    @staticmethod
+    def isprime_naive(n):
+        """Naive primality test using naive and unoptimized trial division.
+
+        >>> Awful.isprime_naive(17)
+        True
+        >>> Awful.isprime_naive(18)
+        False
+
+        Naive, slow but thorough test for primality using unoptimized trial
+        division. This function does far too much work, and consequently is very
+        slow, but it is simple enough to verify by eye.
+        """
+        _validate_int(n)
+        if n == 2:  return True
+        if n < 2 or n % 2 == 0:  return False
+        for i in range(3, int(n**0.5)+1, 2):
+            if n % i == 0:
+                return False
+        return True
+
+    @staticmethod
+    def isprime_regex(n):
+        """Slow primality test using a regular expression.
+
+        >>> Awful.isprime_regex(11)
+        True
+        >>> Awful.isprime_regex(15)
+        False
+
+        Unsurprisingly, this is not efficient, and should be treated as a
+        novelty rather than a serious implementation. It is O(N^2) in time
+        and O(N) in memory: in other words, slow and expensive.
+        """
+        _validate_int(n)
+        return not _re_match(r'^1?$|^(11+?)\1+$', '1'*n)
+        # For a Perl or Ruby version of this, see here:
+        # http://montreal.pm.org/tech/neil_kandalgaonkar.shtml
+        # http://www.noulakaz.net/weblog/2007/03/18/a-regular-expression-to-check-for-prime-numbers/
+
+
+
+# =====================
+# Convenience functions
+# =====================
+
+def checked_ints():
+    """Yield tuples (isprime(i), i) for integers i=0, 1, 2, 3, 4, ...
+
+    >>> it = checked_ints()
+    >>> [next(it) for _ in range(6)]
+    [(False, 0), (False, 1), (True, 2), (True, 3), (False, 4), (True, 5)]
+
+    """
+    oddnums = checked_oddints()
+    yield (False, 0)
+    yield next(oddnums)
+    yield (True, 2)
+    for t in oddnums:
+        yield t
+        yield (False, t[1]+1)
+
+
+def checked_oddints():
+    """Yield tuples (isprime(i), i) for odd integers i=1, 3, 5, 7, 9, ...
+
+    >>> it = checked_oddints()
+    >>> [next(it) for _ in range(6)]
+    [(False, 1), (True, 3), (True, 5), (True, 7), (False, 9), (True, 11)]
+    >>> [next(it) for _ in range(6)]
+    [(True, 13), (False, 15), (True, 17), (True, 19), (False, 21), (True, 23)]
+
+    """
+    yield (False, 1)
+    odd_primes = primes()
+    _ = next(odd_primes)  # Skip 2.
+    prev = 1
+    for p in odd_primes:
+        # Yield the non-primes between the previous prime and
+        # the current one.
+        for i in itertools.islice(itertools.count(prev + 2), 0, None, 2):
+            if i >= p: break
+            yield (False, i)
+        # And yield the current prime.
+        yield (True, p)
+        prev = p
+
+
+def nprimes(n):
+    """Convenience function that yields the first n primes.
+
+    >>> list(nprimes(10))
+    [2, 3, 5, 7, 11, 13, 17, 19, 23, 29]
+
+    """
+    _validate_int(n)
+    return itertools.islice(primes(), n)
+
+
+def primes_above(x):
+    """Convenience function that yields primes strictly greater than x.
+
+    >>> next(primes_above(200))
+    211
+
+    """
+    _validate_num(x)
+    it = primes()
+    # Consume the primes below x as fast as possible, then yield the rest.
+    p = next(it)
+    while p <= x:
+        p = next(it)
+    yield p
+    for p in it:
+        yield p
+
+
+def primes_below(x):
+    """Convenience function yielding primes less than or equal to x.
+
+    >>> list(primes_below(20))
+    [2, 3, 5, 7, 11, 13, 17, 19]
+
+    """
+    _validate_num(x)
+    for p in primes():
+        if p > x:
+            return
+        yield p
+
+
+def nth_prime(n):
+    """nth_prime(n) -> int
+
+    Return the nth prime number, starting counting from 1. Equivalent to
+    p-subscript-n in standard maths notation.
+
+    >>> nth_prime(1)  # First prime is 2.
+    2
+    >>> nth_prime(5)
+    11
+    >>> nth_prime(50)
+    229
+
+    """
+    # http://www.research.att.com/~njas/sequences/A000040
+    _validate_int(n)
+    if n < 1:
+        raise ValueError('argument must be a positive integer')
+    return next(itertools.islice(primes(), n-1, None))
+
+
+def prime_count(x):
+    """prime_count(x) -> int
+
+    Returns the number of prime numbers less than or equal to x.
+    It is also known as the Prime Counting Function, or pi(x).
+    (Not to be confused with the constant pi = 3.1415....)
+
+    >>> prime_count(20)
+    8
+    >>> prime_count(10000)
+    1229
+
+    The number of primes less than x is approximately x/(ln x - 1).
+    """
+    # See also:  http://primes.utm.edu/howmany.shtml
+    # http://mathworld.wolfram.com/PrimeCountingFunction.html
+    _validate_num(x)
+    return sum(1 for p in primes_below(x))
+
+
+def primesum(n):
+    """primesum(n) -> int
+
+    primesum(n) returns the sum of the first n primes.
+
+    >>> primesum(9)
+    100
+    >>> primesum(49)
+    4888
+
+    The sum of the first n primes is approximately n**2*ln(n)/2.
+    """
+    # See:  http://mathworld.wolfram.com/PrimeSums.html
+    # http://www.research.att.com/~njas/sequences/A007504
+    _validate_int(n)
+    return sum(nprimes(n))
+
+
+def primesums():
+    """Yield the partial sums of the prime numbers.
+
+    >>> p = primesums()
+    >>> [next(p) for _ in range(5)]  # primes 2, 3, 5, 7, 11, ...
+    [2, 5, 10, 17, 28]
+
+    """
+    n = 0
+    for p in primes():
+        n += p
+        yield n
+
+
+# =================
+# Primality testing
+# =================
+
+def isprime(n, trials=25, warn=False):
+    """Return True if n is a prime number, and False if it is not.
+
+    >>> isprime(101)
+    True
+    >>> isprime(102)
+    False
+
+    ==========  =======================================================
+    Argument    Description
+    ==========  =======================================================
+    n           Number being tested for primality.
+    trials      Count of primality tests to perform (default 25).
+    warn        If true, warn on inexact results. (Default is false.)
+    ==========  =======================================================
+
+    For values of ``n`` under approximately 341 trillion, this function is
+    exact and the arguments ``trials`` and ``warn`` are ignored.
+
+    Above this cut-off value, this function may be probabilistic with a small
+    chance of wrongly reporting a composite (non-prime) number as prime. Such
+    composite numbers wrongly reported as prime are "false positive" errors.
+
+    The argument ``trials`` controls  the risk of a false positive error. The
+    larger number of trials, the less the chance of an error (and the slower
+    the function). With the default value of 25, you can expect roughly one
+    such error every million trillion tests, which in practical terms is
+    essentially "never".
+
+    ``isprime`` cannot give a false negative error: if it reports a number is
+    composite, it is certainly composite, but if it reports a number is prime,
+    it may be only probably prime. If you pass a true value for argument
+    ``warn``, then a warning will be raised if the result is probabilistic.
+    """
+    _validate_int(n)
+    # Deal with trivial cases first.
+    if n < 2:
+        return False
+    elif n == 2:
+        return True
+    elif n%2 == 0:
+        return False
+    elif n <= 7:  # 3, 5, 7
+        return True
+    is_probabilistic, bases = _choose_bases(n, trials)
+    is_prime = miller_rabin(n, bases)
+    if is_prime and is_probabilistic and warn:
+        import warnings
+        warnings.warn("number is only probably prime not certainly prime")
+    return is_prime
+
+
+def _choose_bases(n, count):
+    """Choose appropriate bases for the Miller-Rabin primality test.
+
+    If n is small enough, returns a tuple of bases which are provably
+    deterministic for that n. If n is too large, return a selection of
+    possibly random bases.
+
+    With k distinct Miller-Rabin tests, the probability of a false
+    positive result is no more than 1/(4**k).
+    """
+    # The Miller-Rabin test is deterministic and completely accurate for
+    # moderate sizes of n using a surprisingly tiny number of tests.
+    # See: Pomerance, Selfridge and Wagstaff (1980), and Jaeschke (1993)
+    # http://en.wikipedia.org/wiki/Miller%E2%80%93Rabin_primality_test
+    prob = False
+    if n < 1373653:  # ~1.3 million
+        bases = (2, 3)
+    elif n < 9080191:  # ~9.0 million
+        bases = (31, 73)
+    elif n < 4759123141:  # ~4.7 billion
+        # Note to self: checked up to approximately 394 million in 9 hours.
+        bases = (2, 7, 61)
+    elif n < 2152302898747:  # ~2.1 trillion
+        bases = (2, 3, 5, 7, 11)
+    elif n < 3474749660383:  # ~3.4 trillion
+        bases = (2, 3, 5, 7, 11, 13)
+    elif n < 341550071728321:  # ~341 trillion
+        bases = (2, 3, 5, 7, 11, 13, 17)
+    else:
+        # n is sufficiently large that we have to use a probabilistic test.
+        prob = True
+        bases = tuple([random.randint(2, n-1) for _ in range(count)])
+        # FIXME Because bases are chosen at random, there may be duplicates
+        # although with extremely small probability given the size of n.
+        # FIXME Is it worthwhile to special case some of the lower, easier
+        # bases? bases = [2, 3, 5, 7, 11, 13, 17] + [random... ]?
+        # Note: we can always be deterministic, no matter how large N is, by
+        # exhaustive testing against each i in the inclusive range
+        # 1 ... min(n-1, floor(2*(ln N)**2)). We don't do this, because it is
+        # expensive for large N, and of no real practical benefit.
+    return prob, bases
+
+
+def isprime_division(n):
+    """isprime_division(integer) -> True|False
+
+    Exact primality test returning True if the argument is a prime number,
+    otherwise False.
+
+    >>> isprime_division(11)
+    True
+    >>> isprime_division(12)
+    False
+
+    This function uses trial division by the primes, skipping non-primes.
+    """
+    _validate_int(n)
+    if n < 2:
+        return False
+    limit = n**0.5
+    for divisor in primes():
+        if divisor > limit: break
+        if n % divisor == 0: return False
+    return True
+
+
+# === Probabilistic primality tests ===
+
+def fermat(n, base=2):
+    """fermat(n [, base]) -> True|False
+
+    ``fermat(n, base)`` is a probabilistic test for primality which returns
+    True if integer n is a weak probable prime to the given integer base,
+    otherwise n is definitely composite and False is returned.
+
+    ``base`` must be a positive integer between 1 and n-1 inclusive, or a
+    tuple of such bases. By default, base=2.
+
+    If ``fermat`` returns False, that is definite proof that n is composite:
+    there are no false negatives. However, if it returns True, that is only
+    provisional evidence that n is prime. For example:
+
+    >>> fermat(99, 7)
+    False
+    >>> fermat(29, 7)
+    True
+
+    We can conclude that 99 is definitely composite, and state that 7 is a
+    witness that 29 may be prime.
+
+    As the Fermat test is probabilistic, composite numbers will sometimes
+    pass a test, or even repeated tests:
+
+    >>> fermat(3*11*17, 7)  # A pseudoprime to base 7.
+    True
+
+    You can perform multiple tests with a single call by passing a tuple of
+    ints as ``base``. The number must pass the Fermat test for all the bases
+    in order to return True. If any test fails, ``fermat`` will return False.
+
+    >>> fermat(41041, (17, 23, 356, 359))  # 41041 = 7*11*13*41
+    True
+    >>> fermat(41041, (17, 23, 356, 359, 363))
+    False
+
+    If a number passes ``k`` Fermat tests, we can conclude that the
+    probability that it is either a prime number, or a particular type of
+    pseudoprime known as a Carmichael number, is at least ``1 - (1/2**k)``.
+    """
+    # http://en.wikipedia.org/wiki/Fermat_primality_test
+    _validate_int(n)
+    bases = _base_to_bases(base, n)
+    # Deal with the simple deterministic cases first.
+    if n < 2:
+        return False
+    elif n == 2:
+        return True
+    elif n % 2 == 0:
+        return False
+    # Now the Fermat test proper.
+    for a in bases:
+        if pow(a, n-1, n) != 1:
+            return False  # n is certainly composite.
+    return True  # All of the bases are witnesses for n being prime.
+
+
+def miller_rabin(n, base=2):
+    """miller_rabin(integer [, base]) -> True|False
+
+    ``miller_rabin(n, base)`` is a probabilistic test for primality which
+    returns True if integer n is a strong probable prime to the given integer
+    base, otherwise n is definitely composite and False is returned.
+
+    ``base`` must be a positive integer between 1 and n-1 inclusive, or a
+    tuple of such bases. By default, base=2.
+
+    If ``miller_rabin`` returns False, that is definite proof that n is
+    composite: there are no false negatives. However, if it returns True,
+    that is only provisional evidence that n is prime:
+
+    >>> miller_rabin(99, 7)
+    False
+    >>> miller_rabin(29, 7)
+    True
+
+    We can conclude from this that 99 is definitely composite, and that 29 is
+    possibly prime.
+
+    As the Miller-Rabin test is probabilistic, composite numbers will
+    sometimes pass one or more tests:
+
+    >>> miller_rabin(3*11*17, 103)  # 3*11*17=561, the 1st Carmichael number.
+    True
+
+    You can perform multiple tests with a single call by passing a tuple of
+    ints as ``base``. The number must pass the Miller-Rabin test for each of
+    the bases before it will return True. If any test fails, ``miller_rabin``
+    will return False.
+
+    >>> miller_rabin(41041, (16, 92, 100, 256))  # 41041 = 7*11*13*41
+    True
+    >>> miller_rabin(41041, (16, 92, 100, 256, 288))
+    False
+
+    If a number passes ``k`` Miller-Rabin tests, we can conclude that the
+    probability that it is a prime number is at least ``1 - (1/4**k)``.
+    """
+    # http://en.wikipedia.org/wiki/Miller%E2%80%93Rabin_primality_test
+    _validate_int(n)
+    bases = _base_to_bases(base, n)
+    # Deal with the trivial cases.
+    if n < 2:
+        return False
+    if n == 2:
+        return True
+    elif n % 2 == 0:
+        return False
+    # Now perform the Miller-Rabin test proper.
+    # Start by writing n-1 as 2**s * d.
+    d, s = _factor2(n-1)
+    for a in bases:
+        if _is_composite(a, d, s, n):
+            return False  # n is definitely composite.
+    # If we get here, all of the bases are witnesses for n being prime.
+    return True
+
+
+def _factor2(n):
+    """Factorise positive integer n as d*2**i, and return (d, i).
+
+    >>> _factor2(768)
+    (3, 8)
+    >>> _factor2(18432)
+    (9, 11)
+
+    Private function used internally by ``miller_rabin``.
+    """
+    assert n > 0 and int(n) == n
+    i = 0
+    d = n
+    while 1:
+        q, r = divmod(d, 2)
+        if r == 1:
+            break
+        i += 1
+        d = q
+    assert d%2 == 1
+    assert d*2**i == n
+    return (d, i)
+
+
+def _is_composite(b, d, s, n):
+    """_is_composite(b, d, s, n) -> True|False
+
+    Tests base b to see if it is a witness for n being composite. Returns
+    True if n is definitely composite, otherwise False if it *may* be prime.
+
+    >>> _is_composite(4, 3, 7, 385)
+    True
+    >>> _is_composite(221, 3, 7, 385)
+    False
+
+    Private function used internally by ``miller_rabin``.
+    """
+    assert d*2**s == n-1
+    if pow(b, d, n) == 1:
+        return False
+    for i in range(s):
+        if pow(b, 2**i * d, n) == n-1:
+            return False
+    return True
+
+
+# ===================
+# Prime factorisation
+# ===================
+
+if __debug__:
+    # Set _EXTRA_CHECKS to True to enable potentially expensive assertions
+    # in the factors() and factorise() functions. This is only defined or
+    # checked when assertions are enabled.
+    _EXTRA_CHECKS = False
+
+
+def factors(n):
+    """factors(integer) -> [list of factors]
+
+    Returns a list of the (mostly) prime factors of integer n. For negative
+    integers, -1 is included as a factor. If n is 0 or 1, [n] is returned as
+    the only factor. Otherwise all the factors will be prime.
+
+    >>> factors(-693)
+    [-1, 3, 3, 7, 11]
+    >>> factors(55614)
+    [2, 3, 13, 23, 31]
+
+    """
+    _validate_int(n)
+    result = []
+    for p, count in factorise(n):
+        result.extend([p]*count)
+    if __debug__:
+        # The following test only occurs if assertions are on.
+        if _EXTRA_CHECKS:
+            prod = 1
+            for x in result:
+                prod *= x
+            assert prod == n, ('factors(%d) failed multiplication test' % n)
+    return result
+
+
+def factorise(n):
+    """factorise(integer) -> yield factors of integer lazily
+
+    >>> list(factorise(3*7*7*7*11))
+    [(3, 1), (7, 3), (11, 1)]
+
+    Yields tuples of (factor, count) where each factor is unique and usually
+    prime, and count is an integer 1 or larger.
+
+    The factors are prime, except under the following circumstances: if the
+    argument n is negative, -1 is included as a factor; if n is 0 or 1, it
+    is given as the only factor. For all other integer n, all of the factors
+    returned are prime.
+    """
+    _validate_int(n)
+    if n in (0, 1, -1):
+        yield (n, 1)
+        return
+    elif n < 0:
+        yield (-1, 1)
+        n = -n
+    assert n >= 2
+    for p in primes():
+        if p*p > n: break
+        count = 0
+        while n % p == 0:
+            count += 1
+            n //= p
+        if count:
+            yield (p, count)
+    if n != 1:
+        if __debug__:
+            # The following test only occurs if assertions are on.
+            if _EXTRA_CHECKS:
+                assert isprime(n), ('failed isprime test for %d' % n)
+        yield (n, 1)
+
+
+
+if __name__ == '__main__':
+    import doctest
+    doctest.testmod()
+

From noreply at buildbot.pypy.org  Mon Jul 21 18:39:11 2014
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Mon, 21 Jul 2014 18:39:11 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk storage: Fixed WeakMessageSend class.
Message-ID: <20140721163911.05C0D1C06A7@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: storage
Changeset: r926:64d91d5de341
Date: 2014-07-21 18:35 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/64d91d5de341/

Log:	Fixed WeakMessageSend class. Errors at startup are gone, and print-
	its in workspace work!!

diff --git a/images/Squeak4.5-noBitBlt.changes b/images/Squeak4.5-noBitBlt.changes
--- a/images/Squeak4.5-noBitBlt.changes
+++ b/images/Squeak4.5-noBitBlt.changes
@@ -12620,4 +12620,15 @@
 	
 	1 to: self splayTreeSize do: [:i |
 		self insertNewNode.
-	]! !

----SNAPSHOT----{15 July 2014 . 6:10:56 pm} Squeak4.5-noBitBlt.image priorSource: 15894330!
\ No newline at end of file
+	]! !

----SNAPSHOT----{15 July 2014 . 6:10:56 pm} Squeak4.5-noBitBlt.image priorSource: 15894330!
+
+----QUIT/NOSAVE----{21 July 2014 . 4:18:39 pm} Squeak4.5-noBitBlt.image priorSource: 15894825!

----STARTUP----{21 July 2014 . 6:19:06 pm} as C:\Dev\lang-smalltalk\images\Squeak4.5-noBitBlt.image!

!WeakMessageSend methodsFor: 'private' stamp: 'ag 7/21/2014 18:20' prior: 34321504!
withEnsuredReceiverAndArgumentsDo: aBlock otherwise: altBlock
	"Grab real references to receiver and arguments. If they still exist, evaluate aBlock."

	"Return if my receiver has gone away"
	| r a |
	r := self receiver.
	r ifNil: [ ^altBlock value ].

	"Make sure that my arguments haven't gone away"
	arguments ifNil: [ ^ altBlock value ].
	a := Array withAll: arguments.
	a with: shouldBeNil do: [ :arg :flag |
		arg ifNil: [ flag ifFalse: [ ^altBlock value ]]
	].

	^aBlock value: r value: a! !

----QUIT----{21 July 2014 . 6:20:43 pm} Squeak4.5-noBitBlt.image priorSource: 15894825!
+
+----QUIT/NOSAVE----{21 July 2014 . 4:21:36 pm} Squeak4.5-noBitBlt.image priorSource: 15895702!

----STARTUP----{21 July 2014 . 6:21:54 pm} as C:\Dev\lang-smalltalk\images\Squeak4.5-noBitBlt.image!

!WeakMessageSend methodsFor: 'comparing' stamp: 'ag 7/21/2014 18:22' prior: 33144463!
= anObject
	"Compare equal to equivalent MessageSend"
	^ anObject isMessageSend
		and: [self receiver == anObject receiver
		and: [selector == anObject selector
		and: [(Array withAll: self arguments) = (Array withAll: anObject arguments)]]]
! !
!WeakMessageSend methodsFor: 'private' stamp: 'ag 7/21/2014 18:23' prior: 49449636!
withEnsuredReceiverAndArgumentsDo: aBlock otherwise: altBlock
	"Grab real references to receiver and arguments. If they still exist, evaluate aBlock."

	"Return if my receiver has gone away"
	| r a |
	r := self receiver.
	r ifNil: [ ^altBlock value ].

	"Make sure that my arguments haven't gone away"
	a := Array withAll: self arguments.
	a with: shouldBeNil do: [ :arg :flag |
		arg ifNil: [ flag ifFalse: [ ^altBlock value ]]
	].

	^aBlock value: r value: a! !

----QUIT----{21 July 2014 . 6:23:49 pm} Squeak4.5-noBitBlt.image priorSource: 15895702!

----STARTUP----{21 July 2014 . 6:31:05 pm} as C:\Dev\lang-smalltalk\images\Squeak4.5-noBitBlt.image!

!WeakMessageSend methodsFor: 'accessing' stamp: 'ag 7/21/2014 18:31'!
shouldBeNil
	
	^ shouldBeNil ifNil: [ Array new ]! !
!WeakMessageSend methodsFor: 'private' stamp: 'ag 7/21/2014 18:31' prior: 33148869!
isAnyArgumentGarbage
	"Make sure that my arguments haven't gone away"
	arguments ifNotNil: [
		arguments with: self shouldBeNil do: [ :arg :flag |
			(flag not and: [arg isNil])
				ifTrue: [^true]
		]
	].
	^false
! !
!WeakMessageSend methodsFor: 'private' stamp: 'ag 7/21/2014 18:31' prior: 49450841!
withEnsuredReceiverAndArgumentsDo: aBlock otherwise: altBlock
	"Grab real references to receiver and arguments. If they still exist, evaluate aBlock."

	"Return if my receiver has gone away"
	| r a |
	r := self receiver.
	r ifNil: [ ^altBlock value ].

	"Make sure that my arguments haven't gone away"
	a := Array withAll: self arguments.
	a with: self shouldBeNil do: [ :arg :flag |
		arg ifNil: [ flag ifFalse: [ ^altBlock value ]]
	].

	^aBlock value: r value: a! !
!WeakMessageSend methodsFor: 'private' stamp: 'ag 7/21/2014 18:32' prior: 34360552!
withEnsuredReceiverAndArgumentsDo: aBlock withEnoughArguments: anArray otherwise: altBlock
	"call the selector with enough arguments from arguments and anArray"
	| r selfArgs enoughArgs |
	r := self receiver.
	r ifNil: [ ^altBlock value ].
	
	selfArgs := self arguments.
	selfArgs with: self shouldBeNil do: [ :arg :flag |
		arg ifNil: [ flag ifFalse: [ ^altBlock value ]]
	].

	enoughArgs := Array new: selector numArgs.
	enoughArgs replaceFrom: 1
		to: ( selfArgs size min: enoughArgs size)
		with: selfArgs
		startingAt: 1.
	enoughArgs size > selfArgs size ifTrue: [
		enoughArgs replaceFrom: selfArgs size + 1
			to: (selfArgs size + anArray size min: enoughArgs size)
			with: anArray
			startingAt: 1.
	].
	^aBlock value: r value: enoughArgs! !

----QUIT----{21 July 2014 . 6:32:32 pm} Squeak4.5-noBitBlt.image priorSource: 15896872!
+
+----STARTUP----{21 July 2014 . 4:32:52 pm} as C:\Dev\lang-smalltalk\images\Squeak4.5-noBitBlt.image!
+
+
+1+1!
+
+----QUIT/NOSAVE----{21 July 2014 . 4:33:15 pm} Squeak4.5-noBitBlt.image priorSource: 15898877!
\ No newline at end of file
diff --git a/images/Squeak4.5-noBitBlt.image b/images/Squeak4.5-noBitBlt.image
index ed92c78c940799d91bb94a8ed8527076db6816c7..00843c2c83f9c11e5dcfa3b9927bd415d0a22cd8
GIT binary patch

[cut]


From noreply at buildbot.pypy.org  Tue Jul 22 04:34:25 2014
From: noreply at buildbot.pypy.org (wenzhuman)
Date: Tue, 22 Jul 2014 04:34:25 +0200 (CEST)
Subject: [pypy-commit] pypy gc_no_cleanup_nursery: merge default
Message-ID: <20140722023425.A15CD1C06A7@cobra.cs.uni-duesseldorf.de>

Author: wenzhuman 
Branch: gc_no_cleanup_nursery
Changeset: r72477:733a54b31533
Date: 2014-07-21 15:11 +0000
http://bitbucket.org/pypy/pypy/changeset/733a54b31533/

Log:	merge default

diff too long, truncating to 2000 out of 9153 lines

diff --git a/lib-python/2.7/ctypes/__init__.py b/lib-python/2.7/ctypes/__init__.py
--- a/lib-python/2.7/ctypes/__init__.py
+++ b/lib-python/2.7/ctypes/__init__.py
@@ -389,12 +389,13 @@
             func.__name__ = name_or_ordinal
         return func
 
-class PyDLL(CDLL):
-    """This class represents the Python library itself.  It allows to
-    access Python API functions.  The GIL is not released, and
-    Python exceptions are handled correctly.
-    """
-    _func_flags_ = _FUNCFLAG_CDECL | _FUNCFLAG_PYTHONAPI
+# Not in PyPy
+#class PyDLL(CDLL):
+#    """This class represents the Python library itself.  It allows to
+#    access Python API functions.  The GIL is not released, and
+#    Python exceptions are handled correctly.
+#    """
+#    _func_flags_ = _FUNCFLAG_CDECL | _FUNCFLAG_PYTHONAPI
 
 if _os.name in ("nt", "ce"):
 
@@ -447,15 +448,8 @@
         return self._dlltype(name)
 
 cdll = LibraryLoader(CDLL)
-pydll = LibraryLoader(PyDLL)
-
-if _os.name in ("nt", "ce"):
-    pythonapi = PyDLL("python dll", None, _sys.dllhandle)
-elif _sys.platform == "cygwin":
-    pythonapi = PyDLL("libpython%d.%d.dll" % _sys.version_info[:2])
-else:
-    pythonapi = PyDLL(None)
-
+# not on PyPy
+#pydll = LibraryLoader(PyDLL)
 
 if _os.name in ("nt", "ce"):
     windll = LibraryLoader(WinDLL)
diff --git a/lib-python/2.7/ctypes/test/test_values.py b/lib-python/2.7/ctypes/test/test_values.py
--- a/lib-python/2.7/ctypes/test/test_values.py
+++ b/lib-python/2.7/ctypes/test/test_values.py
@@ -4,6 +4,7 @@
 
 import unittest
 from ctypes import *
+from ctypes.test import xfail
 
 import _ctypes_test
 
@@ -23,7 +24,8 @@
 
     class Win_ValuesTestCase(unittest.TestCase):
         """This test only works when python itself is a dll/shared library"""
-
+        
+        @xfail
         def test_optimizeflag(self):
             # This test accesses the Py_OptimizeFlag intger, which is
             # exported by the Python dll.
@@ -40,6 +42,7 @@
             else:
                 self.assertEqual(opt, 2)
 
+        @xfail
         def test_frozentable(self):
             # Python exports a PyImport_FrozenModules symbol. This is a
             # pointer to an array of struct _frozen entries.  The end of the
@@ -75,6 +78,7 @@
             from ctypes import _pointer_type_cache
             del _pointer_type_cache[struct_frozen]
 
+        @xfail
         def test_undefined(self):
             self.assertRaises(ValueError, c_int.in_dll, pydll, "Undefined_Symbol")
 
diff --git a/lib-python/2.7/test/test_gdbm.py b/lib-python/2.7/test/test_gdbm.py
--- a/lib-python/2.7/test/test_gdbm.py
+++ b/lib-python/2.7/test/test_gdbm.py
@@ -98,6 +98,17 @@
         self.assertTrue(key in self.g)
         self.assertTrue(self.g.has_key(key))
 
+    def test_unicode_key(self):
+        key = u'ab'
+        value = u'cd'
+        self.g = gdbm.open(filename, 'cf')
+        self.g[key] = value
+        self.g.close()
+        self.g = gdbm.open(filename, 'r')
+        self.assertEquals(self.g[key], value)
+        self.assertTrue(key in self.g)
+        self.assertTrue(self.g.has_key(key))
+
 def test_main():
     run_unittest(TestGdbm)
 
diff --git a/lib-python/2.7/timeit.py b/lib-python/2.7/timeit.py
--- a/lib-python/2.7/timeit.py
+++ b/lib-python/2.7/timeit.py
@@ -55,11 +55,6 @@
 import gc
 import sys
 import time
-try:
-    import itertools
-except ImportError:
-    # Must be an older Python version (see timeit() below)
-    itertools = None
 
 __all__ = ["Timer"]
 
@@ -81,7 +76,8 @@
 def inner(_it, _timer):
     %(setup)s
     _t0 = _timer()
-    for _i in _it:
+    while _it > 0:
+        _it -= 1
         %(stmt)s
     _t1 = _timer()
     return _t1 - _t0
@@ -96,7 +92,8 @@
     def inner(_it, _timer, _func=func):
         setup()
         _t0 = _timer()
-        for _i in _it:
+        while _it > 0:
+            _it -= 1
             _func()
         _t1 = _timer()
         return _t1 - _t0
@@ -133,9 +130,19 @@
             else:
                 raise ValueError("setup is neither a string nor callable")
             self.src = src # Save for traceback display
-            code = compile(src, dummy_src_name, "exec")
-            exec code in globals(), ns
-            self.inner = ns["inner"]
+            def make_inner():
+                # PyPy tweak: recompile the source code each time before
+                # calling inner(). There are situations like Issue #1776
+                # where PyPy tries to reuse the JIT code from before,
+                # but that's not going to work: the first thing the
+                # function does is the "-s" statement, which may declare
+                # new classes (here a namedtuple). We end up with
+                # bridges from the inner loop; more and more of them
+                # every time we call inner().
+                code = compile(src, dummy_src_name, "exec")
+                exec code in globals(), ns
+                return ns["inner"]
+            self.make_inner = make_inner
         elif hasattr(stmt, '__call__'):
             self.src = None
             if isinstance(setup, basestring):
@@ -144,7 +151,8 @@
                     exec _setup in globals(), ns
             elif not hasattr(setup, '__call__'):
                 raise ValueError("setup is neither a string nor callable")
-            self.inner = _template_func(setup, stmt)
+            inner = _template_func(setup, stmt)
+            self.make_inner = lambda: inner
         else:
             raise ValueError("stmt is neither a string nor callable")
 
@@ -185,15 +193,12 @@
         to one million.  The main statement, the setup statement and
         the timer function to be used are passed to the constructor.
         """
-        if itertools:
-            it = itertools.repeat(None, number)
-        else:
-            it = [None] * number
+        inner = self.make_inner()
         gcold = gc.isenabled()
         if '__pypy__' not in sys.builtin_module_names:
             gc.disable()    # only do that on CPython
         try:
-            timing = self.inner(it, self.timer)
+            timing = inner(number, self.timer)
         finally:
             if gcold:
                 gc.enable()
diff --git a/lib-python/2.7/xml/sax/saxutils.py b/lib-python/2.7/xml/sax/saxutils.py
--- a/lib-python/2.7/xml/sax/saxutils.py
+++ b/lib-python/2.7/xml/sax/saxutils.py
@@ -98,13 +98,14 @@
         except AttributeError:
             pass
     # wrap a binary writer with TextIOWrapper
-    class UnbufferedTextIOWrapper(io.TextIOWrapper):
-        def write(self, s):
-            super(UnbufferedTextIOWrapper, self).write(s)
-            self.flush()
-    return UnbufferedTextIOWrapper(buffer, encoding=encoding,
+    return _UnbufferedTextIOWrapper(buffer, encoding=encoding,
                                    errors='xmlcharrefreplace',
                                    newline='\n')
+# PyPy: moved this class outside the function above
+class _UnbufferedTextIOWrapper(io.TextIOWrapper):
+    def write(self, s):
+        super(_UnbufferedTextIOWrapper, self).write(s)
+        self.flush()
 
 class XMLGenerator(handler.ContentHandler):
 
diff --git a/lib_pypy/_ctypes/function.py b/lib_pypy/_ctypes/function.py
--- a/lib_pypy/_ctypes/function.py
+++ b/lib_pypy/_ctypes/function.py
@@ -219,6 +219,8 @@
             if restype is None:
                 import ctypes
                 restype = ctypes.c_int
+            if self._argtypes_ is None:
+                self._argtypes_ = []
             self._ptr = self._getfuncptr_fromaddress(self._argtypes_, restype)
             self._check_argtypes_for_fastpath()
             return
diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py
--- a/lib_pypy/cffi/__init__.py
+++ b/lib_pypy/cffi/__init__.py
@@ -4,5 +4,5 @@
 from .api import FFI, CDefError, FFIError
 from .ffiplatform import VerificationError, VerificationMissing
 
-__version__ = "0.8.2"
-__version_info__ = (0, 8, 2)
+__version__ = "0.8.6"
+__version_info__ = (0, 8, 6)
diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py
--- a/lib_pypy/cffi/api.py
+++ b/lib_pypy/cffi/api.py
@@ -55,8 +55,7 @@
             # _cffi_backend.so compiled.
             import _cffi_backend as backend
             from . import __version__
-            assert (backend.__version__ == __version__ or
-                    backend.__version__ == __version__[:3])
+            assert backend.__version__ == __version__
             # (If you insist you can also try to pass the option
             # 'backend=backend_ctypes.CTypesBackend()', but don't
             # rely on it!  It's probably not going to work well.)
@@ -443,6 +442,10 @@
                 for enumname, enumval in zip(tp.enumerators, tp.enumvalues):
                     if enumname not in library.__dict__:
                         library.__dict__[enumname] = enumval
+            for key, val in ffi._parser._int_constants.items():
+                if key not in library.__dict__:
+                    library.__dict__[key] = val
+
             copied_enums.append(True)
             if name in library.__dict__:
                 return
diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py
--- a/lib_pypy/cffi/cparser.py
+++ b/lib_pypy/cffi/cparser.py
@@ -24,6 +24,7 @@
 _r_partial_array = re.compile(r"\[\s*\.\.\.\s*\]")
 _r_words = re.compile(r"\w+|\S")
 _parser_cache = None
+_r_int_literal = re.compile(r"^0?x?[0-9a-f]+u?l?$", re.IGNORECASE)
 
 def _get_parser():
     global _parser_cache
@@ -99,6 +100,7 @@
         self._structnode2type = weakref.WeakKeyDictionary()
         self._override = False
         self._packed = False
+        self._int_constants = {}
 
     def _parse(self, csource):
         csource, macros = _preprocess(csource)
@@ -128,9 +130,10 @@
         finally:
             if lock is not None:
                 lock.release()
-        return ast, macros
+        # csource will be used to find buggy source text
+        return ast, macros, csource
 
-    def convert_pycparser_error(self, e, csource):
+    def _convert_pycparser_error(self, e, csource):
         # xxx look for ":NUM:" at the start of str(e) and try to interpret
         # it as a line number
         line = None
@@ -142,6 +145,12 @@
                 csourcelines = csource.splitlines()
                 if 1 <= linenum <= len(csourcelines):
                     line = csourcelines[linenum-1]
+        return line
+
+    def convert_pycparser_error(self, e, csource):
+        line = self._convert_pycparser_error(e, csource)
+
+        msg = str(e)
         if line:
             msg = 'cannot parse "%s"\n%s' % (line.strip(), msg)
         else:
@@ -160,14 +169,9 @@
             self._packed = prev_packed
 
     def _internal_parse(self, csource):
-        ast, macros = self._parse(csource)
+        ast, macros, csource = self._parse(csource)
         # add the macros
-        for key, value in macros.items():
-            value = value.strip()
-            if value != '...':
-                raise api.CDefError('only supports the syntax "#define '
-                                    '%s ..." for now (literally)' % key)
-            self._declare('macro ' + key, value)
+        self._process_macros(macros)
         # find the first "__dotdotdot__" and use that as a separator
         # between the repeated typedefs and the real csource
         iterator = iter(ast.ext)
@@ -175,27 +179,61 @@
             if decl.name == '__dotdotdot__':
                 break
         #
-        for decl in iterator:
-            if isinstance(decl, pycparser.c_ast.Decl):
-                self._parse_decl(decl)
-            elif isinstance(decl, pycparser.c_ast.Typedef):
-                if not decl.name:
-                    raise api.CDefError("typedef does not declare any name",
-                                        decl)
-                if (isinstance(decl.type.type, pycparser.c_ast.IdentifierType)
-                        and decl.type.type.names == ['__dotdotdot__']):
-                    realtype = model.unknown_type(decl.name)
-                elif (isinstance(decl.type, pycparser.c_ast.PtrDecl) and
-                      isinstance(decl.type.type, pycparser.c_ast.TypeDecl) and
-                      isinstance(decl.type.type.type,
-                                 pycparser.c_ast.IdentifierType) and
-                      decl.type.type.type.names == ['__dotdotdot__']):
-                    realtype = model.unknown_ptr_type(decl.name)
+        try:
+            for decl in iterator:
+                if isinstance(decl, pycparser.c_ast.Decl):
+                    self._parse_decl(decl)
+                elif isinstance(decl, pycparser.c_ast.Typedef):
+                    if not decl.name:
+                        raise api.CDefError("typedef does not declare any name",
+                                            decl)
+                    if (isinstance(decl.type.type, pycparser.c_ast.IdentifierType)
+                            and decl.type.type.names == ['__dotdotdot__']):
+                        realtype = model.unknown_type(decl.name)
+                    elif (isinstance(decl.type, pycparser.c_ast.PtrDecl) and
+                          isinstance(decl.type.type, pycparser.c_ast.TypeDecl) and
+                          isinstance(decl.type.type.type,
+                                     pycparser.c_ast.IdentifierType) and
+                          decl.type.type.type.names == ['__dotdotdot__']):
+                        realtype = model.unknown_ptr_type(decl.name)
+                    else:
+                        realtype = self._get_type(decl.type, name=decl.name)
+                    self._declare('typedef ' + decl.name, realtype)
                 else:
-                    realtype = self._get_type(decl.type, name=decl.name)
-                self._declare('typedef ' + decl.name, realtype)
+                    raise api.CDefError("unrecognized construct", decl)
+        except api.FFIError as e:
+            msg = self._convert_pycparser_error(e, csource)
+            if msg:
+                e.args = (e.args[0] + "\n    *** Err: %s" % msg,)
+            raise
+
+    def _add_constants(self, key, val):
+        if key in self._int_constants:
+            raise api.FFIError(
+                "multiple declarations of constant: %s" % (key,))
+        self._int_constants[key] = val
+
+    def _process_macros(self, macros):
+        for key, value in macros.items():
+            value = value.strip()
+            match = _r_int_literal.search(value)
+            if match is not None:
+                int_str = match.group(0).lower().rstrip("ul")
+
+                # "010" is not valid oct in py3
+                if (int_str.startswith("0") and
+                        int_str != "0" and
+                        not int_str.startswith("0x")):
+                    int_str = "0o" + int_str[1:]
+
+                pyvalue = int(int_str, 0)
+                self._add_constants(key, pyvalue)
+            elif value == '...':
+                self._declare('macro ' + key, value)
             else:
-                raise api.CDefError("unrecognized construct", decl)
+                raise api.CDefError('only supports the syntax "#define '
+                                    '%s ..." (literally) or "#define '
+                                    '%s 0x1FF" for now' % (key, key))
 
     def _parse_decl(self, decl):
         node = decl.type
@@ -227,7 +265,7 @@
                     self._declare('variable ' + decl.name, tp)
 
     def parse_type(self, cdecl):
-        ast, macros = self._parse('void __dummy(\n%s\n);' % cdecl)
+        ast, macros = self._parse('void __dummy(\n%s\n);' % cdecl)[:2]
         assert not macros
         exprnode = ast.ext[-1].type.args.params[0]
         if isinstance(exprnode, pycparser.c_ast.ID):
@@ -306,7 +344,8 @@
                 if ident == 'void':
                     return model.void_type
                 if ident == '__dotdotdot__':
-                    raise api.FFIError('bad usage of "..."')
+                    raise api.FFIError(':%d: bad usage of "..."' %
+                            typenode.coord.line)
                 return resolve_common_type(ident)
             #
             if isinstance(type, pycparser.c_ast.Struct):
@@ -333,7 +372,8 @@
             return self._get_struct_union_enum_type('union', typenode, name,
                                                     nested=True)
         #
-        raise api.FFIError("bad or unsupported type declaration")
+        raise api.FFIError(":%d: bad or unsupported type declaration" %
+                typenode.coord.line)
 
     def _parse_function_type(self, typenode, funcname=None):
         params = list(getattr(typenode.args, 'params', []))
@@ -499,6 +539,10 @@
         if (isinstance(exprnode, pycparser.c_ast.UnaryOp) and
                 exprnode.op == '-'):
             return -self._parse_constant(exprnode.expr)
+        # load previously defined int constant
+        if (isinstance(exprnode, pycparser.c_ast.ID) and
+                exprnode.name in self._int_constants):
+            return self._int_constants[exprnode.name]
         #
         if partial_length_ok:
             if (isinstance(exprnode, pycparser.c_ast.ID) and
@@ -506,8 +550,8 @@
                 self._partial_length = True
                 return '...'
         #
-        raise api.FFIError("unsupported expression: expected a "
-                           "simple numeric constant")
+        raise api.FFIError(":%d: unsupported expression: expected a "
+                           "simple numeric constant" % exprnode.coord.line)
 
     def _build_enum_type(self, explicit_name, decls):
         if decls is not None:
@@ -522,6 +566,7 @@
                 if enum.value is not None:
                     nextenumvalue = self._parse_constant(enum.value)
                 enumvalues.append(nextenumvalue)
+                self._add_constants(enum.name, nextenumvalue)
                 nextenumvalue += 1
             enumvalues = tuple(enumvalues)
             tp = model.EnumType(explicit_name, enumerators, enumvalues)
@@ -535,3 +580,5 @@
             kind = name.split(' ', 1)[0]
             if kind in ('typedef', 'struct', 'union', 'enum'):
                 self._declare(name, tp)
+        for k, v in other._int_constants.items():
+            self._add_constants(k, v)
diff --git a/lib_pypy/cffi/ffiplatform.py b/lib_pypy/cffi/ffiplatform.py
--- a/lib_pypy/cffi/ffiplatform.py
+++ b/lib_pypy/cffi/ffiplatform.py
@@ -38,6 +38,7 @@
     import distutils.errors
     #
     dist = Distribution({'ext_modules': [ext]})
+    dist.parse_config_files()
     options = dist.get_option_dict('build_ext')
     options['force'] = ('ffiplatform', True)
     options['build_lib'] = ('ffiplatform', tmpdir)
diff --git a/lib_pypy/cffi/vengine_cpy.py b/lib_pypy/cffi/vengine_cpy.py
--- a/lib_pypy/cffi/vengine_cpy.py
+++ b/lib_pypy/cffi/vengine_cpy.py
@@ -89,43 +89,54 @@
         # by generate_cpy_function_method().
         prnt('static PyMethodDef _cffi_methods[] = {')
         self._generate("method")
-        prnt('  {"_cffi_setup", _cffi_setup, METH_VARARGS},')
-        prnt('  {NULL, NULL}    /* Sentinel */')
+        prnt('  {"_cffi_setup", _cffi_setup, METH_VARARGS, NULL},')
+        prnt('  {NULL, NULL, 0, NULL}    /* Sentinel */')
         prnt('};')
         prnt()
         #
         # standard init.
         modname = self.verifier.get_module_name()
-        if sys.version_info >= (3,):
-            prnt('static struct PyModuleDef _cffi_module_def = {')
-            prnt('  PyModuleDef_HEAD_INIT,')
-            prnt('  "%s",' % modname)
-            prnt('  NULL,')
-            prnt('  -1,')
-            prnt('  _cffi_methods,')
-            prnt('  NULL, NULL, NULL, NULL')
-            prnt('};')
-            prnt()
-            initname = 'PyInit_%s' % modname
-            createmod = 'PyModule_Create(&_cffi_module_def)'
-            errorcase = 'return NULL'
-            finalreturn = 'return lib'
-        else:
-            initname = 'init%s' % modname
-            createmod = 'Py_InitModule("%s", _cffi_methods)' % modname
-            errorcase = 'return'
-            finalreturn = 'return'
+        constants = self._chained_list_constants[False]
+        prnt('#if PY_MAJOR_VERSION >= 3')
+        prnt()
+        prnt('static struct PyModuleDef _cffi_module_def = {')
+        prnt('  PyModuleDef_HEAD_INIT,')
+        prnt('  "%s",' % modname)
+        prnt('  NULL,')
+        prnt('  -1,')
+        prnt('  _cffi_methods,')
+        prnt('  NULL, NULL, NULL, NULL')
+        prnt('};')
+        prnt()
         prnt('PyMODINIT_FUNC')
-        prnt('%s(void)' % initname)
+        prnt('PyInit_%s(void)' % modname)
         prnt('{')
         prnt('  PyObject *lib;')
-        prnt('  lib = %s;' % createmod)
-        prnt('  if (lib == NULL || %s < 0)' % (
-            self._chained_list_constants[False],))
-        prnt('    %s;' % errorcase)
-        prnt('  _cffi_init();')
-        prnt('  %s;' % finalreturn)
+        prnt('  lib = PyModule_Create(&_cffi_module_def);')
+        prnt('  if (lib == NULL)')
+        prnt('    return NULL;')
+        prnt('  if (%s < 0 || _cffi_init() < 0) {' % (constants,))
+        prnt('    Py_DECREF(lib);')
+        prnt('    return NULL;')
+        prnt('  }')
+        prnt('  return lib;')
         prnt('}')
+        prnt()
+        prnt('#else')
+        prnt()
+        prnt('PyMODINIT_FUNC')
+        prnt('init%s(void)' % modname)
+        prnt('{')
+        prnt('  PyObject *lib;')
+        prnt('  lib = Py_InitModule("%s", _cffi_methods);' % modname)
+        prnt('  if (lib == NULL)')
+        prnt('    return;')
+        prnt('  if (%s < 0 || _cffi_init() < 0)' % (constants,))
+        prnt('    return;')
+        prnt('  return;')
+        prnt('}')
+        prnt()
+        prnt('#endif')
 
     def load_library(self):
         # XXX review all usages of 'self' here!
@@ -394,7 +405,7 @@
             meth = 'METH_O'
         else:
             meth = 'METH_VARARGS'
-        self._prnt('  {"%s", _cffi_f_%s, %s},' % (name, name, meth))
+        self._prnt('  {"%s", _cffi_f_%s, %s, NULL},' % (name, name, meth))
 
     _loading_cpy_function = _loaded_noop
 
@@ -481,8 +492,8 @@
         if tp.fldnames is None:
             return     # nothing to do with opaque structs
         layoutfuncname = '_cffi_layout_%s_%s' % (prefix, name)
-        self._prnt('  {"%s", %s, METH_NOARGS},' % (layoutfuncname,
-                                                   layoutfuncname))
+        self._prnt('  {"%s", %s, METH_NOARGS, NULL},' % (layoutfuncname,
+                                                         layoutfuncname))
 
     def _loading_struct_or_union(self, tp, prefix, name, module):
         if tp.fldnames is None:
@@ -589,13 +600,7 @@
                                                           'variable type'),))
             assert delayed
         else:
-            prnt('  if (LONG_MIN <= (%s) && (%s) <= LONG_MAX)' % (name, name))
-            prnt('    o = PyInt_FromLong((long)(%s));' % (name,))
-            prnt('  else if ((%s) <= 0)' % (name,))
-            prnt('    o = PyLong_FromLongLong((long long)(%s));' % (name,))
-            prnt('  else')
-            prnt('    o = PyLong_FromUnsignedLongLong('
-                 '(unsigned long long)(%s));' % (name,))
+            prnt('  o = _cffi_from_c_int_const(%s);' % name)
         prnt('  if (o == NULL)')
         prnt('    return -1;')
         if size_too:
@@ -632,13 +637,18 @@
     # ----------
     # enums
 
+    def _enum_funcname(self, prefix, name):
+        # "$enum_$1" => "___D_enum____D_1"
+        name = name.replace('$', '___D_')
+        return '_cffi_e_%s_%s' % (prefix, name)
+
     def _generate_cpy_enum_decl(self, tp, name, prefix='enum'):
         if tp.partial:
             for enumerator in tp.enumerators:
                 self._generate_cpy_const(True, enumerator, delayed=False)
             return
         #
-        funcname = '_cffi_e_%s_%s' % (prefix, name)
+        funcname = self._enum_funcname(prefix, name)
         prnt = self._prnt
         prnt('static int %s(PyObject *lib)' % funcname)
         prnt('{')
@@ -760,17 +770,30 @@
 #include 
 #include 
 
-#ifdef MS_WIN32
-#include    /* for alloca() */
-typedef __int8 int8_t;
-typedef __int16 int16_t;
-typedef __int32 int32_t;
-typedef __int64 int64_t;
-typedef unsigned __int8 uint8_t;
-typedef unsigned __int16 uint16_t;
-typedef unsigned __int32 uint32_t;
-typedef unsigned __int64 uint64_t;
-typedef unsigned char _Bool;
+/* this block of #ifs should be kept exactly identical between
+   c/_cffi_backend.c, cffi/vengine_cpy.py, cffi/vengine_gen.py */
+#if defined(_MSC_VER)
+# include    /* for alloca() */
+# if _MSC_VER < 1600   /* MSVC < 2010 */
+   typedef __int8 int8_t;
+   typedef __int16 int16_t;
+   typedef __int32 int32_t;
+   typedef __int64 int64_t;
+   typedef unsigned __int8 uint8_t;
+   typedef unsigned __int16 uint16_t;
+   typedef unsigned __int32 uint32_t;
+   typedef unsigned __int64 uint64_t;
+# else
+#  include 
+# endif
+# if _MSC_VER < 1800   /* MSVC < 2013 */
+   typedef unsigned char _Bool;
+# endif
+#else
+# include 
+# if (defined (__SVR4) && defined (__sun)) || defined(_AIX)
+#  include 
+# endif
 #endif
 
 #if PY_MAJOR_VERSION < 3
@@ -795,6 +818,15 @@
 #define _cffi_to_c_double PyFloat_AsDouble
 #define _cffi_to_c_float PyFloat_AsDouble
 
+#define _cffi_from_c_int_const(x)                                        \
+    (((x) > 0) ?                                                         \
+        ((unsigned long long)(x) <= (unsigned long long)LONG_MAX) ?      \
+            PyInt_FromLong((long)(x)) :                                  \
+            PyLong_FromUnsignedLongLong((unsigned long long)(x)) :       \
+        ((long long)(x) >= (long long)LONG_MIN) ?                        \
+            PyInt_FromLong((long)(x)) :                                  \
+            PyLong_FromLongLong((long long)(x)))
+
 #define _cffi_from_c_int(x, type)                                        \
     (((type)-1) > 0 ?   /* unsigned */                                   \
         (sizeof(type) < sizeof(long) ? PyInt_FromLong(x) :               \
@@ -804,14 +836,14 @@
                                         PyLong_FromLongLong(x)))
 
 #define _cffi_to_c_int(o, type)                                          \
-    (sizeof(type) == 1 ? (((type)-1) > 0 ? _cffi_to_c_u8(o)              \
-                                         : _cffi_to_c_i8(o)) :           \
-     sizeof(type) == 2 ? (((type)-1) > 0 ? _cffi_to_c_u16(o)             \
-                                         : _cffi_to_c_i16(o)) :          \
-     sizeof(type) == 4 ? (((type)-1) > 0 ? _cffi_to_c_u32(o)             \
-                                         : _cffi_to_c_i32(o)) :          \
-     sizeof(type) == 8 ? (((type)-1) > 0 ? _cffi_to_c_u64(o)             \
-                                         : _cffi_to_c_i64(o)) :          \
+    (sizeof(type) == 1 ? (((type)-1) > 0 ? (type)_cffi_to_c_u8(o)        \
+                                         : (type)_cffi_to_c_i8(o)) :     \
+     sizeof(type) == 2 ? (((type)-1) > 0 ? (type)_cffi_to_c_u16(o)       \
+                                         : (type)_cffi_to_c_i16(o)) :    \
+     sizeof(type) == 4 ? (((type)-1) > 0 ? (type)_cffi_to_c_u32(o)       \
+                                         : (type)_cffi_to_c_i32(o)) :    \
+     sizeof(type) == 8 ? (((type)-1) > 0 ? (type)_cffi_to_c_u64(o)       \
+                                         : (type)_cffi_to_c_i64(o)) :    \
      (Py_FatalError("unsupported size for type " #type), 0))
 
 #define _cffi_to_c_i8                                                    \
@@ -885,25 +917,32 @@
     return PyBool_FromLong(was_alive);
 }
 
-static void _cffi_init(void)
+static int _cffi_init(void)
 {
-    PyObject *module = PyImport_ImportModule("_cffi_backend");
-    PyObject *c_api_object;
+    PyObject *module, *c_api_object = NULL;
 
+    module = PyImport_ImportModule("_cffi_backend");
     if (module == NULL)
-        return;
+        goto failure;
 
     c_api_object = PyObject_GetAttrString(module, "_C_API");
     if (c_api_object == NULL)
-        return;
+        goto failure;
     if (!PyCapsule_CheckExact(c_api_object)) {
-        Py_DECREF(c_api_object);
         PyErr_SetNone(PyExc_ImportError);
-        return;
+        goto failure;
     }
     memcpy(_cffi_exports, PyCapsule_GetPointer(c_api_object, "cffi"),
            _CFFI_NUM_EXPORTS * sizeof(void *));
+
+    Py_DECREF(module);
     Py_DECREF(c_api_object);
+    return 0;
+
+  failure:
+    Py_XDECREF(module);
+    Py_XDECREF(c_api_object);
+    return -1;
 }
 
 #define _cffi_type(num) ((CTypeDescrObject *)PyList_GET_ITEM(_cffi_types, num))
diff --git a/lib_pypy/cffi/vengine_gen.py b/lib_pypy/cffi/vengine_gen.py
--- a/lib_pypy/cffi/vengine_gen.py
+++ b/lib_pypy/cffi/vengine_gen.py
@@ -249,10 +249,10 @@
                     prnt('  /* %s */' % str(e))   # cannot verify it, ignore
         prnt('}')
         self.export_symbols.append(layoutfuncname)
-        prnt('ssize_t %s(ssize_t i)' % (layoutfuncname,))
+        prnt('intptr_t %s(intptr_t i)' % (layoutfuncname,))
         prnt('{')
         prnt('  struct _cffi_aligncheck { char x; %s y; };' % cname)
-        prnt('  static ssize_t nums[] = {')
+        prnt('  static intptr_t nums[] = {')
         prnt('    sizeof(%s),' % cname)
         prnt('    offsetof(struct _cffi_aligncheck, y),')
         for fname, ftype, fbitsize in tp.enumfields():
@@ -276,7 +276,7 @@
             return     # nothing to do with opaque structs
         layoutfuncname = '_cffi_layout_%s_%s' % (prefix, name)
         #
-        BFunc = self.ffi._typeof_locked("ssize_t(*)(ssize_t)")[0]
+        BFunc = self.ffi._typeof_locked("intptr_t(*)(intptr_t)")[0]
         function = module.load_function(BFunc, layoutfuncname)
         layout = []
         num = 0
@@ -410,13 +410,18 @@
     # ----------
     # enums
 
+    def _enum_funcname(self, prefix, name):
+        # "$enum_$1" => "___D_enum____D_1"
+        name = name.replace('$', '___D_')
+        return '_cffi_e_%s_%s' % (prefix, name)
+
     def _generate_gen_enum_decl(self, tp, name, prefix='enum'):
         if tp.partial:
             for enumerator in tp.enumerators:
                 self._generate_gen_const(True, enumerator)
             return
         #
-        funcname = '_cffi_e_%s_%s' % (prefix, name)
+        funcname = self._enum_funcname(prefix, name)
         self.export_symbols.append(funcname)
         prnt = self._prnt
         prnt('int %s(char *out_error)' % funcname)
@@ -430,14 +435,14 @@
                     enumerator, enumerator, enumvalue))
             prnt('    char buf[64];')
             prnt('    if ((%s) < 0)' % enumerator)
-            prnt('        snprintf(buf, 63, "%%ld", (long)(%s));' % enumerator)
+            prnt('        sprintf(buf, "%%ld", (long)(%s));' % enumerator)
             prnt('    else')
-            prnt('        snprintf(buf, 63, "%%lu", (unsigned long)(%s));' %
+            prnt('        sprintf(buf, "%%lu", (unsigned long)(%s));' %
                  enumerator)
-            prnt('    snprintf(out_error, 255,'
+            prnt('    sprintf(out_error,'
                              ' "%s has the real value %s, not %s",')
             prnt('            "%s", buf, "%d");' % (
-                enumerator, enumvalue))
+                enumerator[:100], enumvalue))
             prnt('    return -1;')
             prnt('  }')
         prnt('  return 0;')
@@ -453,7 +458,7 @@
         else:
             BType = self.ffi._typeof_locked("char[]")[0]
             BFunc = self.ffi._typeof_locked("int(*)(char*)")[0]
-            funcname = '_cffi_e_%s_%s' % (prefix, name)
+            funcname = self._enum_funcname(prefix, name)
             function = module.load_function(BFunc, funcname)
             p = self.ffi.new(BType, 256)
             if function(p) < 0:
@@ -547,20 +552,29 @@
 #include 
 #include    /* XXX for ssize_t on some platforms */
 
-#ifdef _WIN32
-#  include 
-#  define snprintf _snprintf
-typedef __int8 int8_t;
-typedef __int16 int16_t;
-typedef __int32 int32_t;
-typedef __int64 int64_t;
-typedef unsigned __int8 uint8_t;
-typedef unsigned __int16 uint16_t;
-typedef unsigned __int32 uint32_t;
-typedef unsigned __int64 uint64_t;
-typedef SSIZE_T ssize_t;
-typedef unsigned char _Bool;
+/* this block of #ifs should be kept exactly identical between
+   c/_cffi_backend.c, cffi/vengine_cpy.py, cffi/vengine_gen.py */
+#if defined(_MSC_VER)
+# include    /* for alloca() */
+# if _MSC_VER < 1600   /* MSVC < 2010 */
+   typedef __int8 int8_t;
+   typedef __int16 int16_t;
+   typedef __int32 int32_t;
+   typedef __int64 int64_t;
+   typedef unsigned __int8 uint8_t;
+   typedef unsigned __int16 uint16_t;
+   typedef unsigned __int32 uint32_t;
+   typedef unsigned __int64 uint64_t;
+# else
+#  include 
+# endif
+# if _MSC_VER < 1800   /* MSVC < 2013 */
+   typedef unsigned char _Bool;
+# endif
 #else
-#  include 
+# include 
+# if (defined (__SVR4) && defined (__sun)) || defined(_AIX)
+#  include 
+# endif
 #endif
 '''
diff --git a/lib_pypy/gdbm.py b/lib_pypy/gdbm.py
--- a/lib_pypy/gdbm.py
+++ b/lib_pypy/gdbm.py
@@ -50,6 +50,8 @@
     pass
 
 def _fromstr(key):
+    if isinstance(key, unicode):
+        key = key.encode("ascii")
     if not isinstance(key, str):
         raise TypeError("gdbm mappings have string indices only")
     return {'dptr': ffi.new("char[]", key), 'dsize': len(key)}
@@ -71,8 +73,8 @@
 
     def _raise_from_errno(self):
         if ffi.errno:
-            raise error(os.strerror(ffi.errno))
-        raise error(lib.gdbm_strerror(lib.gdbm_errno))
+            raise error(ffi.errno, os.strerror(ffi.errno))
+        raise error(lib.gdbm_errno, lib.gdbm_strerror(lib.gdbm_errno))
 
     def __len__(self):
         if self.size < 0:
@@ -141,7 +143,7 @@
 
     def _check_closed(self):
         if not self.ll_dbm:
-            raise error("GDBM object has already been closed")
+            raise error(0, "GDBM object has already been closed")
 
     __del__ = close
 
@@ -159,7 +161,7 @@
     elif flags[0] == 'n':
         iflags = lib.GDBM_NEWDB
     else:
-        raise error("First flag must be one of 'r', 'w', 'c' or 'n'")
+        raise error(0, "First flag must be one of 'r', 'w', 'c' or 'n'")
     for flag in flags[1:]:
         if flag == 'f':
             iflags |= lib.GDBM_FAST
@@ -168,7 +170,7 @@
         elif flag == 'u':
             iflags |= lib.GDBM_NOLOCK
         else:
-            raise error("Flag '%s' not supported" % flag)
+            raise error(0, "Flag '%s' not supported" % flag)
     return gdbm(filename, iflags, mode)
 
 open_flags = "rwcnfsu"
diff --git a/pypy/doc/ctypes-implementation.rst b/pypy/doc/ctypes-implementation.rst
--- a/pypy/doc/ctypes-implementation.rst
+++ b/pypy/doc/ctypes-implementation.rst
@@ -72,13 +72,11 @@
 Here is a list of the limitations and missing features of the
 current implementation:
 
-* ``ctypes.pythonapi`` lets you access the CPython C API emulation layer
-  of PyPy, at your own risks and without doing anything sensible about
-  the GIL.  Since PyPy 2.3, these functions are also named with an extra
-  "Py", for example ``PyPyInt_FromLong()``.  Basically, don't use this,
-  but it might more or less work in simple cases if you do.  (Obviously,
-  assuming the PyObject pointers you get have any particular fields in
-  any particular order is just going to crash.)
+* ``ctypes.pythonapi`` is missing.  In previous versions, it was present
+  and redirected to the `cpyext` C API emulation layer, but our
+  implementation did not do anything sensible about the GIL and the
+  functions were named with an extra "Py", for example
+  ``PyPyInt_FromLong()``.  It was removed for being unhelpful.
 
 * We copy Python strings instead of having pointers to raw buffers
 
diff --git a/pypy/doc/extradoc.rst b/pypy/doc/extradoc.rst
--- a/pypy/doc/extradoc.rst
+++ b/pypy/doc/extradoc.rst
@@ -8,6 +8,9 @@
 *Articles about PyPy published so far, most recent first:* (bibtex_ file)
 
 
+* `A Way Forward in Parallelising Dynamic Languages`_,
+  R. Meier, A. Rigo
+
 * `Runtime Feedback in a Meta-Tracing JIT for Efficient Dynamic Languages`_,
   C.F. Bolz, A. Cuni, M. Fijalkowski, M. Leuschel, S. Pedroni, A. Rigo
 
@@ -71,6 +74,7 @@
 
 
 .. _bibtex: https://bitbucket.org/pypy/extradoc/raw/tip/talk/bibtex.bib
+.. _`A Way Forward in Parallelising Dynamic Languages`: https://bitbucket.org/pypy/extradoc/raw/extradoc/talk/icooolps2014/position-paper.pdf
 .. _`Runtime Feedback in a Meta-Tracing JIT for Efficient Dynamic Languages`: https://bitbucket.org/pypy/extradoc/raw/extradoc/talk/icooolps2011/jit-hints.pdf
 .. _`Allocation Removal by Partial Evaluation in a Tracing JIT`: https://bitbucket.org/pypy/extradoc/raw/extradoc/talk/pepm2011/bolz-allocation-removal.pdf
 .. _`Towards a Jitting VM for Prolog Execution`: http://www.stups.uni-duesseldorf.de/mediawiki/images/a/a7/Pub-BoLeSch2010.pdf
@@ -93,6 +97,11 @@
 Talks and Presentations 
 ----------------------------------
 
+*This part is no longer updated.*  The complete list is here__ (in
+alphabetical order).
+
+.. __: https://bitbucket.org/pypy/extradoc/src/extradoc/talk/
+
 Talks in 2010
 +++++++++++++
 
diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst
--- a/pypy/doc/faq.rst
+++ b/pypy/doc/faq.rst
@@ -171,16 +171,21 @@
 You might be interested in our `benchmarking site`_ and our 
 `jit documentation`_.
 
-Note that the JIT has a very high warm-up cost, meaning that the
-programs are slow at the beginning.  If you want to compare the timings
-with CPython, even relatively simple programs need to run *at least* one
-second, preferrably at least a few seconds.  Large, complicated programs
-need even more time to warm-up the JIT.
+`Your tests are not a benchmark`_: tests tend to be slow under PyPy
+because they run exactly once; if they are good tests, they exercise
+various corner cases in your code.  This is a bad case for JIT
+compilers.  Note also that our JIT has a very high warm-up cost, meaning
+that any program is slow at the beginning.  If you want to compare the
+timings with CPython, even relatively simple programs need to run *at
+least* one second, preferrably at least a few seconds.  Large,
+complicated programs need even more time to warm-up the JIT.
 
 .. _`benchmarking site`: http://speed.pypy.org
 
 .. _`jit documentation`: jit/index.html
 
+.. _`your tests are not a benchmark`: http://alexgaynor.net/2013/jul/15/your-tests-are-not-benchmark/
+
 ---------------------------------------------------------------
 Couldn't the JIT dump and reload already-compiled machine code?
 ---------------------------------------------------------------
@@ -465,9 +470,13 @@
 
 This is documented (here__ and here__).  It needs 4 GB of RAM to run
 "rpython targetpypystandalone" on top of PyPy, a bit more when running
-on CPython.  If you have less than 4 GB it will just swap forever (or
-fail if you don't have enough swap).  On 32-bit, divide the numbers by
-two.
+on top of CPython.  If you have less than 4 GB free, it will just swap
+forever (or fail if you don't have enough swap).  And we mean *free:*
+if the machine has 4 GB *in total,* then it will swap.
+
+On 32-bit, divide the numbers by two.  (We didn't try recently, but in
+the past it was possible to compile a 32-bit version on a 2 GB Linux
+machine with nothing else running: no Gnome/KDE, for example.)
 
 .. __: http://pypy.org/download.html#building-from-source
 .. __: https://pypy.readthedocs.org/en/latest/getting-started-python.html#translating-the-pypy-python-interpreter
diff --git a/pypy/doc/jit-hooks.rst b/pypy/doc/jit-hooks.rst
--- a/pypy/doc/jit-hooks.rst
+++ b/pypy/doc/jit-hooks.rst
@@ -34,7 +34,7 @@
     aborted due to some reason.
 
     The hook will be invoked with the siagnture:
-    ``hook(jitdriver_name, greenkey, reason)``
+    ``hook(jitdriver_name, greenkey, reason, oplist)``
 
     Reason is a string, the meaning of other arguments is the same
     as attributes on JitLoopInfo object
diff --git a/pypy/doc/objspace-proxies.rst b/pypy/doc/objspace-proxies.rst
--- a/pypy/doc/objspace-proxies.rst
+++ b/pypy/doc/objspace-proxies.rst
@@ -26,6 +26,16 @@
 Transparent Proxies
 ================================
 
+.. warning::
+
+    This is a feature that was tried experimentally long ago, and we
+    found no really good use cases.  The basic functionality is still
+    there, but we don't recommend using it.  Some of the examples below
+    might not work any more (e.g. you can't tproxy a list object any
+    more).  The rest can be done by hacking in standard Python.  If
+    anyone is interested in working on tproxy again, he is welcome, but
+    we don't regard this as an interesting extension.
+
 PyPy's Transparent Proxies allow routing of operations on objects
 to a callable.  Application level code can customize objects without
 interfering with the type system - ``type(proxied_list) is list`` holds true
diff --git a/pypy/doc/stm.rst b/pypy/doc/stm.rst
--- a/pypy/doc/stm.rst
+++ b/pypy/doc/stm.rst
@@ -28,7 +28,8 @@
 Introduction
 ============
 
-``pypy-stm`` is a variant of the regular PyPy interpreter.  With caveats_
+``pypy-stm`` is a variant of the regular PyPy interpreter.  (This
+version supports Python 2.7; see below for `Python 3`_.)  With caveats_
 listed below, it should be in theory within 20%-50% slower than a
 regular PyPy, comparing the JIT version in both cases (but see below!).
 It is called
@@ -92,9 +93,9 @@
   We're busy fixing them as we find them; feel free to `report bugs`_.
 
 * It runs with an overhead as low as 20% on examples like "richards".
-  There are also other examples with higher overheads --up to 10x for
-  "translate.py"-- which we are still trying to understand.  One suspect
-  is our partial GC implementation, see below.
+  There are also other examples with higher overheads --currently up to
+  2x for "translate.py"-- which we are still trying to understand.
+  One suspect is our partial GC implementation, see below.
 
 * Currently limited to 1.5 GB of RAM (this is just a parameter in
   `core.h`__).  Memory overflows are not correctly handled; they cause
@@ -111,9 +112,8 @@
 
 * The GC is new; although clearly inspired by PyPy's regular GC, it
   misses a number of optimizations for now.  Programs allocating large
-  numbers of small objects that don't immediately die, as well as
-  programs that modify large lists or dicts, suffer from these missing
-  optimizations.
+  numbers of small objects that don't immediately die (surely a common
+  situation) suffer from these missing optimizations.
 
 * The GC has no support for destructors: the ``__del__`` method is never
   called (including on file objects, which won't be closed for you).
@@ -138,6 +138,25 @@
 
 
 
+Python 3
+========
+
+In this document I describe "pypy-stm", which is based on PyPy's Python
+2.7 interpreter.  Supporting Python 3 should take about half an
+afternoon of work.  Obviously, what I *don't* mean is that by tomorrow
+you can have a finished and polished "pypy3-stm" product.  General py3k
+work is still missing; and general stm work is also still missing.  But
+they are rather independent from each other, as usual in PyPy.  The
+required afternoon of work will certainly be done one of these days now
+that the internal interfaces seem to stabilize.
+
+The same is true for other languages implemented in the RPython
+framework, although the amount of work to put there might vary, because
+the STM framework within RPython is currently targeting the PyPy
+interpreter and other ones might have slightly different needs.
+
+
+
 User Guide
 ==========
   
@@ -490,8 +509,6 @@
 The last two lines are special; they are an internal marker read by
 ``transactional_memory.print_abort_info()``.
 
-These statistics are not printed out for the main thread, for now.
-
 
 Reference to implementation details
 -----------------------------------
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -22,3 +22,31 @@
 conditional_calls). I would expect the net result to be a slight
 slow-down on some simple benchmarks and a speed-up on bigger
 programs.
+
+.. branch: ec-threadlocal
+Change the executioncontext's lookup to be done by reading a thread-
+local variable (which is implemented in C using '__thread' if
+possible, and pthread_getspecific() otherwise). On Linux x86 and
+x86-64, the JIT backend has a special optimization that lets it emit
+directly a single MOV from a %gs- or %fs-based address. It seems
+actually to give a good boost in performance.
+
+.. branch: fast-gil
+A faster way to handle the GIL, particularly in JIT code. The GIL is
+now a composite of two concepts: a global number (it's just set from
+1 to 0 and back around CALL_RELEASE_GIL), and a real mutex. If there
+are threads waiting to acquire the GIL, one of them is actively
+checking the global number every 0.1 ms to 1 ms.  Overall, JIT loops
+full of external function calls now run a bit faster (if no thread was
+started yet), or a *lot* faster (if threads were started already).
+
+.. branch: jit-get-errno
+Optimize the errno handling in the JIT, notably around external
+function calls. Linux-only.
+
+.. branch: disable_pythonapi
+Remove non-functioning ctypes.pyhonapi and ctypes.PyDLL, document this
+incompatibility with cpython. Recast sys.dllhandle to an int.
+
+.. branch: scalar-operations
+Fix performance regression on ufunc(, ) in numpy.
diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst
--- a/pypy/doc/windows.rst
+++ b/pypy/doc/windows.rst
@@ -132,19 +132,23 @@
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Download http://www.gzip.org/zlib/zlib-1.2.3.tar.gz and extract it in
-the base directory.  Then compile::
+the base directory.  Then compile as a static library::
 
     cd zlib-1.2.3
     nmake -f win32\Makefile.msc
-    copy zlib1.dll \zlib.dll
+    copy zlib1.lib 
+    copy zlib.h zconf.h 
 
 The bz2 compression library
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Get the same version of bz2 used by python and compile as a static library::
 
     svn export http://svn.python.org/projects/external/bzip2-1.0.6
     cd bzip2-1.0.6
     nmake -f makefile.msc
-    copy bzip.dll \bzip.dll
+    copy libbz2.lib 
+    copy bzlib.h 
+
     
 The sqlite3 database library
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -166,7 +170,8 @@
 is actually enough for pypy).
 
 Then, copy the file ``win32\bin\release\libexpat.dll`` somewhere in
-your PATH.
+your PATH, ``win32\bin\release\libexpat.lib`` somewhere in LIB, and
+both ``lib\expat.h`` and ``lib\expat_external.h`` somewhere in INCLUDE.
 
 The OpenSSL library
 ~~~~~~~~~~~~~~~~~~~
diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -30,8 +30,6 @@
     if w_dict is not None: # for tests
         w_entry_point = space.getitem(w_dict, space.wrap('entry_point'))
         w_run_toplevel = space.getitem(w_dict, space.wrap('run_toplevel'))
-        w_call_finish_gateway = space.wrap(gateway.interp2app(call_finish))
-        w_call_startup_gateway = space.wrap(gateway.interp2app(call_startup))
         withjit = space.config.objspace.usemodules.pypyjit
 
     def entry_point(argv):
@@ -53,7 +51,7 @@
             argv = argv[:1] + argv[3:]
         try:
             try:
-                space.call_function(w_run_toplevel, w_call_startup_gateway)
+                space.startup()
                 w_executable = space.wrap(argv[0])
                 w_argv = space.newlist([space.wrap(s) for s in argv[1:]])
                 w_exitcode = space.call_function(w_entry_point, w_executable, w_argv)
@@ -69,7 +67,7 @@
                 return 1
         finally:
             try:
-                space.call_function(w_run_toplevel, w_call_finish_gateway)
+                space.finish()
             except OperationError, e:
                 debug("OperationError:")
                 debug(" operror-type: " + e.w_type.getname(space))
@@ -184,11 +182,6 @@
                          'pypy_thread_attach': pypy_thread_attach,
                          'pypy_setup_home': pypy_setup_home}
 
-def call_finish(space):
-    space.finish()
-
-def call_startup(space):
-    space.startup()
 
 # _____ Define and setup target ___
 
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -395,6 +395,7 @@
 
     def startup(self):
         # To be called before using the space
+        self.threadlocals.enter_thread(self)
 
         # Initialize already imported builtin modules
         from pypy.interpreter.module import Module
@@ -639,30 +640,36 @@
         """NOT_RPYTHON: Abstract method that should put some minimal
         content into the w_builtins."""
 
-    @jit.loop_invariant
     def getexecutioncontext(self):
         "Return what we consider to be the active execution context."
         # Important: the annotator must not see a prebuilt ExecutionContext:
         # you should not see frames while you translate
         # so we make sure that the threadlocals never *have* an
         # ExecutionContext during translation.
-        if self.config.translating and not we_are_translated():
-            assert self.threadlocals.getvalue() is None, (
-                "threadlocals got an ExecutionContext during translation!")
-            try:
-                return self._ec_during_translation
-            except AttributeError:
-                ec = self.createexecutioncontext()
-                self._ec_during_translation = ec
+        if not we_are_translated():
+            if self.config.translating:
+                assert self.threadlocals.get_ec() is None, (
+                    "threadlocals got an ExecutionContext during translation!")
+                try:
+                    return self._ec_during_translation
+                except AttributeError:
+                    ec = self.createexecutioncontext()
+                    self._ec_during_translation = ec
+                    return ec
+            else:
+                ec = self.threadlocals.get_ec()
+                if ec is None:
+                    self.threadlocals.enter_thread(self)
+                    ec = self.threadlocals.get_ec()
                 return ec
-        # normal case follows.  The 'thread' module installs a real
-        # thread-local object in self.threadlocals, so this builds
-        # and caches a new ec in each thread.
-        ec = self.threadlocals.getvalue()
-        if ec is None:
-            ec = self.createexecutioncontext()
-            self.threadlocals.setvalue(ec)
-        return ec
+        else:
+            # translated case follows.  self.threadlocals is either from
+            # 'pypy.interpreter.miscutils' or 'pypy.module.thread.threadlocals'.
+            # the result is assumed to be non-null: enter_thread() was called
+            # by space.startup().
+            ec = self.threadlocals.get_ec()
+            assert ec is not None
+            return ec
 
     def _freeze_(self):
         return True
@@ -963,6 +970,13 @@
         """
         return self.unpackiterable(w_iterable, expected_length)
 
+    def listview_no_unpack(self, w_iterable):
+        """ Same as listview() if cheap.  If 'w_iterable' is something like
+        a generator, for example, then return None instead.
+        May return None anyway.
+        """
+        return None
+
     def listview_bytes(self, w_list):
         """ Return a list of unwrapped strings out of a list of strings. If the
         argument is not a list or does not contain only strings, return None.
@@ -1487,9 +1501,7 @@
             return buf.as_str()
 
     def str_or_None_w(self, w_obj):
-        if self.is_w(w_obj, self.w_None):
-            return None
-        return self.str_w(w_obj)
+        return None if self.is_none(w_obj) else self.str_w(w_obj)
 
     def str_w(self, w_obj):
         return w_obj.str_w(self)
diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py
--- a/pypy/interpreter/executioncontext.py
+++ b/pypy/interpreter/executioncontext.py
@@ -496,6 +496,13 @@
     """
 
 
+class UserDelCallback(object):
+    def __init__(self, w_obj, callback, descrname):
+        self.w_obj = w_obj
+        self.callback = callback
+        self.descrname = descrname
+        self.next = None
+
 class UserDelAction(AsyncAction):
     """An action that invokes all pending app-level __del__() method.
     This is done as an action instead of immediately when the
@@ -506,12 +513,18 @@
 
     def __init__(self, space):
         AsyncAction.__init__(self, space)
-        self.dying_objects = []
+        self.dying_objects = None
+        self.dying_objects_last = None
         self.finalizers_lock_count = 0
         self.enabled_at_app_level = True
 
     def register_callback(self, w_obj, callback, descrname):
-        self.dying_objects.append((w_obj, callback, descrname))
+        cb = UserDelCallback(w_obj, callback, descrname)
+        if self.dying_objects_last is None:
+            self.dying_objects = cb
+        else:
+            self.dying_objects_last.next = cb
+        self.dying_objects_last = cb
         self.fire()
 
     def perform(self, executioncontext, frame):
@@ -525,13 +538,33 @@
         # avoid too deep recursions of the kind of __del__ being called
         # while in the middle of another __del__ call.
         pending = self.dying_objects
-        self.dying_objects = []
+        self.dying_objects = None
+        self.dying_objects_last = None
         space = self.space
-        for i in range(len(pending)):
-            w_obj, callback, descrname = pending[i]
-            pending[i] = (None, None, None)
+        while pending is not None:
             try:
-                callback(w_obj)
+                pending.callback(pending.w_obj)
             except OperationError, e:
-                e.write_unraisable(space, descrname, w_obj)
+                e.write_unraisable(space, pending.descrname, pending.w_obj)
                 e.clear(space)   # break up reference cycles
+            pending = pending.next
+        #
+        # Note: 'dying_objects' used to be just a regular list instead
+        # of a chained list.  This was the cause of "leaks" if we have a
+        # program that constantly creates new objects with finalizers.
+        # Here is why: say 'dying_objects' is a long list, and there
+        # are n instances in it.  Then we spend some time in this
+        # function, possibly triggering more GCs, but keeping the list
+        # of length n alive.  Then the list is suddenly freed at the
+        # end, and we return to the user program.  At this point the
+        # GC limit is still very high, because just before, there was
+        # a list of length n alive.  Assume that the program continues
+        # to allocate a lot of instances with finalizers.  The high GC
+        # limit means that it could allocate a lot of instances before
+        # reaching it --- possibly more than n.  So the whole procedure
+        # repeats with higher and higher values of n.
+        #
+        # This does not occur in the current implementation because
+        # there is no list of length n: if n is large, then the GC
+        # will run several times while walking the list, but it will
+        # see lower and lower memory usage, with no lower bound of n.
diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py
--- a/pypy/interpreter/gateway.py
+++ b/pypy/interpreter/gateway.py
@@ -895,7 +895,7 @@
                     "use unwrap_spec(...=WrappedDefault(default))" % (
                     self._code.identifier, name, defaultval))
                 defs_w.append(None)
-            else:
+            elif name != '__args__' and name != 'args_w':
                 defs_w.append(space.wrap(defaultval))
         if self._code._unwrap_spec:
             UNDEFINED = object()
diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -61,6 +61,13 @@
         return self.send_ex(w_arg)
 
     def send_ex(self, w_arg, operr=None):
+        pycode = self.pycode
+        if jit.we_are_jitted() and should_not_inline(pycode):
+            generatorentry_driver.jit_merge_point(gen=self, w_arg=w_arg,
+                                                  operr=operr, pycode=pycode)
+        return self._send_ex(w_arg, operr)
+
+    def _send_ex(self, w_arg, operr):
         space = self.space
         if self.running:
             raise OperationError(space.w_ValueError,
@@ -72,8 +79,7 @@
             if operr is None:
                 operr = OperationError(space.w_StopIteration, space.w_None)
             raise operr
-        # XXX it's not clear that last_instr should be promoted at all
-        # but as long as it is necessary for call_assembler, let's do it early
+
         last_instr = jit.promote(frame.last_instr)
         if last_instr == -1:
             if w_arg and not space.is_w(w_arg, space.w_None):
@@ -214,3 +220,38 @@
                                                  "interrupting generator of ")
                     break
                 block = block.previous
+
+
+
+def get_printable_location_genentry(bytecode):
+    return '%s ' % (bytecode.get_repr(),)
+generatorentry_driver = jit.JitDriver(greens=['pycode'],
+                                      reds=['gen', 'w_arg', 'operr'],
+                                      get_printable_location =
+                                          get_printable_location_genentry,
+                                      name='generatorentry')
+
+from pypy.tool.stdlib_opcode import HAVE_ARGUMENT, opmap
+YIELD_VALUE = opmap['YIELD_VALUE']
+
+ at jit.elidable_promote()
+def should_not_inline(pycode):
+    # Should not inline generators with more than one "yield",
+    # as an approximative fix (see issue #1782).  There are cases
+    # where it slows things down; for example calls to a simple
+    # generator that just produces a few simple values with a few
+    # consecutive "yield" statements.  It fixes the near-infinite
+    # slow-down in issue #1782, though...
+    count_yields = 0
+    code = pycode.co_code
+    n = len(code)
+    i = 0
+    while i < n:
+        c = code[i]
+        op = ord(c)
+        if op == YIELD_VALUE:
+            count_yields += 1
+        i += 1
+        if op >= HAVE_ARGUMENT:
+            i += 2
+    return count_yields >= 2
diff --git a/pypy/interpreter/miscutils.py b/pypy/interpreter/miscutils.py
--- a/pypy/interpreter/miscutils.py
+++ b/pypy/interpreter/miscutils.py
@@ -11,11 +11,14 @@
     """
     _value = None
 
-    def getvalue(self):
+    def get_ec(self):
         return self._value
 
-    def setvalue(self, value):
-        self._value = value
+    def enter_thread(self, space):
+        self._value = space.createexecutioncontext()
+
+    def try_enter_thread(self, space):
+        return False
 
     def signals_enabled(self):
         return True
diff --git a/pypy/interpreter/test/test_gateway.py b/pypy/interpreter/test/test_gateway.py
--- a/pypy/interpreter/test/test_gateway.py
+++ b/pypy/interpreter/test/test_gateway.py
@@ -726,6 +726,22 @@
             never_called
         py.test.raises(AssertionError, space.wrap, gateway.interp2app_temp(g))
 
+    def test_unwrap_spec_default_applevel_bug2(self):
+        space = self.space
+        def g(space, w_x, w_y=None, __args__=None):
+            return w_x
+        w_g = space.wrap(gateway.interp2app_temp(g))
+        w_42 = space.call_function(w_g, space.wrap(42))
+        assert space.int_w(w_42) == 42
+        py.test.raises(gateway.OperationError, space.call_function, w_g)
+        #
+        def g(space, w_x, w_y=None, args_w=None):
+            return w_x
+        w_g = space.wrap(gateway.interp2app_temp(g))
+        w_42 = space.call_function(w_g, space.wrap(42))
+        assert space.int_w(w_42) == 42
+        py.test.raises(gateway.OperationError, space.call_function, w_g)
+
     def test_interp2app_doc(self):
         space = self.space
         def f(space, w_x):
diff --git a/pypy/interpreter/test/test_generator.py b/pypy/interpreter/test/test_generator.py
--- a/pypy/interpreter/test/test_generator.py
+++ b/pypy/interpreter/test/test_generator.py
@@ -17,7 +17,7 @@
             yield 1
             assert g.gi_running
         g = f()
-        assert g.gi_code is f.func_code
+        assert g.gi_code is f.__code__
         assert g.__name__ == 'f'
         assert g.gi_frame is not None
         assert not g.gi_running
@@ -26,7 +26,7 @@
         raises(StopIteration, g.next)
         assert not g.gi_running
         assert g.gi_frame is None
-        assert g.gi_code is f.func_code
+        assert g.gi_code is f.__code__
         assert g.__name__ == 'f'
 
     def test_generator3(self):
@@ -278,4 +278,21 @@
         def f():
             yield 1
             raise StopIteration
-        assert tuple(f()) == (1,)
\ No newline at end of file
+        assert tuple(f()) == (1,)
+
+
+def test_should_not_inline(space):
+    from pypy.interpreter.generator import should_not_inline
+    w_co = space.appexec([], '''():
+        def g(x):
+            yield x + 5
+        return g.__code__
+    ''')
+    assert should_not_inline(w_co) == False
+    w_co = space.appexec([], '''():
+        def g(x):
+            yield x + 5
+            yield x + 6
+        return g.__code__
+    ''')
+    assert should_not_inline(w_co) == True
diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -73,13 +73,12 @@
         'builtinify'                : 'interp_magic.builtinify',
         'lookup_special'            : 'interp_magic.lookup_special',
         'do_what_I_mean'            : 'interp_magic.do_what_I_mean',
-        'list_strategy'             : 'interp_magic.list_strategy',
         'validate_fd'               : 'interp_magic.validate_fd',
         'resizelist_hint'           : 'interp_magic.resizelist_hint',
         'newlist_hint'              : 'interp_magic.newlist_hint',
         'add_memory_pressure'       : 'interp_magic.add_memory_pressure',
         'newdict'                   : 'interp_dict.newdict',
-        'dictstrategy'              : 'interp_dict.dictstrategy',
+        'strategy'                  : 'interp_magic.strategy',  # dict,set,list
         'set_debug'                 : 'interp_magic.set_debug',
         'locals_to_fast'            : 'interp_magic.locals_to_fast',
     }
diff --git a/pypy/module/__pypy__/interp_dict.py b/pypy/module/__pypy__/interp_dict.py
--- a/pypy/module/__pypy__/interp_dict.py
+++ b/pypy/module/__pypy__/interp_dict.py
@@ -1,7 +1,6 @@
 
 from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import unwrap_spec
-from pypy.objspace.std.dictmultiobject import W_DictMultiObject
 
 @unwrap_spec(type=str)
 def newdict(space, type):
@@ -31,13 +30,3 @@
         return space.newdict(strdict=True)
     else:
         raise oefmt(space.w_TypeError, "unknown type of dict %s", type)
-
-def dictstrategy(space, w_obj):
-    """ dictstrategy(dict)
-
-    show the underlaying strategy used by a dict object
-    """
-    if not isinstance(w_obj, W_DictMultiObject):
-        raise OperationError(space.w_TypeError,
-                             space.wrap("expecting dict object"))
-    return space.wrap('%r' % (w_obj.strategy,))
diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py
--- a/pypy/module/__pypy__/interp_magic.py
+++ b/pypy/module/__pypy__/interp_magic.py
@@ -2,7 +2,9 @@
 from pypy.interpreter.gateway import unwrap_spec
 from pypy.interpreter.pyframe import PyFrame
 from rpython.rlib.objectmodel import we_are_translated
+from pypy.objspace.std.dictmultiobject import W_DictMultiObject
 from pypy.objspace.std.listobject import W_ListObject
+from pypy.objspace.std.setobject import W_BaseSetObject
 from pypy.objspace.std.typeobject import MethodCache
 from pypy.objspace.std.mapdict import MapAttrCache
 from rpython.rlib import rposix, rgc
@@ -70,12 +72,23 @@
 def do_what_I_mean(space):
     return space.wrap(42)
 
-def list_strategy(space, w_list):
-    if isinstance(w_list, W_ListObject):
-        return space.wrap(w_list.strategy._applevel_repr)
+
+def strategy(space, w_obj):
+    """ strategy(dict or list or set)
+
+    Return the underlying strategy currently used by a dict, list or set object
+    """
+    if isinstance(w_obj, W_DictMultiObject):
+        name = w_obj.strategy.__class__.__name__
+    elif isinstance(w_obj, W_ListObject):
+        name = w_obj.strategy.__class__.__name__
+    elif isinstance(w_obj, W_BaseSetObject):
+        name = w_obj.strategy.__class__.__name__
     else:
-        w_msg = space.wrap("Can only get the list strategy of a list")
-        raise OperationError(space.w_TypeError, w_msg)
+        raise OperationError(space.w_TypeError,
+                             space.wrap("expecting dict or list or set object"))
+    return space.wrap(name)
+
 
 @unwrap_spec(fd='c_int')
 def validate_fd(space, fd):
diff --git a/pypy/module/__pypy__/test/test_special.py b/pypy/module/__pypy__/test/test_special.py
--- a/pypy/module/__pypy__/test/test_special.py
+++ b/pypy/module/__pypy__/test/test_special.py
@@ -46,26 +46,42 @@
         assert x == 42
 
     def test_list_strategy(self):
-        from __pypy__ import list_strategy
+        from __pypy__ import strategy
 
         l = [1, 2, 3]
-        assert list_strategy(l) == "int"
+        assert strategy(l) == "IntegerListStrategy"
         l = ["a", "b", "c"]
-        assert list_strategy(l) == "bytes"
+        assert strategy(l) == "BytesListStrategy"
         l = [u"a", u"b", u"c"]
-        assert list_strategy(l) == "unicode"
+        assert strategy(l) == "UnicodeListStrategy"
         l = [1.1, 2.2, 3.3]
-        assert list_strategy(l) == "float"
+        assert strategy(l) == "FloatListStrategy"
         l = range(3)
-        assert list_strategy(l) == "simple_range"
+        assert strategy(l) == "SimpleRangeListStrategy"
         l = range(1, 2)
-        assert list_strategy(l) == "range"
+        assert strategy(l) == "RangeListStrategy"
         l = [1, "b", 3]
-        assert list_strategy(l) == "object"
+        assert strategy(l) == "ObjectListStrategy"
         l = []
-        assert list_strategy(l) == "empty"
+        assert strategy(l) == "EmptyListStrategy"
         o = 5
-        raises(TypeError, list_strategy, 5)
+        raises(TypeError, strategy, 5)
+
+    def test_dict_strategy(self):
+        from __pypy__ import strategy
+
+        d = {}
+        assert strategy(d) == "EmptyDictStrategy"
+        d = {1: None, 5: None}
+        assert strategy(d) == "IntDictStrategy"
+
+    def test_set_strategy(self):
+        from __pypy__ import strategy
+
+        s = set()
+        assert strategy(s) == "EmptySetStrategy"
+        s = set([2, 3, 4])
+        assert strategy(s) == "IntegerSetStrategy"
 
 
 class AppTestJitFeatures(object):
diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py
--- a/pypy/module/_cffi_backend/__init__.py
+++ b/pypy/module/_cffi_backend/__init__.py
@@ -8,7 +8,7 @@
     appleveldefs = {
         }
     interpleveldefs = {
-        '__version__': 'space.wrap("0.8.2")',
+        '__version__': 'space.wrap("0.8.6")',
 
         'load_library': 'libraryobj.load_library',
 
diff --git a/pypy/module/_cffi_backend/ccallback.py b/pypy/module/_cffi_backend/ccallback.py
--- a/pypy/module/_cffi_backend/ccallback.py
+++ b/pypy/module/_cffi_backend/ccallback.py
@@ -183,9 +183,12 @@
         misc._raw_memclear(ll_res, SIZE_OF_FFI_ARG)
         return
     #
+    must_leave = False
     ec = None
+    space = callback.space
     try:
-        ec = cerrno.get_errno_container(callback.space)
+        must_leave = space.threadlocals.try_enter_thread(space)
+        ec = cerrno.get_errno_container(space)
         cerrno.save_errno_into(ec, e)
         extra_line = ''
         try:
@@ -206,5 +209,7 @@
         except OSError:
             pass
         callback.write_error_return_value(ll_res)
+    if must_leave:
+        space.threadlocals.leave_thread(space)
     if ec is not None:
         cerrno.restore_errno_from(ec)
diff --git a/pypy/module/_cffi_backend/ctypefunc.py b/pypy/module/_cffi_backend/ctypefunc.py
--- a/pypy/module/_cffi_backend/ctypefunc.py
+++ b/pypy/module/_cffi_backend/ctypefunc.py
@@ -4,7 +4,7 @@
 
 import sys
 
-from rpython.rlib import jit, clibffi, jit_libffi
+from rpython.rlib import jit, clibffi, jit_libffi, rgc
 from rpython.rlib.jit_libffi import (CIF_DESCRIPTION, CIF_DESCRIPTION_P,
     FFI_TYPE, FFI_TYPE_P, FFI_TYPE_PP, SIZE_OF_FFI_ARG)
 from rpython.rlib.objectmodel import we_are_translated, instantiate
@@ -63,6 +63,7 @@
         CifDescrBuilder(fvarargs, self.ctitem).rawallocate(ctypefunc)
         return ctypefunc
 
+    @rgc.must_be_light_finalizer
     def __del__(self):
         if self.cif_descr:
             lltype.free(self.cif_descr, flavor='raw')
@@ -156,8 +157,8 @@
                     data = rffi.ptradd(buffer, cif_descr.exchange_args[i])
                     flag = get_mustfree_flag(data)
                     if flag == 1:
-                        raw_string = rffi.cast(rffi.CCHARPP, data)[0]
-                        lltype.free(raw_string, flavor='raw')
+                        raw_cdata = rffi.cast(rffi.CCHARPP, data)[0]
+                        lltype.free(raw_cdata, flavor='raw')
             lltype.free(buffer, flavor='raw')
         return w_res
 
diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py
--- a/pypy/module/_cffi_backend/test/_backend_test_c.py
+++ b/pypy/module/_cffi_backend/test/_backend_test_c.py
@@ -3188,4 +3188,4 @@
 
 def test_version():
     # this test is here mostly for PyPy
-    assert __version__ == "0.8.2"
+    assert __version__ == "0.8.6"
diff --git a/pypy/module/_io/interp_textio.py b/pypy/module/_io/interp_textio.py
--- a/pypy/module/_io/interp_textio.py
+++ b/pypy/module/_io/interp_textio.py
@@ -458,6 +458,10 @@
         self._check_init(space)
         return space.call_method(self.w_buffer, "seekable")
 
+    def isatty_w(self, space):
+        self._check_init(space)
+        return space.call_method(self.w_buffer, "isatty")
+
     def fileno_w(self, space):
         self._check_init(space)
         return space.call_method(self.w_buffer, "fileno")
@@ -1035,6 +1039,7 @@
     readable = interp2app(W_TextIOWrapper.readable_w),
     writable = interp2app(W_TextIOWrapper.writable_w),
     seekable = interp2app(W_TextIOWrapper.seekable_w),
+    isatty = interp2app(W_TextIOWrapper.isatty_w),
     fileno = interp2app(W_TextIOWrapper.fileno_w),
     name = GetSetProperty(W_TextIOWrapper.name_get_w),
     buffer = interp_attrproperty_w("w_buffer", cls=W_TextIOWrapper),
diff --git a/pypy/module/_io/test/test_textio.py b/pypy/module/_io/test/test_textio.py
--- a/pypy/module/_io/test/test_textio.py
+++ b/pypy/module/_io/test/test_textio.py
@@ -25,6 +25,12 @@
         t = _io.TextIOWrapper(b)
         assert t.readable()
         assert t.seekable()
+        #
+        class CustomFile(object):
+            def isatty(self): return 'YES'
+            readable = writable = seekable = lambda self: False
+        t = _io.TextIOWrapper(CustomFile())
+        assert t.isatty() == 'YES'
 
     def test_default_implementations(self):
         import _io
diff --git a/pypy/module/_lsprof/interp_lsprof.py b/pypy/module/_lsprof/interp_lsprof.py
--- a/pypy/module/_lsprof/interp_lsprof.py
+++ b/pypy/module/_lsprof/interp_lsprof.py
@@ -11,7 +11,7 @@
 from rpython.rlib.rtimer import read_timestamp, _is_64_bit
 from rpython.rtyper.lltypesystem import rffi, lltype
 from rpython.translator.tool.cbuild import ExternalCompilationInfo
-from rpython.conftest import cdir
+from rpython.translator import cdir
 from rpython.rlib.rarithmetic import r_longlong
 
 import time, sys
diff --git a/pypy/module/_rawffi/interp_rawffi.py b/pypy/module/_rawffi/interp_rawffi.py
--- a/pypy/module/_rawffi/interp_rawffi.py
+++ b/pypy/module/_rawffi/interp_rawffi.py
@@ -508,7 +508,10 @@
     argshapes = unpack_argshapes(space, w_args)
     resshape = unpack_resshape(space, w_res)
     ffi_args = [shape.get_basic_ffi_type() for shape in argshapes]
-    ffi_res = resshape.get_basic_ffi_type()
+    if resshape is not None:
+        ffi_res = resshape.get_basic_ffi_type()
+    else:
+        ffi_res = ffi_type_void
     try:
         ptr = RawFuncPtr('???', ffi_args, ffi_res, rffi.cast(rffi.VOIDP, addr),
                          flags)
diff --git a/pypy/module/_rawffi/test/test__rawffi.py b/pypy/module/_rawffi/test/test__rawffi.py
--- a/pypy/module/_rawffi/test/test__rawffi.py
+++ b/pypy/module/_rawffi/test/test__rawffi.py
@@ -353,6 +353,11 @@
         assert ptr[0] == rawcall.buffer
         ptr.free()
 
+    def test_raw_callable_returning_void(self):
+        import _rawffi
+        _rawffi.FuncPtr(0, [], None)
+        # assert did not crash
+
     def test_short_addition(self):
         import _rawffi
         lib = _rawffi.CDLL(self.lib_name)
diff --git a/pypy/module/_socket/__init__.py b/pypy/module/_socket/__init__.py
--- a/pypy/module/_socket/__init__.py
+++ b/pypy/module/_socket/__init__.py
@@ -6,8 +6,8 @@
     }
 
     interpleveldefs = {
-        'SocketType':  'interp_socket.W_RSocket',
-        'socket'    :  'interp_socket.W_RSocket',
+        'SocketType':  'interp_socket.W_Socket',
+        'socket'    :  'interp_socket.W_Socket',
         'error'     :  'interp_socket.get_error(space, "error")',
         'herror'    :  'interp_socket.get_error(space, "herror")',
         'gaierror'  :  'interp_socket.get_error(space, "gaierror")',
diff --git a/pypy/module/_socket/interp_func.py b/pypy/module/_socket/interp_func.py
--- a/pypy/module/_socket/interp_func.py
+++ b/pypy/module/_socket/interp_func.py
@@ -1,8 +1,12 @@
-from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
-from pypy.module._socket.interp_socket import converted_error, W_RSocket, addr_as_object, ipaddr_from_object
 from rpython.rlib import rsocket
 from rpython.rlib.rsocket import SocketError, INVALID_SOCKET
+
 from pypy.interpreter.error import OperationError
+from pypy.interpreter.gateway import unwrap_spec, WrappedDefault
+from pypy.module._socket.interp_socket import (
+    converted_error, W_Socket, addr_as_object, ipaddr_from_object
+)
+
 
 def gethostname(space):
     """gethostname() -> string
@@ -136,10 +140,10 @@
     The remaining arguments are the same as for socket().
     """
     try:
-        sock = rsocket.fromfd(fd, family, type, proto, W_RSocket)
+        sock = rsocket.fromfd(fd, family, type, proto)
     except SocketError, e:
         raise converted_error(space, e)
-    return space.wrap(sock)
+    return space.wrap(W_Socket(sock))
 
 @unwrap_spec(family=int, type=int, proto=int)
 def socketpair(space, family=rsocket.socketpair_default_family,
@@ -153,10 +157,13 @@
     AF_UNIX if defined on the platform; otherwise, the default is AF_INET.
     """
     try:
-        sock1, sock2 = rsocket.socketpair(family, type, proto, W_RSocket)
+        sock1, sock2 = rsocket.socketpair(family, type, proto)
     except SocketError, e:
         raise converted_error(space, e)
-    return space.newtuple([space.wrap(sock1), space.wrap(sock2)])
+    return space.newtuple([
+        space.wrap(W_Socket(sock1)),
+        space.wrap(W_Socket(sock2))
+    ])
 
 # The following 4 functions refuse all negative numbers, like CPython 2.6.
 # They could also check that the argument is not too large, but CPython 2.6
diff --git a/pypy/module/_socket/interp_socket.py b/pypy/module/_socket/interp_socket.py
--- a/pypy/module/_socket/interp_socket.py
+++ b/pypy/module/_socket/interp_socket.py
@@ -1,14 +1,18 @@
+from rpython.rlib import rsocket
+from rpython.rlib.rarithmetic import intmask
+from rpython.rlib.rsocket import (
+    RSocket, AF_INET, SOCK_STREAM, SocketError, SocketErrorWithErrno,
+    RSocketError
+)
+from rpython.rtyper.lltypesystem import lltype, rffi
+
+from pypy.interpreter import gateway
 from pypy.interpreter.baseobjspace import W_Root
-from pypy.interpreter.typedef import TypeDef, make_weakref_descr,\
-     interp_attrproperty
+from pypy.interpreter.error import OperationError, oefmt
 from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault
-from rpython.rlib.rarithmetic import intmask
-from rpython.rtyper.lltypesystem import lltype, rffi
-from rpython.rlib import rsocket
-from rpython.rlib.rsocket import RSocket, AF_INET, SOCK_STREAM
-from rpython.rlib.rsocket import SocketError, SocketErrorWithErrno, RSocketError
-from pypy.interpreter.error import OperationError, oefmt
-from pypy.interpreter import gateway
+from pypy.interpreter.typedef import (
+    GetSetProperty, TypeDef, make_weakref_descr
+)
 
 
 # XXX Hack to seperate rpython and pypy
@@ -124,10 +128,18 @@
     return addr
 
 
-class W_RSocket(W_Root, RSocket):
-    def __del__(self):
-        self.clear_all_weakrefs()
-        RSocket.__del__(self)
+class W_Socket(W_Root):
+    def __init__(self, sock):
+        self.sock = sock
+
+    def get_type_w(self, space):
+        return space.wrap(self.sock.type)
+
+    def get_proto_w(self, space):
+        return space.wrap(self.sock.proto)
+
+    def get_family_w(self, space):
+        return space.wrap(self.sock.family)
 
     def accept_w(self, space):
         """accept() -> (socket object, address info)
@@ -137,22 +149,22 @@
         info is a pair (hostaddr, port).
         """
         try:
-            fd, addr = self.accept()
+            fd, addr = self.sock.accept()
             sock = rsocket.make_socket(
-                fd, self.family, self.type, self.proto, W_RSocket)
-            return space.newtuple([space.wrap(sock),
+                fd, self.sock.family, self.sock.type, self.sock.proto)
+            return space.newtuple([space.wrap(W_Socket(sock)),
                                    addr_as_object(addr, sock.fd, space)])
-        except SocketError, e:
+        except SocketError as e:
             raise converted_error(space, e)
 
     # convert an Address into an app-level object
     def addr_as_object(self, space, address):
-        return addr_as_object(address, self.fd, space)
+        return addr_as_object(address, self.sock.fd, space)
 
     # convert an app-level object into an Address

From noreply at buildbot.pypy.org  Tue Jul 22 04:34:27 2014
From: noreply at buildbot.pypy.org (wenzhuman)
Date: Tue, 22 Jul 2014 04:34:27 +0200 (CEST)
Subject: [pypy-commit] pypy gc_no_cleanup_nursery: add zero_gc_pointers for
 array of GcPointer
Message-ID: <20140722023427.06E431C06A7@cobra.cs.uni-duesseldorf.de>

Author: wenzhuman 
Branch: gc_no_cleanup_nursery
Changeset: r72478:50b677698120
Date: 2014-07-21 17:49 +0000
http://bitbucket.org/pypy/pypy/changeset/50b677698120/

Log:	add zero_gc_pointers for array of GcPointer

diff --git a/rpython/jit/metainterp/gc.py b/rpython/jit/metainterp/gc.py
--- a/rpython/jit/metainterp/gc.py
+++ b/rpython/jit/metainterp/gc.py
@@ -26,7 +26,7 @@
     malloc_zero_filled = True
 
 class GC_incminimark(GcDescription):
-    malloc_zero_filled = True
+    malloc_zero_filled = False
 
 
 def get_description(config):
diff --git a/rpython/memory/gctransform/framework.py b/rpython/memory/gctransform/framework.py
--- a/rpython/memory/gctransform/framework.py
+++ b/rpython/memory/gctransform/framework.py
@@ -1266,20 +1266,38 @@
 def gen_zero_gc_pointers(TYPE, v, llops, previous_steps=None):
     if previous_steps is None:
         previous_steps = []
-    assert isinstance(TYPE, lltype.Struct)
-    for name in TYPE._names:
-        c_name = rmodel.inputconst(lltype.Void, name)
-        FIELD = getattr(TYPE, name)
-        if isinstance(FIELD, lltype.Ptr) and FIELD._needsgc():
-            c_null = rmodel.inputconst(FIELD, lltype.nullptr(FIELD.TO))
-            if not previous_steps:
-                llops.genop('bare_setfield', [v, c_name, c_null])
-            else:
-                llops.genop('bare_setinteriorfield',
+    assert (isinstance(TYPE, lltype.Struct) or isinstance(TYPE, lltype.Array))
+    if isinstance(TYPE, lltype.Struct):
+        for name in TYPE._names:
+            c_name = rmodel.inputconst(lltype.Void, name)
+            FIELD = getattr(TYPE, name)
+            #handle ptr field in GcStruct
+            if isinstance(FIELD, lltype.Ptr) and FIELD._needsgc():
+                c_null = rmodel.inputconst(FIELD, lltype.nullptr(FIELD.TO))
+                if not previous_steps:
+                    llops.genop('bare_setfield', [v, c_name, c_null])
+                else:
+                    llops.genop('bare_setinteriorfield',
                             [v] + previous_steps + [c_name, c_null])
-        elif isinstance(FIELD, lltype.Struct):
-            gen_zero_gc_pointers(FIELD, v, llops, previous_steps + [c_name])
-
+            #handle inside GcStruct field
+            elif isinstance(FIELD, lltype.Struct):
+                gen_zero_gc_pointers(FIELD, v, llops, previous_steps + [c_name])
+            #handle inside GcArray field 
+            elif isinstance(FIELD, lltype.Array):
+                gen_zero_gc_pointers(FIELD, v, llops, previous_steps + [c_name])
+    if isinstance(TYPE, lltype.Array):
+        ITEM = TYPE.OF
+        if previous_steps:
+            v = llop.genop('getinteriorfield',[v]+previous_steps)
+        arr_size = llops.genop('getarraysize',[v])
+        for i in range(arr_size):
+            #handle an array of GcPtr
+            if isinstance(ITEM, lltype.Ptr) and ITEM._needsgc():
+                c_null = rmodel.inputconst(ITEM, lltype.nullptr(ITEM.TO))
+                llops.genop('bare_setarrayitem',[v, i, c_null])
+            if isinstance(ITEM, lltype.Struct) or isinstance(ITEM, lltype.GcArray):
+                array_item = llops.genop('getarrayitem',[v,i])
+                gen_zero_gc_pointers(FIELD, array_item, llops, previous_steps)
 # ____________________________________________________________
 
 

From noreply at buildbot.pypy.org  Tue Jul 22 12:14:03 2014
From: noreply at buildbot.pypy.org (rguillebert)
Date: Tue, 22 Jul 2014 12:14:03 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: Add info
Message-ID: <20140722101403.CC7EF1D231E@cobra.cs.uni-duesseldorf.de>

Author: Romain Guillebert 
Branch: extradoc
Changeset: r5363:ae1bfc1320b2
Date: 2014-07-22 12:13 +0200
http://bitbucket.org/pypy/extradoc/changeset/ae1bfc1320b2/

Log:	Add info

diff --git a/talk/ep2014/status/talk.pdf b/talk/ep2014/status/talk.pdf
index f9532b263b7bf1aa53ba0d807a875af9991c6a39..8526e0ddddaa7ea79199e3af9c68b27da1de432d
GIT binary patch

[cut]

diff --git a/talk/ep2014/status/talk.rst b/talk/ep2014/status/talk.rst
--- a/talk/ep2014/status/talk.rst
+++ b/talk/ep2014/status/talk.rst
@@ -4,6 +4,17 @@
 PyPy Status
 ================================
 
+Who Am I
+--------
+
+* rguillebert on twitter and irc
+
+* GSoC student on PyPy/Cython
+
+* PyPy contributor since 2011
+
+* Worked on Py3k and Numpy
+
 PyPy is not dead
 ----------------
 
@@ -256,7 +267,7 @@
 
 - Preliminary versions of pypy-jit-stm available
 
-- The overhead is still a bit too high and hard to precict
+- The overhead is still a bit too high and hard to predict
 
 - Lots of polishing needed
 
@@ -272,4 +283,6 @@
 
 - http://morepypy.blogspot.com/
 
+- #pypy at freenode.net
+
 - Any question?

From noreply at buildbot.pypy.org  Tue Jul 22 12:22:06 2014
From: noreply at buildbot.pypy.org (rguillebert)
Date: Tue, 22 Jul 2014 12:22:06 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: Add baroquesoftware
Message-ID: <20140722102206.E2A171D24AF@cobra.cs.uni-duesseldorf.de>

Author: Romain Guillebert 
Branch: extradoc
Changeset: r5364:3334209f26d3
Date: 2014-07-22 12:22 +0200
http://bitbucket.org/pypy/extradoc/changeset/3334209f26d3/

Log:	Add baroquesoftware

diff --git a/talk/ep2014/status/talk.pdf b/talk/ep2014/status/talk.pdf
index 8526e0ddddaa7ea79199e3af9c68b27da1de432d..ddbec1104c2a94e56c7ec21e61c879b534382932
GIT binary patch

[cut]

diff --git a/talk/ep2014/status/talk.rst b/talk/ep2014/status/talk.rst
--- a/talk/ep2014/status/talk.rst
+++ b/talk/ep2014/status/talk.rst
@@ -143,6 +143,8 @@
   improving support in parts of the Python or non-Python interpreters,
   etc.
 
+- http://baroquesoftware.com
+
 Current status
 ---------------
 

From noreply at buildbot.pypy.org  Tue Jul 22 13:41:15 2014
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 22 Jul 2014 13:41:15 +0200 (CEST)
Subject: [pypy-commit] pypy stmgc-c7: add is_atomic()
Message-ID: <20140722114115.342A61C101E@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: stmgc-c7
Changeset: r72479:7e8ed49b6d70
Date: 2014-07-22 13:40 +0200
http://bitbucket.org/pypy/pypy/changeset/7e8ed49b6d70/

Log:	add is_atomic()

diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -42,6 +42,7 @@
         'reset_longest_abort_info':'interp_atomic.reset_longest_abort_info',
         'getsegmentlimit':         'interp_atomic.getsegmentlimit',
         'hint_commit_soon':        'interp_atomic.hint_commit_soon',
+        'is_atomic':               'interp_atomic.is_atomic',
         'error': 'space.fromcache(pypy.module.thread.error.Cache).w_error',
     }
     def activate(self, space):
diff --git a/pypy/module/__pypy__/interp_atomic.py b/pypy/module/__pypy__/interp_atomic.py
--- a/pypy/module/__pypy__/interp_atomic.py
+++ b/pypy/module/__pypy__/interp_atomic.py
@@ -60,6 +60,15 @@
     else:
         return space.wrap(1)
 
+def is_atomic(space):
+    if space.config.translation.stm:
+        from rpython.rlib.rstm import is_atomic
+        return space.wrap(is_atomic())
+    else:
+        giltl = space.threadlocals
+        return space.wrap(giltl.is_atomic)
+
+
 @unwrap_spec(mintime=float)
 def longest_abort_info(space, mintime=0.0):
     if space.config.translation.stm:
diff --git a/pypy/module/__pypy__/test/test_atomic.py b/pypy/module/__pypy__/test/test_atomic.py
--- a/pypy/module/__pypy__/test/test_atomic.py
+++ b/pypy/module/__pypy__/test/test_atomic.py
@@ -1,11 +1,10 @@
 from __future__ import with_statement
 from pypy.module.thread.test.support import GenericTestThread
-from pypy.module.__pypy__.interp_atomic import bdecode
 from rpython.rtyper.lltypesystem import rffi
 
 
 def test_bdecode(space):
-
+    from pypy.module.__pypy__.interp_atomic import bdecode
     def bdec(s, expected):
         p = rffi.str2charp(s)
         w_obj, q = bdecode(space, p)
@@ -27,7 +26,7 @@
         from __pypy__ import thread
         for atomic in thread.atomic, thread.exclusive_atomic:
             with atomic:
-                pass
+                assert thread.is_atomic()
             try:
                 with atomic:
                     raise ValueError
@@ -38,22 +37,28 @@
         from __pypy__ import thread
         with thread.atomic:
             with thread.atomic:
-                pass
+                assert thread.is_atomic()
+            assert thread.is_atomic()
+        assert not thread.is_atomic()
 
     def test_nest_composable_below_exclusive(self):
         from __pypy__ import thread
         with thread.exclusive_atomic:
             with thread.atomic:
                 with thread.atomic:
-                    pass
+                    assert thread.is_atomic()
+                assert thread.is_atomic()
+            assert thread.is_atomic()
+        assert not thread.is_atomic()
 
     def test_nest_exclusive_fails(self):
         from __pypy__ import thread
         try:
             with thread.exclusive_atomic:
                 with thread.exclusive_atomic:
-                    pass
+                    assert thread.is_atomic()
         except thread.error, e:
+            assert not thread.is_atomic()
             assert e.message == "exclusive_atomic block can't be entered inside another atomic block"
 
     def test_nest_exclusive_fails2(self):
@@ -61,6 +66,8 @@
         try:
             with thread.atomic:
                 with thread.exclusive_atomic:
-                    pass
+                    assert thread.is_atomic()
+                assert thread.is_atomic()
         except thread.error, e:
+            assert not thread.is_atomic()
             assert e.message == "exclusive_atomic block can't be entered inside another atomic block"

From noreply at buildbot.pypy.org  Tue Jul 22 13:58:35 2014
From: noreply at buildbot.pypy.org (rguillebert)
Date: Tue, 22 Jul 2014 13:58:35 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: Kill stuff
Message-ID: <20140722115835.16D301C101E@cobra.cs.uni-duesseldorf.de>

Author: Romain Guillebert 
Branch: extradoc
Changeset: r5365:dacc44cdb7af
Date: 2014-07-22 13:57 +0200
http://bitbucket.org/pypy/extradoc/changeset/dacc44cdb7af/

Log:	Kill stuff

diff --git a/talk/ep2014/status/talk.pdf b/talk/ep2014/status/talk.pdf
index ddbec1104c2a94e56c7ec21e61c879b534382932..447f7b485ff254fb76dc366dbce2dc84e936e122
GIT binary patch

[cut]

diff --git a/talk/ep2014/status/talk.rst b/talk/ep2014/status/talk.rst
--- a/talk/ep2014/status/talk.rst
+++ b/talk/ep2014/status/talk.rst
@@ -9,11 +9,11 @@
 
 * rguillebert on twitter and irc
 
+* PyPy contributor since 2011
+
 * GSoC student on PyPy/Cython
 
-* PyPy contributor since 2011
-
-* Worked on Py3k and Numpy
+* Worked on Py3k and Numpy support
 
 PyPy is not dead
 ----------------
@@ -26,12 +26,6 @@
 
 - PyPy is healthy and alive
 
-|pause|
-
-- WARNING: This talk is boring
-
-  * "it just works"
-
 What is PyPy?
 --------------
 
@@ -49,11 +43,6 @@
 
   - **FAST**
 
-* Whatever (dynamic) language you want
-
-  - smalltalk, prolog, PHP, javascript, ...
-
-
 PyPy: past two years (1)
 -----------------------------
 
@@ -63,16 +52,12 @@
 
   * stackless + JIT (eventlet, gevent, ...)
 
-|pause|
-
 - PyPy 2.1 (July 2013)
 
   * stable ARM
 
   * py3k (3.2.3), numpy, general improvements, bugfixes
 
-|pause|
-
 - PyPy 2.2 (November 2013)
 
   * incremental GC, faster JSON
@@ -87,13 +72,13 @@
 
 - PyPy 2.3 (May 2014)
 
-- Lot of internal refactoring
+  * Lot of internal refactoring
 
-- C API for embedding
+  * C API for embedding
 
-  * pypy + uWSGI (thanks to Roberto De Ioris)
+    - pypy + uWSGI (thanks to Roberto De Ioris)
 
-- the usual, boring, general improvements
+  * the usual, boring, general improvements
 
 
 More PyPy-powered languages
@@ -231,26 +216,6 @@
 
 - Fast on CPython, super-fast on PyPy
 
-
-cppyy
-------
-
-- Interface to C++
-
-- Based on reflection, no need to write wrappers
-
-- PyPy-only, similar to PyCintex for CPython
-
-- Main use case: ROOT
-
-  * http://root.cern.ch
-
-  * "a set of OO frameworks with all the functionality needed to handle and
-    analyze large amounts of data in a very efficient way"
-
-- 3x faster than CPython
-
-
 The future: STM
 ----------------
 

From noreply at buildbot.pypy.org  Tue Jul 22 14:07:29 2014
From: noreply at buildbot.pypy.org (rguillebert)
Date: Tue, 22 Jul 2014 14:07:29 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: Kill, kill, kill
Message-ID: <20140722120729.73E451C3273@cobra.cs.uni-duesseldorf.de>

Author: Romain Guillebert 
Branch: extradoc
Changeset: r5366:71780b755a05
Date: 2014-07-22 14:07 +0200
http://bitbucket.org/pypy/extradoc/changeset/71780b755a05/

Log:	Kill, kill, kill

diff --git a/talk/ep2014/status/talk.pdf b/talk/ep2014/status/talk.pdf
index 447f7b485ff254fb76dc366dbce2dc84e936e122..dddab2362990645489cffbd5a4fb451b9b8b7751
GIT binary patch

[cut]

diff --git a/talk/ep2014/status/talk.rst b/talk/ep2014/status/talk.rst
--- a/talk/ep2014/status/talk.rst
+++ b/talk/ep2014/status/talk.rst
@@ -102,8 +102,6 @@
 
   * http://hippyvm.com/
 
-
-
 Fundraising campaign
 ---------------------
 
@@ -147,7 +145,7 @@
 
 - Lots of CFFI modules around:
 
-  * pygame_cffi, psycopg2_cffi, lxml
+  * pygame_cffi, psycopg2cffi, lxml
 
 - numpy: in-progress (more later)
 
@@ -178,9 +176,7 @@
 
 - as usual, in-progress
 
-- ~80% of numpy implemented
-
-  * 2336 passing tests out of 3265
+- 2336 passing tests out of 3265
 
   * http://buildbot.pypy.org/numpy-status/latest.html
 
@@ -214,7 +210,7 @@
 
 - Alternative to C-API, ctypes, Cython, etc.
 
-- Fast on CPython, super-fast on PyPy
+- Fast on CPython, super-fast on PyPy, Jython support in the future
 
 The future: STM
 ----------------
@@ -223,26 +219,16 @@
 
 - Strategy to solve race conditions
 
-- "Finger crossed", rollback in case of conflicts
-
 - On-going research project
 
   * by Armin Rigo and Remi Meier
 
-Current status for STM
-----------------------
-
 - Preliminary versions of pypy-jit-stm available
 
-- The overhead is still a bit too high and hard to predict
-
 - Lots of polishing needed
 
-- More fundamentally, how to best use it is still unknown
-
 - See talk tomorrow
 
-
 Contacts, Q&A
 --------------
 

From noreply at buildbot.pypy.org  Tue Jul 22 14:12:18 2014
From: noreply at buildbot.pypy.org (rguillebert)
Date: Tue, 22 Jul 2014 14:12:18 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: Kill and rephrase
Message-ID: <20140722121218.F1FC91C3273@cobra.cs.uni-duesseldorf.de>

Author: Romain Guillebert 
Branch: extradoc
Changeset: r5367:5ab06d210dc6
Date: 2014-07-22 14:12 +0200
http://bitbucket.org/pypy/extradoc/changeset/5ab06d210dc6/

Log:	Kill and rephrase

diff --git a/talk/ep2014/status/talk.pdf b/talk/ep2014/status/talk.pdf
index dddab2362990645489cffbd5a4fb451b9b8b7751..61bb368d0398a0f044b40b08e1e24e1e24ad1fd3
GIT binary patch

[cut]

diff --git a/talk/ep2014/status/talk.rst b/talk/ep2014/status/talk.rst
--- a/talk/ep2014/status/talk.rst
+++ b/talk/ep2014/status/talk.rst
@@ -217,16 +217,12 @@
 
 - Software Transactional Memory
 
-- Strategy to solve race conditions
+- Solving the GIL problem
 
-- On-going research project
-
-  * by Armin Rigo and Remi Meier
+- Without bringing the threads and locks mess
 
 - Preliminary versions of pypy-jit-stm available
 
-- Lots of polishing needed
-
 - See talk tomorrow
 
 Contacts, Q&A

From noreply at buildbot.pypy.org  Tue Jul 22 14:15:10 2014
From: noreply at buildbot.pypy.org (rguillebert)
Date: Tue, 22 Jul 2014 14:15:10 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: Commit modified pdf
Message-ID: <20140722121510.40AB31C3273@cobra.cs.uni-duesseldorf.de>

Author: Romain Guillebert 
Branch: extradoc
Changeset: r5368:9d5389abdfea
Date: 2014-07-22 14:15 +0200
http://bitbucket.org/pypy/extradoc/changeset/9d5389abdfea/

Log:	Commit modified pdf

diff --git a/talk/ep2014/status/talk.pdf b/talk/ep2014/status/talk.pdf
index 61bb368d0398a0f044b40b08e1e24e1e24ad1fd3..1ad85fa0fe34a4a237ca405797b9dbd596e54ece
GIT binary patch

[cut]


From noreply at buildbot.pypy.org  Tue Jul 22 14:28:17 2014
From: noreply at buildbot.pypy.org (rguillebert)
Date: Tue, 22 Jul 2014 14:28:17 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: Rephrase
Message-ID: <20140722122817.E86531C101E@cobra.cs.uni-duesseldorf.de>

Author: Romain Guillebert 
Branch: extradoc
Changeset: r5369:3ffd361a4541
Date: 2014-07-22 14:28 +0200
http://bitbucket.org/pypy/extradoc/changeset/3ffd361a4541/

Log:	Rephrase

diff --git a/talk/ep2014/status/talk.pdf b/talk/ep2014/status/talk.pdf
index 1ad85fa0fe34a4a237ca405797b9dbd596e54ece..aa6078c72592f3c1704408ce9759135c24a70c28
GIT binary patch

[cut]

diff --git a/talk/ep2014/status/talk.rst b/talk/ep2014/status/talk.rst
--- a/talk/ep2014/status/talk.rst
+++ b/talk/ep2014/status/talk.rst
@@ -113,7 +113,7 @@
 
 - STM, 2nd call: 3'000 $ of 80'000 $ (4%)
 
-- thank to all donors!
+- Thanks to all donors!
 
 Commercial support
 ------------------
@@ -166,23 +166,23 @@
 
 - ~7.5x faster than CPython on ARM
 
-- thanks to Raspberry-Pi foundation
+- Thanks to the Raspberry-Pi foundation
 
-- distributed as part of Raspbian OS
+- Distributed as part of Raspbian OS
 
 
 numpy
 -----
 
-- as usual, in-progress
+- As usual, in-progress
 
 - 2336 passing tests out of 3265
 
   * http://buildbot.pypy.org/numpy-status/latest.html
 
-- just try it
+- Just try it
 
-- no scipy :-/
+- No scipy :-/
 
 
 py3k
@@ -192,9 +192,9 @@
 
 - 3.3: branch started, in-progress
 
-- some missing optimizations
+- Some missing optimizations
 
-  * getting better
+  * Getting better
 
 
 CFFI

From noreply at buildbot.pypy.org  Tue Jul 22 15:01:12 2014
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 22 Jul 2014 15:01:12 +0200 (CEST)
Subject: [pypy-commit] benchmarks default: minor optimisation
Message-ID: <20140722130112.645011D231E@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: 
Changeset: r269:5ccc3f08f3bd
Date: 2014-07-22 14:00 +0200
http://bitbucket.org/pypy/benchmarks/changeset/5ccc3f08f3bd/

Log:	minor optimisation

diff --git a/multithread/common/abstract_threading.py b/multithread/common/abstract_threading.py
--- a/multithread/common/abstract_threading.py
+++ b/multithread/common/abstract_threading.py
@@ -156,9 +156,10 @@
 
 
     def __call__(self):
-        with self._cond:
-            while not self._done:
-                self._cond.wait()
+        if not self._done:
+            with self._cond:
+                while not self._done:
+                    self._cond.wait()
 
         if self._exception:
             raise self._exception

From noreply at buildbot.pypy.org  Tue Jul 22 15:01:13 2014
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 22 Jul 2014 15:01:13 +0200 (CEST)
Subject: [pypy-commit] benchmarks default: some tuning :-/
Message-ID: <20140722130113.E2F6E1D231E@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: 
Changeset: r270:34385976da5c
Date: 2014-07-22 14:01 +0200
http://bitbucket.org/pypy/benchmarks/changeset/34385976da5c/

Log:	some tuning :-/

diff --git a/multithread/quick_sort/quick_sort.py b/multithread/quick_sort/quick_sort.py
--- a/multithread/quick_sort/quick_sort.py
+++ b/multithread/quick_sort/quick_sort.py
@@ -64,21 +64,33 @@
             r -= 1
 
     fs = []
-    #right_amount = 1000 > n // 2 > 505
+    # only start futures on a single level:
     do_futures = level == 4
     largs = (xs, l0, r - l0 + 1, level+1)
     rargs = (xs, l, l0 + n - l, level+1)
+    leftf, rightf = False, False
+
     if do_futures:
-        fs.append(Future(qsort_f, *largs))
-        fs.append(Future(qsort_f, *rargs))
-    else:
-        if level > 4 and n < 100:
+        if largs[2] > 2000:
+            fs.append(Future(qsort_f, *largs))
+            leftf = True
+
+        if rargs[2] > 2000:
+            fs.append(Future(qsort_f, *rargs))
+            rightf = True
+
+    if not leftf:
+        if level >= 4 and largs[2] < 500:
             with atomic:
                 fs.extend(qsort_f(*largs))
+        else:
+            fs.extend(qsort_f(*largs))
+
+    if not rightf:
+        if level >= 4 and rargs[2] < 500:
             with atomic:
                 fs.extend(qsort_f(*rargs))
         else:
-            fs.extend(qsort_f(*largs))
             fs.extend(qsort_f(*rargs))
     #print_abort_info(0.0000001)
 
@@ -104,7 +116,6 @@
             random.shuffle(to_sort)
             s = deque(to_sort)
         # qsort(s, 0, len(s))
-        hint_commit_soon()
 
         t -= time.time()
         # start as future, otherwise we get more threads

From noreply at buildbot.pypy.org  Tue Jul 22 15:01:15 2014
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 22 Jul 2014 15:01:15 +0200 (CEST)
Subject: [pypy-commit] benchmarks default: some other weird benchmark
Message-ID: <20140722130115.5EC0A1D231E@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: 
Changeset: r271:a1256c44c37d
Date: 2014-07-22 15:01 +0200
http://bitbucket.org/pypy/benchmarks/changeset/a1256c44c37d/

Log:	some other weird benchmark

diff --git a/multithread/common/abstract_threading.py b/multithread/common/abstract_threading.py
--- a/multithread/common/abstract_threading.py
+++ b/multithread/common/abstract_threading.py
@@ -172,8 +172,10 @@
     def _task(self, func, *args, **kwargs):
         with self._cond:
             try:
+                hint_commit_soon()
                 with atomic:
                     self._result = func(*args, **kwargs)
+                hint_commit_soon()
             except Exception as e:
                 self._exception = e
             finally:
diff --git a/multithread/perlin_noise/perlin_noise.py b/multithread/perlin_noise/perlin_noise.py
new file mode 100644
--- /dev/null
+++ b/multithread/perlin_noise/perlin_noise.py
@@ -0,0 +1,95 @@
+# -*- coding: utf-8 -*-
+# from http://rosettacode.org/wiki/Perlin_noise#Python
+
+import sys
+import time, random
+from common.abstract_threading import (AtomicFuture,
+    atomic, Future, set_thread_pool, ThreadPool,
+    hint_commit_soon, print_abort_info)
+
+import itertools
+from collections import deque
+
+p = [None] * 512
+permutation = [151,160,137,91,90,15,
+   131,13,201,95,96,53,194,233,7,225,140,36,103,30,69,142,8,99,37,240,21,10,23,
+   190, 6,148,247,120,234,75,0,26,197,62,94,252,219,203,117,35,11,32,57,177,33,
+   88,237,149,56,87,174,20,125,136,171,168, 68,175,74,165,71,134,139,48,27,166,
+   77,146,158,231,83,111,229,122,60,211,133,230,220,105,92,41,55,46,245,40,244,
+   102,143,54, 65,25,63,161, 1,216,80,73,209,76,132,187,208, 89,18,169,200,196,
+   135,130,116,188,159,86,164,100,109,198,173,186, 3,64,52,217,226,250,124,123,
+   5,202,38,147,118,126,255,82,85,212,207,206,59,227,47,16,58,17,182,189,28,42,
+   223,183,170,213,119,248,152, 2,44,154,163, 70,221,153,101,155,167, 43,172,9,
+   129,22,39,253, 19,98,108,110,79,113,224,232,178,185, 112,104,218,246,97,228,
+   251,34,242,193,238,210,144,12,191,179,162,241, 81,51,145,235,249,14,239,107,
+   49,192,214, 31,181,199,106,157,184, 84,204,176,115,121,50,45,127, 4,150,254,
+   138,236,205,93,222,114,67,29,24,72,243,141,128,195,78,66,215,61,156,180]
+
+for i in range(256):
+    p[256+i] = p[i] = permutation[i]
+
+def perlin_noise(x, y, z):
+    X = int(x) & 255                  # FIND UNIT CUBE THAT
+    Y = int(y) & 255                  # CONTAINS POINT.
+    Z = int(z) & 255
+    x -= int(x)                                # FIND RELATIVE X,Y,Z
+    y -= int(y)                                # OF POINT IN CUBE.
+    z -= int(z)
+    u = fade(x)                                # COMPUTE FADE CURVES
+    v = fade(y)                                # FOR EACH OF X,Y,Z.
+    w = fade(z)
+    A = p[X  ]+Y; AA = p[A]+Z; AB = p[A+1]+Z      # HASH COORDINATES OF
+    B = p[X+1]+Y; BA = p[B]+Z; BB = p[B+1]+Z      # THE 8 CUBE CORNERS,
+
+    return lerp(w, lerp(v, lerp(u, grad(p[AA  ], x  , y  , z   ),  # AND ADD
+                                   grad(p[BA  ], x-1, y  , z   )), # BLENDED
+                           lerp(u, grad(p[AB  ], x  , y-1, z   ),  # RESULTS
+                                   grad(p[BB  ], x-1, y-1, z   ))),# FROM  8
+                   lerp(v, lerp(u, grad(p[AA+1], x  , y  , z-1 ),  # CORNERS
+                                   grad(p[BA+1], x-1, y  , z-1 )), # OF CUBE
+                           lerp(u, grad(p[AB+1], x  , y-1, z-1 ),
+                                   grad(p[BB+1], x-1, y-1, z-1 ))))
+
+def fade(t):
+    return t ** 3 * (t * (t * 6 - 15) + 10)
+
+def lerp(t, a, b):
+    return a + t * (b - a)
+
+def grad(hash, x, y, z):
+    h = hash & 15                      # CONVERT LO 4 BITS OF HASH CODE
+    u = x if h<8 else y                # INTO 12 GRADIENT DIRECTIONS.
+    v = y if h<4 else (x if h in (12, 14) else z)
+    return (u if (h&1) == 0 else -u) + (v if (h&2) == 0 else -v)
+
+
+
+
+def work(n, x):
+    res = []
+    hint_commit_soon()
+    for y in range(n):
+        with atomic:
+            for z in range(n):
+                res.append(perlin_noise(x, y, z))
+    hint_commit_soon()
+    return res
+
+
+def run(threads=2, n=60):
+    threads = int(threads)
+    n = int(n)
+
+    set_thread_pool(ThreadPool(threads))
+
+    res = []
+    for x in range(n):
+        res.append(Future(work, 400, x))
+    res = [f() for f in res]
+
+    # shutdown current pool
+    set_thread_pool(None)
+
+
+if __name__ == "__main__":
+    run()

From noreply at buildbot.pypy.org  Tue Jul 22 17:03:05 2014
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 22 Jul 2014 17:03:05 +0200 (CEST)
Subject: [pypy-commit] benchmarks default: optimise with atomic
Message-ID: <20140722150305.6D2191D231E@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: 
Changeset: r272:85cc07891f49
Date: 2014-07-22 17:03 +0200
http://bitbucket.org/pypy/benchmarks/changeset/85cc07891f49/

Log:	optimise with atomic

diff --git a/multithread/perlin_noise/perlin_noise.py b/multithread/perlin_noise/perlin_noise.py
--- a/multithread/perlin_noise/perlin_noise.py
+++ b/multithread/perlin_noise/perlin_noise.py
@@ -68,15 +68,15 @@
 def work(n, x):
     res = []
     hint_commit_soon()
-    for y in range(n):
-        with atomic:
-            for z in range(n):
-                res.append(perlin_noise(x, y, z))
+    with atomic:
+        for y in xrange(n):
+            for z in xrange(n):
+                res.append(perlin_noise(x/3., y/3., z/3.))
     hint_commit_soon()
     return res
 
 
-def run(threads=2, n=60):
+def run(threads=2, n=50):
     threads = int(threads)
     n = int(n)
 

From noreply at buildbot.pypy.org  Tue Jul 22 17:26:37 2014
From: noreply at buildbot.pypy.org (Raemi)
Date: Tue, 22 Jul 2014 17:26:37 +0200 (CEST)
Subject: [pypy-commit] benchmarks default: finding mersenne primes. finally
 some unproblematic microbench
Message-ID: <20140722152637.C376D1D24AF@cobra.cs.uni-duesseldorf.de>

Author: Remi Meier 
Branch: 
Changeset: r273:fd2da4da8f33
Date: 2014-07-22 17:26 +0200
http://bitbucket.org/pypy/benchmarks/changeset/fd2da4da8f33/

Log:	finding mersenne primes. finally some unproblematic microbench

diff --git a/multithread/mersenne/mersenne.py b/multithread/mersenne/mersenne.py
new file mode 100644
--- /dev/null
+++ b/multithread/mersenne/mersenne.py
@@ -0,0 +1,89 @@
+# -*- coding: utf-8 -*-
+
+
+
+import sys
+import time, random
+from common.abstract_threading import (
+    atomic, Future, set_thread_pool, ThreadPool,
+    hint_commit_soon, print_abort_info)
+
+import itertools
+from collections import deque
+
+
+from sys import stdout
+from math import sqrt, log
+
+
+def chunks(iterable, size):
+    it = iter(iterable)
+    item = list(itertools.islice(it, size))
+    while item:
+        yield item
+        item = list(itertools.islice(it, size))
+
+
+def is_prime ( p ):
+    if p == 2: return True # Lucas-Lehmer test only works on odd primes
+    elif p <= 1 or p % 2 == 0: return False
+    else:
+        for i in range(3, int(sqrt(p))+1, 2 ):
+            if p % i == 0: return False
+    return True
+
+def is_mersenne_prime ( p ):
+    if p == 2:
+        return True
+    else:
+        m_p = ( 1 << p ) - 1
+        s = 4
+        for i in range(3, p+1):
+            s = (s ** 2 - 2) % m_p
+        return s == 0
+
+
+def work(ps, counter, upb_count):
+    if counter[0] >= upb_count:
+        return
+
+    for p in ps:
+        with atomic:
+            if is_prime(p) and is_mersenne_prime(p):
+                #print p
+                counter[0] += 1
+        if counter[0] >= upb_count:
+            break
+
+
+def run(threads=2, n=2000):
+    threads = int(threads)
+    n = int(n)
+
+    set_thread_pool(ThreadPool(threads))
+
+
+    precision = n   # maximum requested number of decimal places of 2 ** MP-1 #
+    long_bits_width = precision * log(10, 2)
+    upb_prime = int( long_bits_width - 1 ) / 2    # no unsigned #
+    upb_count = 45      # find 45 mprimes if int was given enough bits #
+
+    print " Finding Mersenne primes in M[2..%d]:" % upb_prime
+
+    counter = [0]
+    fs = []
+    for ps in chunks(xrange(2, upb_prime+1), 500):
+        fs.append(Future(work, ps, counter, upb_count))
+
+    [f() for f in fs]
+    print "found", counter[0]
+
+    # shutdown current pool
+    set_thread_pool(None)
+
+    return
+
+
+
+if __name__ == "__main__":
+    run()

From noreply at buildbot.pypy.org  Tue Jul 22 18:01:40 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Tue, 22 Jul 2014 18:01:40 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: Tweaks
Message-ID: <20140722160140.87A2A1D231E@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: extradoc
Changeset: r5370:3190704f9cd2
Date: 2014-07-22 18:01 +0200
http://bitbucket.org/pypy/extradoc/changeset/3190704f9cd2/

Log:	Tweaks

diff --git a/talk/ep2014/stm/talk.html b/talk/ep2014/stm/talk.html
--- a/talk/ep2014/stm/talk.html
+++ b/talk/ep2014/stm/talk.html
@@ -488,7 +488,7 @@
 
 Transactional Memory
 
-like GIL, but instead of locking, each thread runs optimistically
+like GIL, but instead of blocking, each thread runs optimistically
 "easy" to implement:
 GIL acquire -> transaction start
 GIL release -> transaction commit
@@ -507,7 +507,7 @@
 
 application-level locks still needed...
 but can be very coarse:
-even two big transactions can hopefully run in parallel
+even two big transactions can optimistically run in parallel
 even if they both acquire and release the same lock
 
 
@@ -546,6 +546,7 @@
 current status:
 basics work
 best case 25-40% overhead (much better than originally planned)
+parallelizing user locks not done yet
 tons of things to improve
 tons of things to improve
 tons of things to improve
@@ -563,18 +564,38 @@
 counting primes
 
 
+
+Benefits
+
+Keep locks coarse-grained
+Potential to enable parallelism:
+in CPU-bound multithreaded programs
+or as a replacement of multiprocessing
+but also in existing applications not written for that
+as long as they do multiple things that are "often independent"
+
+
+
+
+
+Issues
+
+Performance hit: 25-40% everywhere (may be ok)
+Keep locks coarse-grained:
+but in case of systematic conflicts, performance is bad again
+need to track and fix them
+need tool support (debugger/profiler)
+
+
+
+
 
 Summary
 
 Transactional Memory is still too researchy for production
-Potential to enable parallelism:
-as a replacement of multiprocessing
-but also in existing applications not written for that
-as long as they do multiple things that are "often independent"
-
-
-Keep locks coarse-grained:
-need to track and fix issues in case of systematic conflicts
+But it has the potential to enable "easier parallelism"
+Still alpha but slowly getting there!
+see http://morepypy.blogspot.com/
 
 
 
diff --git a/talk/ep2014/stm/talk.rst b/talk/ep2014/stm/talk.rst
--- a/talk/ep2014/stm/talk.rst
+++ b/talk/ep2014/stm/talk.rst
@@ -122,7 +122,7 @@
 Transactional Memory
 --------------------
 
-* like GIL, but instead of locking, each thread runs optimistically
+* like GIL, but instead of blocking, each thread runs optimistically
 
 * "easy" to implement:
 
@@ -145,7 +145,7 @@
 
 * but *can be very coarse:*
 
-  - even two big transactions can hopefully run in parallel
+  - even two big transactions can optimistically run in parallel
 
   - even if they both *acquire and release the same lock*
 
@@ -186,6 +186,7 @@
 
   - basics work
   - best case 25-40% overhead (much better than originally planned)
+  - parallelizing user locks not done yet
   - tons of things to improve
   - tons of things to improve
   - tons of things to improve
@@ -201,22 +202,46 @@
 * counting primes
 
 
+Benefits
+--------
+
+* Keep locks coarse-grained
+
+* Potential to enable parallelism:
+
+  - in CPU-bound multithreaded programs
+
+  - or as a replacement of ``multiprocessing``
+
+  - but also in existing applications not written for that
+
+  - as long as they do multiple things that are "often independent"
+
+
+Issues
+------
+
+* Performance hit: 25-40% everywhere (may be ok)
+
+* Keep locks coarse-grained:
+
+  - but in case of systematic conflicts, performance is bad again
+
+  - need to track and fix them
+
+  - need tool support (debugger/profiler)
+
+
 Summary
 -------
 
 * Transactional Memory is still too researchy for production
 
-* Potential to enable parallelism:
+* But it has the potential to enable "easier parallelism"
 
-  - as a replacement of ``multiprocessing``
+* Still alpha but slowly getting there!
 
-  - but also in existing applications not written for that
-
-  - as long as they do multiple things that are "often independent"
-
-* Keep locks coarse-grained:
-    
-  - need to track and fix issues in case of systematic conflicts
+  - see http://morepypy.blogspot.com/
 
 
 Part 2 - Under The Hood

From noreply at buildbot.pypy.org  Tue Jul 22 19:47:37 2014
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Tue, 22 Jul 2014 19:47:37 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk storage-display-refactoring: Deleted
 obsolete Exception class
Message-ID: <20140722174737.D1B321C0748@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: storage-display-refactoring
Changeset: r927:3b12d3eb24dc
Date: 2014-07-21 17:02 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/3b12d3eb24dc/

Log:	Deleted obsolete Exception class

diff --git a/spyvm/model.py b/spyvm/model.py
--- a/spyvm/model.py
+++ b/spyvm/model.py
@@ -1467,8 +1467,3 @@
             if isinstance(s_class, ClassShadow):
                 return "%s >> #%s" % (s_class.getname(), self.lookup_selector)
         return "#%s" % self.lookup_selector
-
-class DetachingShadowError(Exception):
-    def __init__(self, old_shadow, new_shadow_class):
-        self.old_shadow = old_shadow
-        self.new_shadow_class = new_shadow_class

From noreply at buildbot.pypy.org  Tue Jul 22 19:47:39 2014
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Tue, 22 Jul 2014 19:47:39 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk storage-display-refactoring: Trying to
 get AssertionErrors to print their message.
Message-ID: <20140722174739.0E6B31C0748@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: storage-display-refactoring
Changeset: r928:6a516e497789
Date: 2014-07-21 18:05 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/6a516e497789/

Log:	Trying to get AssertionErrors to print their message.

diff --git a/targetimageloadingsmalltalk.py b/targetimageloadingsmalltalk.py
--- a/targetimageloadingsmalltalk.py
+++ b/targetimageloadingsmalltalk.py
@@ -69,7 +69,7 @@
     except error.Exit, e:
         print_error("Exited: %s" % e.msg)
         return -1
-    except Exception, e:
+    except BaseException, e:
         print_error("Exception: %s" % str(e))
         if not objectmodel.we_are_translated():
             import traceback

From noreply at buildbot.pypy.org  Tue Jul 22 19:47:40 2014
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Tue, 22 Jul 2014 19:47:40 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk storage-display-refactoring: Extracted
 Display classes from model into a seperate module model_display.
Message-ID: <20140722174740.43FC51C0748@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: storage-display-refactoring
Changeset: r929:2dac2e226121
Date: 2014-07-21 18:08 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/2dac2e226121/

Log:	Extracted Display classes from model into a seperate module
	model_display.

diff --git a/spyvm/constants.py b/spyvm/constants.py
--- a/spyvm/constants.py
+++ b/spyvm/constants.py
@@ -54,6 +54,11 @@
 BLKCLSR_NUMARGS = 2
 BLKCLSR_SIZE = 3
 
+FORM_BITS = 0
+FORM_WIDTH = 1
+FORM_HEIGHT = 2
+FORM_DEPTH = 3
+
 # ___________________________________________________________________________
 # Miscellaneous constants
 
diff --git a/spyvm/interpreter_proxy.py b/spyvm/interpreter_proxy.py
--- a/spyvm/interpreter_proxy.py
+++ b/spyvm/interpreter_proxy.py
@@ -17,7 +17,7 @@
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib.unroll import unrolling_iterable
 
-from spyvm import error, model, objspace
+from spyvm import error, model, model_display, objspace
 
 sqInt = rffi.INT
 sqLong = rffi.LONG
@@ -242,7 +242,7 @@
         return w_object.convert_to_c_layout()
     elif isinstance(w_object, model.W_BytesObject):
         return rffi.cast(sqIntArrayPtr, w_object.convert_to_c_layout())
-    elif isinstance(w_object, model.W_DisplayBitmap):
+    elif isinstance(w_object, model_display.W_DisplayBitmap):
         return rffi.cast(sqIntArrayPtr, w_object.convert_to_c_layout())
     else:
         raise ProxyFunctionFailed
@@ -529,7 +529,7 @@
 @expose_on_virtual_machine_proxy([], int)
 def fullDisplayUpdate():
     w_display = IProxy.space.objtable['w_display']
-    if isinstance(w_display, model.W_DisplayBitmap):
+    if isinstance(w_display, model_display.W_DisplayBitmap):
         w_display.update_from_buffer()
         w_display.flush_to_screen()
         return 0
@@ -559,16 +559,7 @@
     # display memory
     space = IProxy.space
     if w_dest_form.is_same_object(space.objtable['w_display']):
-        w_bitmap = w_dest_form.fetch(space, 0)
-        if not isinstance(w_bitmap, model.W_DisplayBitmap):
-            assert isinstance(w_bitmap, model.W_WordsObject)
-            w_display_bitmap = w_bitmap.as_display_bitmap(
-                w_dest_form,
-                IProxy.interp,
-                sdldisplay=None
-            )
-        else:
-            w_display_bitmap = w_bitmap
+        w_display_bitmap = model_display.get_display_bitmap(IProxy.interp, w_dest_form)
         w_display_bitmap.update_from_buffer()
         w_display_bitmap.flush_to_screen()
     return 0
diff --git a/spyvm/model.py b/spyvm/model.py
--- a/spyvm/model.py
+++ b/spyvm/model.py
@@ -987,26 +987,6 @@
                 c_words[i] = intmask(old_words[i])
             self.words = None
             return c_words
-
-    def as_display_bitmap(self, w_form, interp, sdldisplay=None):
-        width = interp.space.unwrap_int(w_form.fetch(interp.space, 1))
-        height = interp.space.unwrap_int(w_form.fetch(interp.space, 2))
-        depth = interp.space.unwrap_int(w_form.fetch(interp.space, 3))
-        if not sdldisplay:
-            from spyvm import display
-            sdldisplay = display.SDLDisplay(interp.image_name)
-            sdldisplay.set_video_mode(width, height, depth)
-        w_display_bitmap = W_DisplayBitmap.create(
-            interp.space,
-            self.getclass(interp.space),
-            self.size(),
-            depth,
-            sdldisplay
-        )
-        for idx in range(self.size()):
-            w_display_bitmap.setword(idx, self.getword(idx))
-        w_form.store(interp.space, 0, w_display_bitmap)
-        return w_display_bitmap
     
     def _become(self, w_other):
         assert isinstance(w_other, W_WordsObject)
@@ -1019,144 +999,6 @@
         if self.words is None:
             lltype.free(self.c_words, flavor='raw')
 
-class W_DisplayBitmap(W_AbstractObjectWithClassReference):
-    _attrs_ = ['pixelbuffer', '_realsize', '_real_depth_buffer', 'display', '_depth']
-    _immutable_fields_ = ['_realsize', 'display', '_depth']
-    repr_classname = "W_DisplayBitmap"
-    
-    pixelbuffer = None
-    
-    @staticmethod
-    def create(space, w_class, size, depth, display):
-        if depth < 8:
-            return W_MappingDisplayBitmap(space, w_class, size * (8 / depth), depth, display)
-        elif depth == 8:
-            return W_8BitDisplayBitmap(space, w_class, size, depth, display)
-        elif depth == 16:
-            return W_16BitDisplayBitmap(space, w_class, size, depth, display)
-        else:
-            return W_DisplayBitmap(space, w_class, size, depth, display)
-
-    def repr_content(self):
-        return "len=%d depth=%d %s" % (self.size(), self._depth, self.str_content())
-    
-    def __init__(self, space, w_class, size, depth, display):
-        W_AbstractObjectWithClassReference.__init__(self, space, w_class)
-        self._real_depth_buffer = lltype.malloc(rffi.CArray(rffi.UINT), size, flavor='raw')
-        self._realsize = size
-        self.display = display
-        self._depth = depth
-
-    def at0(self, space, index0):
-        val = self.getword(index0)
-        return space.wrap_uint(val)
-
-    def atput0(self, space, index0, w_value):
-        word = space.unwrap_uint(w_value)
-        self.setword(index0, word)
-
-    def flush_to_screen(self):
-        self.display.flip()
-
-    def size(self):
-        return self._realsize
-        
-    def invariant(self):
-        return False
-
-    def clone(self, space):
-        w_result = W_WordsObject(space, self.getclass(space), self._realsize)
-        n = 0
-        while n < self._realsize:
-            w_result.words[n] = self.getword(n)
-            n += 1
-        return w_result
-
-    def getword(self, n):
-        assert self.size() > n >= 0
-        return self._real_depth_buffer[n]
-
-    def setword(self, n, word):
-        self._real_depth_buffer[n] = word
-        self.display.get_pixelbuffer()[n] = word
-
-    def is_array_object(self):
-        return True
-
-    def update_from_buffer(self):
-        for i in range(self._realsize):
-            self.setword(i, self.getword(i))
-
-    def convert_to_c_layout(self):
-        return self._real_depth_buffer
-
-    def can_become(self, w_other):
-        # TODO - implement _become() for this class. Impossible due to _immutable_fields_?
-        return False
-    
-    def __del__(self):
-        lltype.free(self._real_depth_buffer, flavor='raw')
-
-
-class W_16BitDisplayBitmap(W_DisplayBitmap):
-    repr_classname = "W_16BitDisplayBitmap"
-    def setword(self, n, word):
-        self._real_depth_buffer[n] = word
-        mask = 0b11111
-        lsb = (r_uint(word) & r_uint(0xffff0000)) >> 16
-        msb = (r_uint(word) & r_uint(0x0000ffff))
-
-        lsb = (
-            ((lsb >> 10) & mask) |
-            (((lsb >> 5) & mask) << 6) |
-            ((lsb & mask) << 11)
-        )
-        msb = (
-            ((msb >> 10) & mask) |
-            (((msb >> 5) & mask) << 6) |
-            ((msb & mask) << 11)
-        )
-
-        self.display.get_pixelbuffer()[n] = r_uint(lsb | (msb << 16))
-
-
-class W_8BitDisplayBitmap(W_DisplayBitmap):
-    repr_classname = "W_8BitDisplayBitmap"
-    def setword(self, n, word):
-        self._real_depth_buffer[n] = word
-        self.display.get_pixelbuffer()[n] = r_uint(
-            (word >> 24) |
-            ((word >> 8) & 0x0000ff00) |
-            ((word << 8) & 0x00ff0000) |
-            (word << 24)
-        )
-
-NATIVE_DEPTH = 8
-class W_MappingDisplayBitmap(W_DisplayBitmap):
-    repr_classname = "W_MappingDisplayBitmap"
-    @jit.unroll_safe
-    def setword(self, n, word):
-        self._real_depth_buffer[n] = word
-        word = r_uint(word)
-        pos = self.compute_pos(n)
-        assert self._depth <= 4
-        rshift = 32 - self._depth
-        for i in xrange(8 / self._depth):
-            if pos >= self.size():
-                return
-            mapword = r_uint(0)
-            for i in xrange(4):
-                pixel = r_uint(word) >> rshift
-                mapword |= (r_uint(pixel) << (i * 8))
-                word <<= self._depth
-            self.display.get_pixelbuffer()[pos] = mapword
-            pos += 1
-
-    def compute_pos(self, n):
-        return n * (NATIVE_DEPTH / self._depth)
-        
-# XXX Shouldn't compiledmethod have class reference for subclassed compiled
-# methods?
 class W_CompiledMethod(W_AbstractObjectWithIdentityHash):
     """My instances are methods suitable for interpretation by the virtual machine.  This is the only class in the system whose instances intermix both indexable pointer fields and indexable integer fields.
 
diff --git a/spyvm/objspace.py b/spyvm/objspace.py
--- a/spyvm/objspace.py
+++ b/spyvm/objspace.py
@@ -1,6 +1,6 @@
 import os
 
-from spyvm import constants, model, shadow, wrapper, version
+from spyvm import constants, model, model_display, shadow, wrapper, version
 from spyvm.error import UnwrappingError, WrappingError, PrimitiveFailedError
 from rpython.rlib import jit, rpath
 from rpython.rlib.objectmodel import instantiate, specialize
@@ -237,7 +237,7 @@
         w_display = self.objtable['w_display']
         if w_display:
             w_bitmap = w_display.fetch(self, 0)
-            if isinstance(w_bitmap, model.W_DisplayBitmap):
+            if isinstance(w_bitmap, model_display.W_DisplayBitmap):
                 return w_bitmap.display
         raise PrimitiveFailedError("No display")
 
diff --git a/spyvm/plugins/bitblt.py b/spyvm/plugins/bitblt.py
--- a/spyvm/plugins/bitblt.py
+++ b/spyvm/plugins/bitblt.py
@@ -1,4 +1,4 @@
-from spyvm import model
+from spyvm import model_display, model
 from spyvm.error import PrimitiveFailedError
 from spyvm.shadow import AbstractCachingShadow
 from spyvm.plugins.plugin import Plugin
@@ -32,7 +32,7 @@
         s_frame.push(interp.space.wrap_int(s_bitblt.bitCount))
     elif w_dest_form.is_same_object(space.objtable['w_display']):
         w_bitmap = w_dest_form.fetch(space, 0)
-        assert isinstance(w_bitmap, model.W_DisplayBitmap)
+        assert isinstance(w_bitmap, model_display.W_DisplayBitmap)
         w_bitmap.flush_to_screen()
     return w_rcvr
 
@@ -741,7 +741,7 @@
         self.w_bits = self.fetch(0)
         if self.w_bits.is_nil(self.space):
             return
-        if not (isinstance(self.w_bits, model.W_WordsObject) or isinstance(self.w_bits, model.W_DisplayBitmap)):
+        if not (isinstance(self.w_bits, model.W_WordsObject) or isinstance(self.w_bits, model_display.W_DisplayBitmap)):
             return
         self.width = self.intOrIfNil(self.fetch(1), 0)
         self.height = self.intOrIfNil(self.fetch(2), 0)
diff --git a/spyvm/primitives.py b/spyvm/primitives.py
--- a/spyvm/primitives.py
+++ b/spyvm/primitives.py
@@ -2,7 +2,7 @@
 import inspect
 import math
 import operator
-from spyvm import model, shadow, error, constants, display
+from spyvm import model, model_display, shadow, error, constants, display
 from spyvm.error import PrimitiveFailedError, PrimitiveNotYetWrittenError
 from spyvm import wrapper
 
@@ -676,7 +676,7 @@
         w_display = interp.space.objtable['w_display']
         if w_dest_form.is_same_object(w_display):
             w_bitmap = w_display.fetch(interp.space, 0)
-            assert isinstance(w_bitmap, model.W_DisplayBitmap)
+            assert isinstance(w_bitmap, model_display.W_DisplayBitmap)
             w_bitmap.flush_to_screen()
         return w_rcvr
     except shadow.MethodNotFound:
@@ -748,23 +748,19 @@
     w_prev_display = interp.space.objtable['w_display']
     if w_prev_display:
         w_prev_bitmap = w_prev_display.fetch(interp.space, 0)
-        if isinstance(w_prev_bitmap, model.W_DisplayBitmap):
+        if isinstance(w_prev_bitmap, model_display.W_DisplayBitmap):
             sdldisplay = w_prev_bitmap.display
             sdldisplay.set_video_mode(width, height, depth)
 
-    if isinstance(w_bitmap, model.W_DisplayBitmap):
+    if isinstance(w_bitmap, model_display.W_DisplayBitmap):
         assert (sdldisplay is None) or (sdldisplay is w_bitmap.display)
         sdldisplay = w_bitmap.display
         sdldisplay.set_video_mode(width, height, depth)
         w_display_bitmap = w_bitmap
     else:
         assert isinstance(w_bitmap, model.W_WordsObject)
-        w_display_bitmap = w_bitmap.as_display_bitmap(
-            w_rcvr,
-            interp,
-            sdldisplay=sdldisplay
-        )
-
+        w_display_bitmap = model_display.get_display_bitmap(interp, w_rcvr, sdldisplay=sdldisplay)
+    
     w_display_bitmap.flush_to_screen()
     if interp.image:
         interp.image.lastWindowSize = (width << 16)  + height
@@ -1142,7 +1138,7 @@
             raise PrimitiveFailedError
         for i in xrange(w_arg.size()):
             w_arg.setchar(i, chr(new_value))
-    elif isinstance(w_arg, model.W_WordsObject) or isinstance(w_arg, model.W_DisplayBitmap):
+    elif isinstance(w_arg, model.W_WordsObject) or isinstance(w_arg, model_display.W_DisplayBitmap):
         for i in xrange(w_arg.size()):
             w_arg.setword(i, new_value)
     else:
diff --git a/spyvm/test/test_model.py b/spyvm/test/test_model.py
--- a/spyvm/test/test_model.py
+++ b/spyvm/test/test_model.py
@@ -1,5 +1,5 @@
 import py, math, socket
-from spyvm import model, shadow, objspace, error, display
+from spyvm import model, model_display, shadow, objspace, error, display
 from spyvm.shadow import MethodNotFound, WEAK_POINTERS
 from rpython.rlib.rarithmetic import intmask, r_uint
 from rpython.rtyper.lltypesystem import lltype, rffi
@@ -387,7 +387,7 @@
     d = display.SDLDisplay("test")
     d.set_video_mode(32, 10, 1)
 
-    target = model.W_DisplayBitmap.create(space, space.w_Array, 10, 1, d)
+    target = model_display.W_DisplayBitmap.create(space, space.w_Array, 10, 1, d)
     target.setword(0, r_uint(0xFF00))
     assert bin(target.getword(0)) == bin(0xFF00)
     target.setword(0, r_uint(0x00FF00FF))
@@ -411,7 +411,7 @@
     d = display.SDLDisplay("test")
     d.set_video_mode(18, 5, 1)
 
-    dbitmap = model.W_DisplayBitmap.create(space, space.w_Array, 5, 1, d)
+    dbitmap = model_display.W_DisplayBitmap.create(space, space.w_Array, 5, 1, d)
 
     assert dbitmap.compute_pos(0) == 0
     assert dbitmap.compute_pos(1) == 8
diff --git a/spyvm/test/test_primitives.py b/spyvm/test/test_primitives.py
--- a/spyvm/test/test_primitives.py
+++ b/spyvm/test/test_primitives.py
@@ -1,5 +1,5 @@
 import py, os, math, time
-from spyvm import model, shadow, interpreter, constants, primitives, objspace, wrapper, display
+from spyvm import model, model_display, shadow, interpreter, constants, primitives, objspace, wrapper, display
 from spyvm.primitives import prim_table, PrimitiveFailedError
 from spyvm.plugins import bitblt
 from rpython.rlib.rfloat import INFINITY, NAN, isinf, isnan
@@ -721,7 +721,7 @@
     assert space.objtable["w_display"] is mock_display
     w_bitmap = mock_display.fetch(space, 0)
     assert w_bitmap is not w_wordbmp
-    assert isinstance(w_bitmap, model.W_DisplayBitmap)
+    assert isinstance(w_bitmap, model_display.W_DisplayBitmap)
     sdldisplay = w_bitmap.display
     assert isinstance(sdldisplay, display.SDLDisplay)
 
@@ -733,7 +733,7 @@
     prim(primitives.BE_DISPLAY, [mock_display2])
     assert space.objtable["w_display"] is mock_display2
     w_bitmap2 = mock_display.fetch(space, 0)
-    assert isinstance(w_bitmap2, model.W_DisplayBitmap)
+    assert isinstance(w_bitmap2, model_display.W_DisplayBitmap)
     assert w_bitmap.display is w_bitmap2.display
     assert sdldisplay.width == 32
     assert sdldisplay.height == 10

From noreply at buildbot.pypy.org  Tue Jul 22 19:47:41 2014
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Tue, 22 Jul 2014 19:47:41 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk storage-display-refactoring: Added
 --hacks parameter to enable run_spy_hacks method.
Message-ID: <20140722174741.607261C0748@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: storage-display-refactoring
Changeset: r930:b5dbc52fd7e3
Date: 2014-07-21 18:35 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/b5dbc52fd7e3/

Log:	Added --hacks parameter to enable run_spy_hacks method.

diff --git a/spyvm/objspace.py b/spyvm/objspace.py
--- a/spyvm/objspace.py
+++ b/spyvm/objspace.py
@@ -33,6 +33,7 @@
         self.no_specialized_storage = ConstantFlag()
         # This is a hack; see compile_code() in targetimageloadingsmalltalk.py
         self.suppress_process_switch = ConstantFlag()
+        self.run_spy_hacks = ConstantFlag()
         self.headless = ConstantFlag()
         
         self.classtable = {}
diff --git a/spyvm/squeakimage.py b/spyvm/squeakimage.py
--- a/spyvm/squeakimage.py
+++ b/spyvm/squeakimage.py
@@ -386,12 +386,13 @@
         self.startup_time = time.time()
 
     def run_spy_hacks(self, space):
-        pass
-        # w_display = space.objtable["w_display"]
-        # if w_display is not None and not w_display.is_nil(space):
-        #     if space.unwrap_int(w_display.fetch(space, 3)) < 8:
-        #         # non-native indexed color depth not well supported
-        #         w_display.store(space, 3, space.wrap_int(8))
+        if not space.run_spy_hacks.is_set():
+            return
+        w_display = space.objtable["w_display"]
+        if w_display is not None and not w_display.is_nil(space):
+            if space.unwrap_int(w_display.fetch(space, 3)) < 8:
+                # non-native indexed color depth not well supported
+                w_display.store(space, 3, space.wrap_int(8))
 
     def find_symbol(self, space, reader, symbol):
         w_dnu = self.special(constants.SO_DOES_NOT_UNDERSTAND)
diff --git a/targetimageloadingsmalltalk.py b/targetimageloadingsmalltalk.py
--- a/targetimageloadingsmalltalk.py
+++ b/targetimageloadingsmalltalk.py
@@ -36,6 +36,7 @@
             -p|--poll          - Actively poll for events. Try this if the image is not responding well.
             -i|--no-interrupts - Disable timer interrupt. Disables non-cooperative scheduling.
             -S                 - Disable specialized storage strategies; always use generic ListStorage
+            --hacks            - Enable Spy hacks. Set display color depth to 8.
             
           Logging parameters:
             -t|--trace                 - Output a trace of each message, primitive, return value and process switch.
@@ -119,6 +120,8 @@
                 interrupts = False
             elif arg in ["-P", "--process"]:
                 headless = False
+            elif arg in ["--hacks"]:
+                space.run_spy_hacks.set()
             elif arg in ["-S"]:
                 space.no_specialized_storage.set()
             elif arg in ["-u"]:

From noreply at buildbot.pypy.org  Tue Jul 22 19:47:42 2014
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Tue, 22 Jul 2014 19:47:42 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk storage-display-refactoring: Clean
 error message on EXIT_TO_DEBUGGER primitive.
Message-ID: <20140722174742.7C34E1C0748@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: storage-display-refactoring
Changeset: r931:ef90a282ad1f
Date: 2014-07-21 18:43 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/ef90a282ad1f/

Log:	Clean error message on EXIT_TO_DEBUGGER primitive.

diff --git a/spyvm/primitives.py b/spyvm/primitives.py
--- a/spyvm/primitives.py
+++ b/spyvm/primitives.py
@@ -860,8 +860,8 @@
 
 @expose_primitive(EXIT_TO_DEBUGGER, unwrap_spec=[object])
 def func(interp, s_frame, w_rcvr):
-    if not objectmodel.we_are_translated():
-        import pdb; pdb.set_trace()
+    if interp.space.headless.is_set():
+        exitFromHeadlessExecution(s_frame, "EXIT_TO_DEBUGGER")
     raise PrimitiveNotYetWrittenError()
 
 @expose_primitive(CHANGE_CLASS, unwrap_spec=[object, object], no_result=True)

From noreply at buildbot.pypy.org  Tue Jul 22 19:47:43 2014
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Tue, 22 Jul 2014 19:47:43 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk storage-display-refactoring: Added
 FormWrapper to clean up display handling code.
Message-ID: <20140722174743.9FBA81C0748@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: storage-display-refactoring
Changeset: r932:f151c3506b41
Date: 2014-07-21 19:16 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/f151c3506b41/

Log:	Added FormWrapper to clean up display handling code.

diff --git a/spyvm/interpreter_proxy.py b/spyvm/interpreter_proxy.py
--- a/spyvm/interpreter_proxy.py
+++ b/spyvm/interpreter_proxy.py
@@ -17,7 +17,7 @@
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib.unroll import unrolling_iterable
 
-from spyvm import error, model, model_display, objspace
+from spyvm import error, model, model_display, objspace, wrapper
 
 sqInt = rffi.INT
 sqLong = rffi.LONG
@@ -559,7 +559,8 @@
     # display memory
     space = IProxy.space
     if w_dest_form.is_same_object(space.objtable['w_display']):
-        w_display_bitmap = model_display.get_display_bitmap(IProxy.interp, w_dest_form)
+        form = wrapper.FormWrapper(space, w_dest_form)
+        w_display_bitmap = form.get_display_bitmap(IProxy.interp)
         w_display_bitmap.update_from_buffer()
         w_display_bitmap.flush_to_screen()
     return 0
diff --git a/spyvm/primitives.py b/spyvm/primitives.py
--- a/spyvm/primitives.py
+++ b/spyvm/primitives.py
@@ -734,14 +734,13 @@
     
     if not isinstance(w_rcvr, model.W_PointersObject) or w_rcvr.size() < 4:
         raise PrimitiveFailedError
-    # the fields required are bits (a pointer to a Bitmap), width, height, depth
-
-    # XXX: TODO get the initial image TODO: figure out whether we
-    # should decide the width an report it in the other SCREEN_SIZE
-    w_bitmap = w_rcvr.fetch(interp.space, 0)
-    width = interp.space.unwrap_int(w_rcvr.fetch(interp.space, 1))
-    height = interp.space.unwrap_int(w_rcvr.fetch(interp.space, 2))
-    depth = interp.space.unwrap_int(w_rcvr.fetch(interp.space, 3))
+    
+    # TODO: figure out whether we should decide the width an report it in the SCREEN_SIZE primitive
+    form = wrapper.FormWrapper(interp.space, w_rcvr)
+    w_bitmap = form.bits()
+    width = form.width()
+    height = form.height()
+    depth = form.depth()
 
     sdldisplay = None
 
@@ -751,7 +750,7 @@
         if isinstance(w_prev_bitmap, model_display.W_DisplayBitmap):
             sdldisplay = w_prev_bitmap.display
             sdldisplay.set_video_mode(width, height, depth)
-
+    
     if isinstance(w_bitmap, model_display.W_DisplayBitmap):
         assert (sdldisplay is None) or (sdldisplay is w_bitmap.display)
         sdldisplay = w_bitmap.display
@@ -759,7 +758,7 @@
         w_display_bitmap = w_bitmap
     else:
         assert isinstance(w_bitmap, model.W_WordsObject)
-        w_display_bitmap = model_display.get_display_bitmap(interp, w_rcvr, sdldisplay=sdldisplay)
+        w_display_bitmap = form.get_display_bitmap(interp, sdldisplay)
     
     w_display_bitmap.flush_to_screen()
     if interp.image:
diff --git a/spyvm/wrapper.py b/spyvm/wrapper.py
--- a/spyvm/wrapper.py
+++ b/spyvm/wrapper.py
@@ -1,4 +1,4 @@
-from spyvm import model, constants
+from spyvm import model, model_display, constants
 from spyvm.error import FatalError, WrapperException, PrimitiveFailedError
 
 class Wrapper(object):
@@ -263,6 +263,21 @@
     def size(self):
         return self._w_self.size() - constants.BLKCLSR_SIZE
 
+class FormWrapper(Wrapper):
+    bits, store_bits = make_getter_setter(constants.FORM_BITS)
+    width, store_width = make_int_getter_setter(constants.FORM_WIDTH)
+    height, store_height = make_int_getter_setter(constants.FORM_HEIGHT)
+    depth, store_depth = make_int_getter_setter(constants.FORM_DEPTH)
+    
+    def get_display_bitmap(self, interp, sdldisplay=None):
+        w_bitmap = self.bits()
+        if not isinstance(w_bitmap, model_display.W_DisplayBitmap):
+            w_display_bitmap = model_display.from_words_object(interp, w_bitmap, self, sdldisplay)
+            self.store_bits(w_display_bitmap)
+        else:
+            w_display_bitmap = w_bitmap
+        return w_display_bitmap
+
 # XXX Wrappers below are not used yet.
 class OffsetWrapper(Wrapper):
     offset_x  = make_int_getter(0)

From noreply at buildbot.pypy.org  Tue Jul 22 19:47:44 2014
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Tue, 22 Jul 2014 19:47:44 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk storage-display-refactoring: Made
 image_name available from object space.
Message-ID: <20140722174744.CFCAF1C0748@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: storage-display-refactoring
Changeset: r933:97b7b7828d10
Date: 2014-07-21 19:19 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/97b7b7828d10/

Log:	Made image_name available from object space.

diff --git a/spyvm/interpreter.py b/spyvm/interpreter.py
--- a/spyvm/interpreter.py
+++ b/spyvm/interpreter.py
@@ -23,7 +23,7 @@
 
 
 class Interpreter(object):
-    _immutable_fields_ = ["space", "image", "image_name",
+    _immutable_fields_ = ["space", "image",
                           "interrupt_counter_size",
                           "startup_time", "evented", "interrupts"]
 
@@ -34,12 +34,11 @@
         get_printable_location=get_printable_location
     )
 
-    def __init__(self, space, image=None, image_name="",
+    def __init__(self, space, image=None,
                 trace=False, evented=True, interrupts=True):
         # === Initialize immutable variables
         self.space = space
         self.image = image
-        self.image_name = image_name
         if image:
             self.startup_time = image.startup_time
         else:
@@ -1009,9 +1008,8 @@
 # in order to enable tracing/jumping for message sends etc.
 def debugging():
     def stepping_debugger_init(original):
-        def meth(self, space, image=None, image_name="", trace=False):
-            return_value = original(self, space, image=image,
-                                    image_name=image_name, trace=trace)
+        def meth(self, space, image=None, trace=False):
+            return_value = original(self, space, image=image, trace=trace)
             # ##############################################################
 
             self.message_stepping = False
diff --git a/spyvm/objspace.py b/spyvm/objspace.py
--- a/spyvm/objspace.py
+++ b/spyvm/objspace.py
@@ -14,8 +14,7 @@
         self.flag = [set_initially]
     
     def is_set(self):
-        flag = jit.promote(self.flag[0])
-        return flag
+        return jit.promote(self.flag[0])
     
     def set(self):
         self.flag[0] = True
@@ -26,6 +25,16 @@
     def set_to(self, flag):
         self.flag[0] = flag
 
+class ConstantString(object):
+    def __init__(self):
+        self.value = [""]
+    
+    def get(self):
+        return jit.promote(self.value[0])
+    
+    def set(self, value):
+        self.value[0] = value
+
 class ObjSpace(object):
     def __init__(self):
         # If this flag is set, then no optimizing storage strategies will be used.
@@ -38,8 +47,8 @@
         
         self.classtable = {}
         self.objtable = {}
-        self._executable_path = [""] # XXX: we cannot set the attribute
-                                  # directly on the frozen objectspace
+        self._executable_path = ConstantString()
+        self._image_name = ConstantString()
         
         # Create the nil object.
         # Circumvent the constructor because nil is already referenced there.
@@ -62,11 +71,12 @@
                     break
         return rpath.rabspath(executable)
 
-    def runtime_setup(self, executable):
+    def runtime_setup(self, executable, image_name):
         fullpath = rpath.rabspath(self.find_executable(executable))
         i = fullpath.rfind(os.path.sep) + 1
         assert i > 0
-        self._executable_path[0] = fullpath[:i]
+        self._executable_path.set(fullpath[:i])
+        self._image_name.set(image_name)
 
     def populate_special_objects(self, specials):
         for name, idx in constants.objects_in_special_object_table.items():
@@ -77,7 +87,10 @@
         self.classtable["w_Metaclass"] = self.w_SmallInteger.w_class.w_class
         
     def executable_path(self):
-        return self._executable_path[0]
+        return self._executable_path.get()
+    
+    def image_name(self):
+        return self._image_name.get()
     
     def add_bootstrap_class(self, name, cls):
         self.classtable[name] = cls
diff --git a/spyvm/primitives.py b/spyvm/primitives.py
--- a/spyvm/primitives.py
+++ b/spyvm/primitives.py
@@ -982,7 +982,7 @@
 def func(interp, s_frame, argument_count):
     if argument_count == 0:
         s_frame.pop()
-        return interp.space.wrap_string(interp.image_name)
+        return interp.space.wrap_string(interp.space.image_name())
     elif argument_count == 1:
         pass # XXX
     raise PrimitiveFailedError
diff --git a/spyvm/test/test_primitives.py b/spyvm/test/test_primitives.py
--- a/spyvm/test/test_primitives.py
+++ b/spyvm/test/test_primitives.py
@@ -47,7 +47,8 @@
         frame = context
         for i in range(len(stack)):
             frame.as_context_get_shadow(space).push(stack[i])
-    interp = TestInterpreter(space, image_name=IMAGENAME)
+    interp = TestInterpreter(space)
+    interp.space._image_name.set(IMAGENAME)
     return interp, frame, len(stack)
 
 def _prim(space, code, stack, context = None):
@@ -680,7 +681,7 @@
 
     closure = space.newClosure(w_frame, 4, 0, [])
     s_frame = w_frame.as_methodcontext_get_shadow(space)
-    interp = TestInterpreter(space, image_name=IMAGENAME)
+    interp = TestInterpreter(space)
     interp._loop = True
 
     try:
diff --git a/targetimageloadingsmalltalk.py b/targetimageloadingsmalltalk.py
--- a/targetimageloadingsmalltalk.py
+++ b/targetimageloadingsmalltalk.py
@@ -161,10 +161,10 @@
     # Load & prepare image and environment
     image_reader = squeakimage.reader_for_image(space, squeakimage.Stream(data=imagedata))
     image = create_image(space, image_reader)
-    interp = interpreter.Interpreter(space, image, image_name=path,
+    interp = interpreter.Interpreter(space, image,
                 trace=trace, evented=not poll,
                 interrupts=interrupts)
-    space.runtime_setup(argv[0])
+    space.runtime_setup(argv[0], path)
     print_error("") # Line break after image-loading characters
     
     # Create context to be executed

From noreply at buildbot.pypy.org  Tue Jul 22 19:47:47 2014
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Tue, 22 Jul 2014 19:47:47 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk storage-display-refactoring:
	Refactoring.
Message-ID: <20140722174747.05A001C0748@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: storage-display-refactoring
Changeset: r934:2d4fa0b3f3b7
Date: 2014-07-22 15:51 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/2d4fa0b3f3b7/

Log:	Refactoring. The SDLDisplay instance is globally accessible in the
	space. Cleaned up the process for a form to become the display.
	Added model_display.py (forgotten in previous commit). Fixed
	MappingDisplay: handling the case that width is not dividable by 32.

diff --git a/images/Squeak4.5-noBitBlt.changes b/images/Squeak4.5-noBitBlt.changes
--- a/images/Squeak4.5-noBitBlt.changes
+++ b/images/Squeak4.5-noBitBlt.changes
@@ -12620,4 +12620,8 @@
 	
 	1 to: self splayTreeSize do: [:i |
 		self insertNewNode.
-	]! !

----SNAPSHOT----{15 July 2014 . 6:10:56 pm} Squeak4.5-noBitBlt.image priorSource: 15894330!
\ No newline at end of file
+	]! !

----SNAPSHOT----{15 July 2014 . 6:10:56 pm} Squeak4.5-noBitBlt.image priorSource: 15894330!
+
+----QUIT/NOSAVE----{21 July 2014 . 7:09:22 pm} Squeak4.5-noBitBlt.image priorSource: 15894825!
+
+----QUIT----{21 July 2014 . 7:10:12 pm} Squeak4.5-noBitBlt.image priorSource: 15894825!

----STARTUP----{22 July 2014 . 10:55:07 am} as C:\Dev\lang-smalltalk\images\Squeak4.5-noBitBlt.image!


Smalltalk specialObjectsArray at:15!

(Smalltalk specialObjectsArray at:15) depth!
\ No newline at end of file
diff --git a/spyvm/display.py b/spyvm/display.py
--- a/spyvm/display.py
+++ b/spyvm/display.py
@@ -35,12 +35,13 @@
 WindowEventPaint = 5
 WindowEventStinks = 6
 
+MINIMUM_DEPTH = 8
 
 class SDLDisplay(object):
     _attrs_ = ["screen", "width", "height", "depth", "surface", "has_surface",
                "mouse_position", "button", "key", "interrupt_key", "_defer_updates",
-               "_deferred_event", "pixelbuffer"]
-    _immutable_fields_ = ["pixelbuffer?"]
+               "_deferred_event", "bpp", "pitch"]
+    #_immutable_fields_ = ["pixelbuffer?"]
 
     def __init__(self, title):
         assert RSDL.Init(RSDL.INIT_VIDEO) >= 0
@@ -58,26 +59,31 @@
     def set_video_mode(self, w, h, d):
         assert w > 0 and h > 0
         assert d in [1, 2, 4, 8, 16, 32]
+        if d < MINIMUM_DEPTH:
+            d = MINIMUM_DEPTH
         self.width = w
         self.height = h
         self.depth = d
         flags = RSDL.HWPALETTE | RSDL.RESIZABLE | RSDL.ASYNCBLIT | RSDL.DOUBLEBUF
-        if d < 8:
-            d = 8
         self.screen = RSDL.SetVideoMode(w, h, d, flags)
         if not self.screen:
             print "Could not open display at depth %d" % d
             raise RuntimeError
-        elif d == 8:
+        elif d == MINIMUM_DEPTH:
             self.set_squeak_colormap(self.screen)
-        self.pixelbuffer = rffi.cast(rffi.UINTP, self.screen.c_pixels)
-
-    def get_pixelbuffer(self):
-        return jit.promote(self.pixelbuffer)
-
+        self.bpp = rffi.getintfield(self.screen.c_format, 'c_BytesPerPixel')
+        self.pitch = rffi.getintfield(self.screen, 'c_pitch')
+    
+    def get_pixelbuffer_UCHAR(self):
+        # return jit.promote(rffi.cast(RSDL.Uint8P, self.screen.c_pixels))
+        return jit.promote(self.screen.c_pixels)
+    
+    def get_pixelbuffer_UINT(self):
+        return jit.promote(rffi.cast(RSDL.Uint32P, self.screen.c_pixels))
+    
     def defer_updates(self, flag):
         self._defer_updates = flag
-
+    
     def flip(self, force=False):
         if (not self._defer_updates) or force:
             RSDL.Flip(self.screen)
diff --git a/spyvm/interpreter_proxy.py b/spyvm/interpreter_proxy.py
--- a/spyvm/interpreter_proxy.py
+++ b/spyvm/interpreter_proxy.py
@@ -560,7 +560,7 @@
     space = IProxy.space
     if w_dest_form.is_same_object(space.objtable['w_display']):
         form = wrapper.FormWrapper(space, w_dest_form)
-        w_display_bitmap = form.get_display_bitmap(IProxy.interp)
+        w_display_bitmap = form.get_display_bitmap()
         w_display_bitmap.update_from_buffer()
         w_display_bitmap.flush_to_screen()
     return 0
@@ -1000,7 +1000,7 @@
         self.argcount = 0
         self.w_method = None
         self.fail_reason = 0
-        self.trace_proxy.unset()
+        self.trace_proxy.deactivate()
 
     def call(self, signature, interp, s_frame, argcount, w_method):
         self.initialize_from_call(signature, interp, s_frame, argcount, w_method)
@@ -1042,7 +1042,7 @@
         self.argcount = argcount
         self.w_method = w_method
         self.space = interp.space
-        self.trace_proxy.set_to(interp.trace_proxy.is_set())
+        self.trace_proxy.set(interp.trace_proxy.is_set())
         # ensure that space.w_nil gets the first possible oop
         self.object_to_oop(self.space.w_nil)
 
diff --git a/spyvm/model_display.py b/spyvm/model_display.py
new file mode 100644
--- /dev/null
+++ b/spyvm/model_display.py
@@ -0,0 +1,203 @@
+
+from spyvm import model, constants, display
+from rpython.rlib import jit
+from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rlib.rarithmetic import r_uint
+
+def from_words_object(w_obj, form):
+    depth = form.depth()
+    space = form.space
+    size = w_obj.size()
+    w_class = w_obj.getclass(space)
+    
+    if depth < 8:
+        w_display_bitmap = W_MappingDisplayBitmap(space, w_class, size, depth, form.width())
+    elif depth == 8:
+        w_display_bitmap = W_8BitDisplayBitmap(space, w_class, size, depth)
+    elif depth == 16:
+        w_display_bitmap = W_16BitDisplayBitmap(space, w_class, size, depth)
+    else:
+        w_display_bitmap = W_DisplayBitmap(space, w_class, size, depth)
+    
+    for idx in range(size):
+        w_display_bitmap.setword(idx, w_obj.getword(idx))
+    
+    return w_display_bitmap
+
+invert_byte_order = [False]
+
+def invert():
+    inv = invert_byte_order[0]
+    return jit.promote(inv)
+
+class W_DisplayBitmap(model.W_AbstractObjectWithClassReference):
+    _attrs_ = ['pixelbuffer_words', '_real_depth_buffer', '_realsize', 'display', '_depth']
+    _immutable_fields_ = ['pixelbuffer_words?', '_real_depth_buffer', '_realsize', 'display', '_depth']
+    repr_classname = "W_DisplayBitmap"
+    
+    def __init__(self, space, w_class, size, depth):
+        model.W_AbstractObjectWithClassReference.__init__(self, space, w_class)
+        self._real_depth_buffer = lltype.malloc(rffi.CArray(rffi.UINT), size, flavor='raw')
+        self._realsize = size
+        self._depth = depth
+        self.display = space.display()
+        self.relinquish_display()
+    
+    # === Object access
+    
+    def at0(self, space, index0):
+        val = self.getword(index0)
+        return space.wrap_uint(val)
+    
+    def atput0(self, space, index0, w_value):
+        word = space.unwrap_uint(w_value)
+        self.setword(index0, word)
+    
+    def getword(self, n):
+        assert self.size() > n >= 0
+        return self._real_depth_buffer[n]
+    
+    def setword(self, n, word):
+        self._real_depth_buffer[n] = word
+        if self.pixelbuffer_words > 0:
+            self.set_pixelbuffer_word(n, word)
+    
+    def size(self):
+        return self._realsize
+    
+    # === Graphics
+    
+    def pixelbuffer_UINT(self):
+        return self.display.get_pixelbuffer_UINT()
+    
+    def pixelbuffer_UCHAR(self):
+        return self.display.get_pixelbuffer_UCHAR()
+    
+    def set_pixelbuffer_word(self, n, word):
+        self.pixelbuffer_UINT()[n] = word
+    
+    def take_over_display(self):
+        # Make sure FrameWrapper.take_over_display() is called first for the correct Frame object.
+        pixel_per_word = constants.BYTES_PER_WORD / (self.display.depth / 8)
+        self.pixelbuffer_words = self.display.width * self.display.height / pixel_per_word
+        self.update_from_buffer()
+    
+    def relinquish_display(self):
+        self.pixelbuffer_words = 0
+    
+    def flush_to_screen(self):
+        self.display.flip()
+    
+    def update_from_buffer(self):
+        if self.pixelbuffer_words > 0:
+            for i in range(self.size()):
+                self.set_pixelbuffer_word(i, self.getword(i))
+    
+    # === Misc
+    
+    def invariant(self):
+        return False
+
+    def clone(self, space):
+        w_result = model.W_WordsObject(space, self.getclass(space), self.size())
+        for n in range(self.size()):
+            w_result.setword(n, self.getword(n))
+        return w_result
+    
+    def is_array_object(self):
+        return True
+
+    def convert_to_c_layout(self):
+        return self._real_depth_buffer
+
+    def can_become(self, w_other):
+        # TODO - implement _become() for this class. Impossible due to _immutable_fields_?
+        return False
+    
+    def __del__(self):
+        lltype.free(self._real_depth_buffer, flavor='raw')
+    
+    def repr_content(self):
+        return "len=%d depth=%d %s" % (self.size(), self._depth, self.str_content())
+
+class W_16BitDisplayBitmap(W_DisplayBitmap):
+    
+    repr_classname = "W_16BitDisplayBitmap"
+    
+    def set_pixelbuffer_word(self, n, word):
+        mask = 0b11111
+        lsb = (r_uint(word) & r_uint(0xffff0000)) >> 16
+        msb = (r_uint(word) & r_uint(0x0000ffff))
+        
+        # Invert order of rgb-components
+        lsb = (
+            ((lsb >> 10) & mask) |
+            (((lsb >> 5) & mask) << 6) |
+            ((lsb & mask) << 11)
+        )
+        msb = (
+            ((msb >> 10) & mask) |
+            (((msb >> 5) & mask) << 6) |
+            ((msb & mask) << 11)
+        )
+        
+        self.pixelbuffer_UINT()[n] = r_uint(lsb | (msb << 16))
+
+class W_8BitDisplayBitmap(W_DisplayBitmap):
+    
+    repr_classname = "W_8BitDisplayBitmap"
+    
+    def set_pixelbuffer_word(self, n, word):
+        if invert():
+            # Invert the byte-order.
+            self.pixelbuffer_UINT()[n] = r_uint(
+                (word >> 24) |
+                ((word >> 8) & 0x0000ff00) |
+                ((word << 8) & 0x00ff0000) |
+                (word << 24)
+            )
+        else:
+            self.pixelbuffer_UINT()[n] = r_uint(word)
+
+class W_MappingDisplayBitmap(W_DisplayBitmap):
+    
+    repr_classname = "W_MappingDisplayBitmap"
+    _attrs_ = ['mapping_factor', 'words_per_line', 'bits_in_last_word', 'width']
+    _immutable_fields_ = ['mapping_factor', 'words_per_line', 'bits_in_last_word', 'width']
+    
+    pixel_per_word = constants.BYTES_PER_WORD
+    
+    def __init__(self, space, w_class, size, depth, width):
+        assert depth in [1, 2, 4]
+        width = r_uint(width)
+        self.width = width
+        self.mapping_factor = display.MINIMUM_DEPTH / depth
+        self.words_per_line = r_uint(width / 32 + 1)
+        self.bits_in_last_word = width % 32
+        W_DisplayBitmap.__init__(self, space, w_class, size, depth)
+    
+    @jit.unroll_safe
+    def set_pixelbuffer_word(self, n, word):
+        n = r_uint(n)
+        word = r_uint(word)
+        pos = self.compute_pos(n)
+        buf = self.display.screen.c_pixels
+        
+        if (n+1) % self.words_per_line == 0:
+            # This is the last word on the line. A few bits are cut off.
+            bits = self.bits_in_last_word
+        else:
+            bits = 32
+        
+        depth = r_uint(self._depth)
+        rshift = 32 - depth
+        for i in range(r_uint(bits) / depth):
+            pixel = word >> rshift
+            buf[pos] = rffi.cast(rffi.UCHAR, pixel)
+            word <<= self._depth
+            pos += 1
+        
+    def compute_pos(self, n):
+        word_on_line = n % self.words_per_line
+        complete_lines = r_uint((n - word_on_line) / self.words_per_line)
+        return complete_lines * self.width + 32*word_on_line
diff --git a/spyvm/objspace.py b/spyvm/objspace.py
--- a/spyvm/objspace.py
+++ b/spyvm/objspace.py
@@ -1,39 +1,47 @@
 import os
 
-from spyvm import constants, model, model_display, shadow, wrapper, version
+from spyvm import constants, model, model_display, shadow, wrapper, version, display
 from spyvm.error import UnwrappingError, WrappingError, PrimitiveFailedError
 from rpython.rlib import jit, rpath
-from rpython.rlib.objectmodel import instantiate, specialize
+from rpython.rlib.objectmodel import instantiate, specialize, import_from_mixin
 from rpython.rlib.rarithmetic import intmask, r_uint, int_between
 
-class ConstantFlag(object):
-    """Boolean flag that can be edited, but will be promoted
+class ConstantMixin(object):
+    """Mixin for constant values that can be edited, but will be promoted
     to a constant when jitting."""
     
-    def __init__(self, set_initially=False):
-        self.flag = [set_initially]
-    
-    def is_set(self):
-        return jit.promote(self.flag[0])
-    
-    def set(self):
-        self.flag[0] = True
-    
-    def unset(self):
-        self.flag[0] = False
-    
-    def set_to(self, flag):
-        self.flag[0] = flag
-
-class ConstantString(object):
-    def __init__(self):
-        self.value = [""]
-    
-    def get(self):
-        return jit.promote(self.value[0])
+    def __init__(self, initial_value = None):
+        if initial_value is None:
+            initial_value = self.default_value
+        self.value = [initial_value]
     
     def set(self, value):
         self.value[0] = value
+    
+    def get(self):
+        value = jit.promote(self.value[0])
+        return value
+
+class ConstantFlag(object):
+    import_from_mixin(ConstantMixin)
+    default_value = False
+    def is_set(self):
+        return self.get()
+    def activate(self):
+        self.set(True)
+    def deactivate(self):
+        self.set(False)
+
+class ConstantString(object):
+    import_from_mixin(ConstantMixin)
+    default_value = ""
+    def get(self):
+        # Promoting does not work on strings...
+        return self.value[0]
+    
+class ConstantObject(object):
+    import_from_mixin(ConstantMixin)
+    default_value = None
 
 class ObjSpace(object):
     def __init__(self):
@@ -49,6 +57,7 @@
         self.objtable = {}
         self._executable_path = ConstantString()
         self._image_name = ConstantString()
+        self._display = ConstantObject()
         
         # Create the nil object.
         # Circumvent the constructor because nil is already referenced there.
@@ -85,12 +94,6 @@
                 self.objtable[name] = specials[idx]
         # XXX this is kind of hacky, but I don't know where else to get Metaclass
         self.classtable["w_Metaclass"] = self.w_SmallInteger.w_class.w_class
-        
-    def executable_path(self):
-        return self._executable_path.get()
-    
-    def image_name(self):
-        return self._image_name.get()
     
     def add_bootstrap_class(self, name, cls):
         self.classtable[name] = cls
@@ -138,14 +141,8 @@
             name = "w_" + name
             if not name in self.objtable:
                 self.add_bootstrap_object(name, None)
-    
-    @specialize.arg(1)
-    def get_special_selector(self, selector):
-        i0 = constants.find_selectorindex(selector)
-        self.w_special_selectors.as_cached_object_get_shadow(self)
-        return self.w_special_selectors.fetch(self, i0)
 
-    # methods for wrapping and unwrapping stuff
+    # ============= Methods for wrapping and unwrapping stuff =============
 
     def wrap_int(self, val):
         from spyvm import constants
@@ -247,14 +244,30 @@
 
         return [w_array.at0(self, i) for i in range(w_array.size())]
 
-    def get_display(self):
-        w_display = self.objtable['w_display']
-        if w_display:
-            w_bitmap = w_display.fetch(self, 0)
-            if isinstance(w_bitmap, model_display.W_DisplayBitmap):
-                return w_bitmap.display
-        raise PrimitiveFailedError("No display")
-
+    # ============= Access to static information =============
+    
+    @specialize.arg(1)
+    def get_special_selector(self, selector):
+        i0 = constants.find_selectorindex(selector)
+        self.w_special_selectors.as_cached_object_get_shadow(self)
+        return self.w_special_selectors.fetch(self, i0)
+    
+    def executable_path(self):
+        return self._executable_path.get()
+    
+    def image_name(self):
+        return self._image_name.get()
+    
+    def display(self):
+        disp = self._display.get()
+        if disp is None:
+            # Create lazy to allow headless execution.
+            disp = display.SDLDisplay(self.image_name())
+            self._display.set(disp)
+        return disp
+    
+    # ============= Other Methods =============
+    
     def _freeze_(self):
         return True
 
diff --git a/spyvm/plugins/vmdebugging.py b/spyvm/plugins/vmdebugging.py
--- a/spyvm/plugins/vmdebugging.py
+++ b/spyvm/plugins/vmdebugging.py
@@ -20,12 +20,12 @@
 
 @DebuggingPlugin.expose_primitive(unwrap_spec=[object])
 def trace_proxy(interp, s_frame, w_rcvr):
-    interp.trace_proxy.set()
+    interp.trace_proxy.activate()
     return w_rcvr
 
 @DebuggingPlugin.expose_primitive(unwrap_spec=[object])
 def untrace_proxy(interp, s_frame, w_rcvr):
-    interp.trace_proxy.unset()
+    interp.trace_proxy.deactivate()
     return w_rcvr
 
 @DebuggingPlugin.expose_primitive(unwrap_spec=[object])
diff --git a/spyvm/primitives.py b/spyvm/primitives.py
--- a/spyvm/primitives.py
+++ b/spyvm/primitives.py
@@ -644,7 +644,7 @@
 
 @expose_primitive(MOUSE_POINT, unwrap_spec=[object])
 def func(interp, s_frame, w_rcvr):
-    x, y = interp.space.get_display().mouse_point()
+    x, y = interp.space.display().mouse_point()
     w_point = model.W_PointersObject(interp.space, interp.space.w_Point, 2)
     w_point.store(interp.space, 0, interp.space.wrap_int(x))
     w_point.store(interp.space, 1, interp.space.wrap_int(y))
@@ -656,7 +656,7 @@
 def func(interp, s_frame, w_rcvr, w_into):
     if not interp.evented:
         raise PrimitiveFailedError()
-    ary = interp.space.get_display().get_next_event(time=interp.time_now())
+    ary = interp.space.display().get_next_event(time=interp.time_now())
     for i in range(8):
         w_into.store(interp.space, i, interp.space.wrap_int(ary[i]))
     # XXX - hack
@@ -731,40 +731,23 @@
 def func(interp, s_frame, w_rcvr):
     if interp.space.headless.is_set():
         exitFromHeadlessExecution(s_frame)
-    
     if not isinstance(w_rcvr, model.W_PointersObject) or w_rcvr.size() < 4:
         raise PrimitiveFailedError
     
+    old_display = interp.space.objtable['w_display']
+    if isinstance(old_display, model_display.W_DisplayBitmap):
+        old_display.relinquish_display()
+    interp.space.objtable['w_display'] = w_rcvr
+    
     # TODO: figure out whether we should decide the width an report it in the SCREEN_SIZE primitive
     form = wrapper.FormWrapper(interp.space, w_rcvr)
-    w_bitmap = form.bits()
-    width = form.width()
-    height = form.height()
-    depth = form.depth()
-
-    sdldisplay = None
-
-    w_prev_display = interp.space.objtable['w_display']
-    if w_prev_display:
-        w_prev_bitmap = w_prev_display.fetch(interp.space, 0)
-        if isinstance(w_prev_bitmap, model_display.W_DisplayBitmap):
-            sdldisplay = w_prev_bitmap.display
-            sdldisplay.set_video_mode(width, height, depth)
+    form.take_over_display()
+    w_display_bitmap = form.get_display_bitmap()
+    w_display_bitmap.take_over_display()
+    w_display_bitmap.flush_to_screen()
     
-    if isinstance(w_bitmap, model_display.W_DisplayBitmap):
-        assert (sdldisplay is None) or (sdldisplay is w_bitmap.display)
-        sdldisplay = w_bitmap.display
-        sdldisplay.set_video_mode(width, height, depth)
-        w_display_bitmap = w_bitmap
-    else:
-        assert isinstance(w_bitmap, model.W_WordsObject)
-        w_display_bitmap = form.get_display_bitmap(interp, sdldisplay)
-    
-    w_display_bitmap.flush_to_screen()
     if interp.image:
-        interp.image.lastWindowSize = (width << 16)  + height
-    interp.space.objtable['w_display'] = w_rcvr
-
+        interp.image.lastWindowSize = (form.width() << 16) + form.height()
     return w_rcvr
 
 @expose_primitive(STRING_REPLACE, unwrap_spec=[object, index1_0, index1_0, object, index1_0])
@@ -807,12 +790,12 @@
 
 @expose_primitive(MOUSE_BUTTONS, unwrap_spec=[object])
 def func(interp, s_frame, w_rcvr):
-    btn = interp.space.get_display().mouse_button()
+    btn = interp.space.display().mouse_button()
     return interp.space.wrap_int(btn)
 
 @expose_primitive(KBD_NEXT, unwrap_spec=[object])
 def func(interp, s_frame, w_rcvr):
-    code = interp.space.get_display().next_keycode()
+    code = interp.space.display().next_keycode()
     if code & 0xFF == 0:
         return interp.space.w_nil
     else:
@@ -820,7 +803,7 @@
 
 @expose_primitive(KBD_PEEK, unwrap_spec=[object])
 def func(interp, s_frame, w_rcvr):
-    code = interp.space.get_display().peek_keycode()
+    code = interp.space.display().peek_keycode()
     if code & 0xFF == 0:
         return interp.space.w_nil
     else:
@@ -999,7 +982,7 @@
 
 @expose_primitive(DEFER_UPDATES, unwrap_spec=[object, bool])
 def func(interp, s_frame, w_receiver, flag):
-    sdldisplay = interp.space.get_display()
+    sdldisplay = interp.space.display()
     sdldisplay.defer_updates(flag)
     return w_receiver
 
@@ -1053,7 +1036,7 @@
 
 @expose_primitive(SET_INTERRUPT_KEY, unwrap_spec=[object, int])
 def func(interp, s_frame, w_rcvr, encoded_key):
-    interp.space.get_display().set_interrupt_key(interp.space, encoded_key)
+    interp.space.display().set_interrupt_key(interp.space, encoded_key)
     return w_rcvr
 
 @expose_primitive(INTERRUPT_SEMAPHORE, unwrap_spec=[object, object])
@@ -1526,7 +1509,7 @@
 
 @expose_primitive(FORCE_DISPLAY_UPDATE, unwrap_spec=[object])
 def func(interp, s_frame, w_rcvr):
-    interp.space.get_display().flip(force=True)
+    interp.space.display().flip(force=True)
     return w_rcvr
 
 # ___________________________________________________________________________
diff --git a/spyvm/test/test_model.py b/spyvm/test/test_model.py
--- a/spyvm/test/test_model.py
+++ b/spyvm/test/test_model.py
@@ -404,15 +404,7 @@
         assert target.pixelbuffer[i] == 0x0
 
 def test_display_offset_computation():
-
-    def get_pixelbuffer(self):
-        return lltype.malloc(rffi.ULONGP.TO, self.width * self.height * 32, flavor='raw')
-    display.SDLDisplay.get_pixelbuffer = get_pixelbuffer
-    d = display.SDLDisplay("test")
-    d.set_video_mode(18, 5, 1)
-
-    dbitmap = model_display.W_DisplayBitmap.create(space, space.w_Array, 5, 1, d)
-
+    dbitmap = model_display.W_MappingDisplayBitmap(space, space.w_Array, 5, 1)
     assert dbitmap.compute_pos(0) == 0
     assert dbitmap.compute_pos(1) == 8
     assert dbitmap.size() == 5 * 8
diff --git a/spyvm/test/test_primitives.py b/spyvm/test/test_primitives.py
--- a/spyvm/test/test_primitives.py
+++ b/spyvm/test/test_primitives.py
@@ -765,7 +765,7 @@
         raise DisplayFlush
 
     try:
-        monkeypatch.setattr(space.get_display().__class__, "flip", flush_to_screen_mock)
+        monkeypatch.setattr(space.display().__class__, "flip", flush_to_screen_mock)
         with py.test.raises(DisplayFlush):
             prim(primitives.FORCE_DISPLAY_UPDATE, [mock_display])
     finally:
diff --git a/spyvm/wrapper.py b/spyvm/wrapper.py
--- a/spyvm/wrapper.py
+++ b/spyvm/wrapper.py
@@ -269,15 +269,24 @@
     height, store_height = make_int_getter_setter(constants.FORM_HEIGHT)
     depth, store_depth = make_int_getter_setter(constants.FORM_DEPTH)
     
-    def get_display_bitmap(self, interp, sdldisplay=None):
+    def create_display_bitmap(self):
+        w_display_bitmap = model_display.from_words_object(self.bits(), self)
+        self.store_bits(w_display_bitmap)
+        return w_display_bitmap
+    
+    def get_display_bitmap(self):
         w_bitmap = self.bits()
         if not isinstance(w_bitmap, model_display.W_DisplayBitmap):
-            w_display_bitmap = model_display.from_words_object(interp, w_bitmap, self, sdldisplay)
-            self.store_bits(w_display_bitmap)
+            w_display_bitmap = self.create_display_bitmap()
         else:
             w_display_bitmap = w_bitmap
+            if w_display_bitmap._depth != self.depth():
+                w_display_bitmap = self.create_display_bitmap()
         return w_display_bitmap
-
+    
+    def take_over_display(self):
+        self.space.display().set_video_mode(self.width(), self.height(), self.depth())
+    
 # XXX Wrappers below are not used yet.
 class OffsetWrapper(Wrapper):
     offset_x  = make_int_getter(0)
diff --git a/targetimageloadingsmalltalk.py b/targetimageloadingsmalltalk.py
--- a/targetimageloadingsmalltalk.py
+++ b/targetimageloadingsmalltalk.py
@@ -121,9 +121,12 @@
             elif arg in ["-P", "--process"]:
                 headless = False
             elif arg in ["--hacks"]:
-                space.run_spy_hacks.set()
+                space.run_spy_hacks.activate()
+            elif arg in ["--invert"]:
+                from spyvm import model_display
+                model_display.invert_byte_order[0] = True
             elif arg in ["-S"]:
-                space.no_specialized_storage.set()
+                space.no_specialized_storage.activate()
             elif arg in ["-u"]:
                 from spyvm.plugins.vmdebugging import stop_ui_process
                 stop_ui_process()
@@ -177,7 +180,7 @@
             selector = compile_code(interp, w_receiver, code)
         s_frame = create_context(interp, w_receiver, selector, stringarg)
         if headless:
-            space.headless.set()
+            space.headless.activate()
             context = s_frame
         else:
             create_process(interp, s_frame)
@@ -206,7 +209,7 @@
     # registered (primitive 136 not called), so the idle process will never be left once it is entered.
     # TODO - Find a way to cleanly initialize the image, without executing the active_context of the image.
     # Instead, we want to execute our own context. Then remove this flag (and all references to it)
-    space.suppress_process_switch.set()
+    space.suppress_process_switch.activate()
     
     w_result = interp.perform(
         w_receiver_class,
@@ -218,7 +221,7 @@
     # TODO - is this expected in every image?
     if not isinstance(w_result, model.W_BytesObject) or w_result.as_string() != selector:
         raise error.Exit("Unexpected compilation result (probably failed to compile): %s" % result_string(w_result))
-    space.suppress_process_switch.unset()
+    space.suppress_process_switch.deactivate()
     
     w_receiver_class.as_class_get_shadow(space).s_methoddict().sync_method_cache()
     return selector

From noreply at buildbot.pypy.org  Tue Jul 22 19:47:48 2014
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Tue, 22 Jul 2014 19:47:48 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk storage-display-refactoring: Finally
 got the 1-bit drawing correctly when width is no multiple of 32.
Message-ID: <20140722174748.2AC151C0748@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: storage-display-refactoring
Changeset: r935:5dc629a29736
Date: 2014-07-22 19:47 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/5dc629a29736/

Log:	Finally got the 1-bit drawing correctly when width is no multiple of
	32.

diff --git a/spyvm/model_display.py b/spyvm/model_display.py
--- a/spyvm/model_display.py
+++ b/spyvm/model_display.py
@@ -1,6 +1,6 @@
 
 from spyvm import model, constants, display
-from rpython.rlib import jit
+from rpython.rlib import jit, objectmodel
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib.rarithmetic import r_uint
 
@@ -11,7 +11,7 @@
     w_class = w_obj.getclass(space)
     
     if depth < 8:
-        w_display_bitmap = W_MappingDisplayBitmap(space, w_class, size, depth, form.width())
+        w_display_bitmap = W_MappingDisplayBitmap(space, w_class, size, depth)
     elif depth == 8:
         w_display_bitmap = W_8BitDisplayBitmap(space, w_class, size, depth)
     elif depth == 16:
@@ -159,45 +159,47 @@
         else:
             self.pixelbuffer_UINT()[n] = r_uint(word)
 
+BITS = r_uint(32)
 class W_MappingDisplayBitmap(W_DisplayBitmap):
     
     repr_classname = "W_MappingDisplayBitmap"
-    _attrs_ = ['mapping_factor', 'words_per_line', 'bits_in_last_word', 'width']
-    _immutable_fields_ = ['mapping_factor', 'words_per_line', 'bits_in_last_word', 'width']
+    _attrs_ = ['words_per_line', 'bits_in_last_word', 'pitch']
+    _immutable_fields_ = ['words_per_line?', 'bits_in_last_word?', 'pitch?']
     
-    pixel_per_word = constants.BYTES_PER_WORD
+    def __init__(self, space, w_class, size, depth):
+        assert depth in [1, 2, 4]
+        W_DisplayBitmap.__init__(self, space, w_class, size, depth)
     
-    def __init__(self, space, w_class, size, depth, width):
-        assert depth in [1, 2, 4]
-        width = r_uint(width)
-        self.width = width
-        self.mapping_factor = display.MINIMUM_DEPTH / depth
-        self.words_per_line = r_uint(width / 32 + 1)
-        self.bits_in_last_word = width % 32
-        W_DisplayBitmap.__init__(self, space, w_class, size, depth)
+    def take_over_display(self):
+        pitch = r_uint(self.display.pitch)
+        self.pitch = pitch
+        self.bits_in_last_word = pitch % BITS
+        self.words_per_line = r_uint((pitch - self.bits_in_last_word) / BITS)
+        if self.bits_in_last_word > 0:
+            self.words_per_line += 1
+        W_DisplayBitmap.take_over_display(self)
     
     @jit.unroll_safe
     def set_pixelbuffer_word(self, n, word):
         n = r_uint(n)
-        word = r_uint(word)
-        pos = self.compute_pos(n)
-        buf = self.display.screen.c_pixels
-        
-        if (n+1) % self.words_per_line == 0:
+        if ((n+1) % self.words_per_line) == 0 and self.bits_in_last_word > 0:
             # This is the last word on the line. A few bits are cut off.
             bits = self.bits_in_last_word
         else:
-            bits = 32
+            bits = BITS
         
+        word = r_uint(word)
+        pos = self.compute_pos(n)
+        buf = rffi.ptradd(self.display.screen.c_pixels, pos)
         depth = r_uint(self._depth)
-        rshift = 32 - depth
-        for i in range(r_uint(bits) / depth):
+        rshift = BITS - depth
+        for i in range(bits / depth):
             pixel = word >> rshift
-            buf[pos] = rffi.cast(rffi.UCHAR, pixel)
-            word <<= self._depth
-            pos += 1
-        
+            buf[i] = rffi.cast(rffi.UCHAR, pixel)
+            word <<= depth
+    
     def compute_pos(self, n):
         word_on_line = n % self.words_per_line
-        complete_lines = r_uint((n - word_on_line) / self.words_per_line)
-        return complete_lines * self.width + 32*word_on_line
+        y = r_uint((n - word_on_line) / self.words_per_line)
+        x = word_on_line * BITS / r_uint(self._depth)
+        return y * r_uint(self.pitch) + x

From noreply at buildbot.pypy.org  Tue Jul 22 23:27:03 2014
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Tue, 22 Jul 2014 23:27:03 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk storage-display-refactoring: Added
	comment.
Message-ID: <20140722212703.53D301D2875@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: storage-display-refactoring
Changeset: r936:6f3f768d7ce8
Date: 2014-07-22 21:01 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/6f3f768d7ce8/

Log:	Added comment.

diff --git a/spyvm/model_display.py b/spyvm/model_display.py
--- a/spyvm/model_display.py
+++ b/spyvm/model_display.py
@@ -171,7 +171,7 @@
         W_DisplayBitmap.__init__(self, space, w_class, size, depth)
     
     def take_over_display(self):
-        pitch = r_uint(self.display.pitch)
+        pitch = r_uint(self.display.pitch) # The pitch is different from the width input to SDL!
         self.pitch = pitch
         self.bits_in_last_word = pitch % BITS
         self.words_per_line = r_uint((pitch - self.bits_in_last_word) / BITS)
@@ -202,4 +202,4 @@
         word_on_line = n % self.words_per_line
         y = r_uint((n - word_on_line) / self.words_per_line)
         x = word_on_line * BITS / r_uint(self._depth)
-        return y * r_uint(self.pitch) + x
+        return y * self.pitch + x

From noreply at buildbot.pypy.org  Tue Jul 22 23:27:05 2014
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Tue, 22 Jul 2014 23:27:05 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk storage-display-refactoring: Reverted
 changes file back to storage version. Accidentally committed.
Message-ID: <20140722212705.C5BC91D2875@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: storage-display-refactoring
Changeset: r937:123f68a9dffe
Date: 2014-07-22 21:37 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/123f68a9dffe/

Log:	Reverted changes file back to storage version. Accidentally
	committed.

diff --git a/images/Squeak4.5-noBitBlt.changes b/images/Squeak4.5-noBitBlt.changes
--- a/images/Squeak4.5-noBitBlt.changes
+++ b/images/Squeak4.5-noBitBlt.changes
@@ -12622,6 +12622,13 @@
 		self insertNewNode.
 	]! !

----SNAPSHOT----{15 July 2014 . 6:10:56 pm} Squeak4.5-noBitBlt.image priorSource: 15894330!
 
-----QUIT/NOSAVE----{21 July 2014 . 7:09:22 pm} Squeak4.5-noBitBlt.image priorSource: 15894825!
-
-----QUIT----{21 July 2014 . 7:10:12 pm} Squeak4.5-noBitBlt.image priorSource: 15894825!

----STARTUP----{22 July 2014 . 10:55:07 am} as C:\Dev\lang-smalltalk\images\Squeak4.5-noBitBlt.image!


Smalltalk specialObjectsArray at:15!

(Smalltalk specialObjectsArray at:15) depth!
\ No newline at end of file
+----QUIT/NOSAVE----{21 July 2014 . 4:18:39 pm} Squeak4.5-noBitBlt.image priorSource: 15894825!

----STARTUP----{21 July 2014 . 6:19:06 pm} as C:\Dev\lang-smalltalk\images\Squeak4.5-noBitBlt.image!

!WeakMessageSend methodsFor: 'private' stamp: 'ag 7/21/2014 18:20' prior: 34321504!
withEnsuredReceiverAndArgumentsDo: aBlock otherwise: altBlock
	"Grab real references to receiver and arguments. If they still exist, evaluate aBlock."

	"Return if my receiver has gone away"
	| r a |
	r := self receiver.
	r ifNil: [ ^altBlock value ].

	"Make sure that my arguments haven't gone away"
	arguments ifNil: [ ^ altBlock value ].
	a := Array withAll: arguments.
	a with: shouldBeNil do: [ :arg :flag |
		arg ifNil: [ flag ifFalse: [ ^altBlock value ]]
	].

	^aBlock value: r value: a! !

----QUIT----{21 July 2014 . 6:20:43 pm} Squeak4.5-noBitBlt.image priorSource: 15894825!
+
+----QUIT/NOSAVE----{21 July 2014 . 4:21:36 pm} Squeak4.5-noBitBlt.image priorSource: 15895702!

----STARTUP----{21 July 2014 . 6:21:54 pm} as C:\Dev\lang-smalltalk\images\Squeak4.5-noBitBlt.image!

!WeakMessageSend methodsFor: 'comparing' stamp: 'ag 7/21/2014 18:22' prior: 33144463!
= anObject
	"Compare equal to equivalent MessageSend"
	^ anObject isMessageSend
		and: [self receiver == anObject receiver
		and: [selector == anObject selector
		and: [(Array withAll: self arguments) = (Array withAll: anObject arguments)]]]
! !
!WeakMessageSend methodsFor: 'private' stamp: 'ag 7/21/2014 18:23' prior: 49449636!
withEnsuredReceiverAndArgumentsDo: aBlock otherwise: altBlock
	"Grab real references to receiver and arguments. If they still exist, evaluate aBlock."

	"Return if my receiver has gone away"
	| r a |
	r := self receiver.
	r ifNil: [ ^altBlock value ].

	"Make sure that my arguments haven't gone away"
	a := Array withAll: self arguments.
	a with: shouldBeNil do: [ :arg :flag |
		arg ifNil: [ flag ifFalse: [ ^altBlock value ]]
	].

	^aBlock value: r value: a! !

----QUIT----{21 July 2014 . 6:23:49 pm} Squeak4.5-noBitBlt.image priorSource: 15895702!

----STARTUP----{21 July 2014 . 6:31:05 pm} as C:\Dev\lang-smalltalk\images\Squeak4.5-noBitBlt.image!

!WeakMessageSend methodsFor: 'accessing' stamp: 'ag 7/21/2014 18:31'!
shouldBeNil
	
	^ shouldBeNil ifNil: [ Array new ]! !
!WeakMessageSend methodsFor: 'private' stamp: 'ag 7/21/2014 18:31' prior: 33148869!
isAnyArgumentGarbage
	"Make sure that my arguments haven't gone away"
	arguments ifNotNil: [
		arguments with: self shouldBeNil do: [ :arg :flag |
			(flag not and: [arg isNil])
				ifTrue: [^true]
		]
	].
	^false
! !
!WeakMessageSend methodsFor: 'private' stamp: 'ag 7/21/2014 18:31' prior: 49450841!
withEnsuredReceiverAndArgumentsDo: aBlock otherwise: altBlock
	"Grab real references to receiver and arguments. If they still exist, evaluate aBlock."

	"Return if my receiver has gone away"
	| r a |
	r := self receiver.
	r ifNil: [ ^altBlock value ].

	"Make sure that my arguments haven't gone away"
	a := Array withAll: self arguments.
	a with: self shouldBeNil do: [ :arg :flag |
		arg ifNil: [ flag ifFalse: [ ^altBlock value ]]
	].

	^aBlock value: r value: a! !
!WeakMessageSend methodsFor: 'private' stamp: 'ag 7/21/2014 18:32' prior: 34360552!
withEnsuredReceiverAndArgumentsDo: aBlock withEnoughArguments: anArray otherwise: altBlock
	"call the selector with enough arguments from arguments and anArray"
	| r selfArgs enoughArgs |
	r := self receiver.
	r ifNil: [ ^altBlock value ].
	
	selfArgs := self arguments.
	selfArgs with: self shouldBeNil do: [ :arg :flag |
		arg ifNil: [ flag ifFalse: [ ^altBlock value ]]
	].

	enoughArgs := Array new: selector numArgs.
	enoughArgs replaceFrom: 1
		to: ( selfArgs size min: enoughArgs size)
		with: selfArgs
		startingAt: 1.
	enoughArgs size > selfArgs size ifTrue: [
		enoughArgs replaceFrom: selfArgs size + 1
			to: (selfArgs size + anArray size min: enoughArgs size)
			with: anArray
			startingAt: 1.
	].
	^aBlock value: r value: enoughArgs! !

----QUIT----{21 July 2014 . 6:32:32 pm} Squeak4.5-noBitBlt.image priorSource: 15896872!
+
+----STARTUP----{21 July 2014 . 4:32:52 pm} as C:\Dev\lang-smalltalk\images\Squeak4.5-noBitBlt.image!
+
+
+1+1!
+
+----QUIT/NOSAVE----{21 July 2014 . 4:33:15 pm} Squeak4.5-noBitBlt.image priorSource: 15898877!
\ No newline at end of file

From noreply at buildbot.pypy.org  Tue Jul 22 23:27:06 2014
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Tue, 22 Jul 2014 23:27:06 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk storage-display-refactoring: Removed
	--invert flag.
Message-ID: <20140722212706.F09B21D2875@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: storage-display-refactoring
Changeset: r938:39973aaf497d
Date: 2014-07-22 21:54 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/39973aaf497d/

Log:	Removed --invert flag.

diff --git a/spyvm/model_display.py b/spyvm/model_display.py
--- a/spyvm/model_display.py
+++ b/spyvm/model_display.py
@@ -24,12 +24,6 @@
     
     return w_display_bitmap
 
-invert_byte_order = [False]
-
-def invert():
-    inv = invert_byte_order[0]
-    return jit.promote(inv)
-
 class W_DisplayBitmap(model.W_AbstractObjectWithClassReference):
     _attrs_ = ['pixelbuffer_words', '_real_depth_buffer', '_realsize', 'display', '_depth']
     _immutable_fields_ = ['pixelbuffer_words?', '_real_depth_buffer', '_realsize', 'display', '_depth']
@@ -148,16 +142,13 @@
     repr_classname = "W_8BitDisplayBitmap"
     
     def set_pixelbuffer_word(self, n, word):
-        if invert():
-            # Invert the byte-order.
-            self.pixelbuffer_UINT()[n] = r_uint(
-                (word >> 24) |
-                ((word >> 8) & 0x0000ff00) |
-                ((word << 8) & 0x00ff0000) |
-                (word << 24)
-            )
-        else:
-            self.pixelbuffer_UINT()[n] = r_uint(word)
+        # Invert the byte-order.
+        self.pixelbuffer_UINT()[n] = r_uint(
+            (word >> 24) |
+            ((word >> 8) & 0x0000ff00) |
+            ((word << 8) & 0x00ff0000) |
+            (word << 24)
+        )
 
 BITS = r_uint(32)
 class W_MappingDisplayBitmap(W_DisplayBitmap):
diff --git a/targetimageloadingsmalltalk.py b/targetimageloadingsmalltalk.py
--- a/targetimageloadingsmalltalk.py
+++ b/targetimageloadingsmalltalk.py
@@ -122,9 +122,6 @@
                 headless = False
             elif arg in ["--hacks"]:
                 space.run_spy_hacks.activate()
-            elif arg in ["--invert"]:
-                from spyvm import model_display
-                model_display.invert_byte_order[0] = True
             elif arg in ["-S"]:
                 space.no_specialized_storage.activate()
             elif arg in ["-u"]:

From noreply at buildbot.pypy.org  Tue Jul 22 23:27:08 2014
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Tue, 22 Jul 2014 23:27:08 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk storage-display-refactoring: Cleanups.
Message-ID: <20140722212708.263171D2875@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: storage-display-refactoring
Changeset: r939:e363f2952c1e
Date: 2014-07-22 21:58 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/e363f2952c1e/

Log:	Cleanups.

diff --git a/spyvm/display.py b/spyvm/display.py
--- a/spyvm/display.py
+++ b/spyvm/display.py
@@ -74,11 +74,7 @@
         self.bpp = rffi.getintfield(self.screen.c_format, 'c_BytesPerPixel')
         self.pitch = rffi.getintfield(self.screen, 'c_pitch')
     
-    def get_pixelbuffer_UCHAR(self):
-        # return jit.promote(rffi.cast(RSDL.Uint8P, self.screen.c_pixels))
-        return jit.promote(self.screen.c_pixels)
-    
-    def get_pixelbuffer_UINT(self):
+    def get_pixelbuffer(self):
         return jit.promote(rffi.cast(RSDL.Uint32P, self.screen.c_pixels))
     
     def defer_updates(self, flag):
diff --git a/spyvm/model_display.py b/spyvm/model_display.py
--- a/spyvm/model_display.py
+++ b/spyvm/model_display.py
@@ -61,14 +61,14 @@
     
     # === Graphics
     
-    def pixelbuffer_UINT(self):
-        return self.display.get_pixelbuffer_UINT()
+    def pixelbuffer(self):
+        return self.display.get_pixelbuffer()
     
     def pixelbuffer_UCHAR(self):
         return self.display.get_pixelbuffer_UCHAR()
     
     def set_pixelbuffer_word(self, n, word):
-        self.pixelbuffer_UINT()[n] = word
+        self.pixelbuffer()[n] = word
     
     def take_over_display(self):
         # Make sure FrameWrapper.take_over_display() is called first for the correct Frame object.
@@ -135,7 +135,7 @@
             ((msb & mask) << 11)
         )
         
-        self.pixelbuffer_UINT()[n] = r_uint(lsb | (msb << 16))
+        self.pixelbuffer()[n] = r_uint(lsb | (msb << 16))
 
 class W_8BitDisplayBitmap(W_DisplayBitmap):
     
@@ -143,7 +143,7 @@
     
     def set_pixelbuffer_word(self, n, word):
         # Invert the byte-order.
-        self.pixelbuffer_UINT()[n] = r_uint(
+        self.pixelbuffer()[n] = r_uint(
             (word >> 24) |
             ((word >> 8) & 0x0000ff00) |
             ((word << 8) & 0x00ff0000) |

From noreply at buildbot.pypy.org  Tue Jul 22 23:27:10 2014
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Tue, 22 Jul 2014 23:27:10 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk storage: Merged.
Message-ID: <20140722212710.B48211D2875@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: storage
Changeset: r940:ad963bc84269
Date: 2014-07-22 22:36 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/ad963bc84269/

Log:	Merged.

diff --git a/spyvm/constants.py b/spyvm/constants.py
--- a/spyvm/constants.py
+++ b/spyvm/constants.py
@@ -54,6 +54,11 @@
 BLKCLSR_NUMARGS = 2
 BLKCLSR_SIZE = 3
 
+FORM_BITS = 0
+FORM_WIDTH = 1
+FORM_HEIGHT = 2
+FORM_DEPTH = 3
+
 # ___________________________________________________________________________
 # Miscellaneous constants
 
diff --git a/spyvm/display.py b/spyvm/display.py
--- a/spyvm/display.py
+++ b/spyvm/display.py
@@ -35,12 +35,13 @@
 WindowEventPaint = 5
 WindowEventStinks = 6
 
+MINIMUM_DEPTH = 8
 
 class SDLDisplay(object):
     _attrs_ = ["screen", "width", "height", "depth", "surface", "has_surface",
                "mouse_position", "button", "key", "interrupt_key", "_defer_updates",
-               "_deferred_event", "pixelbuffer"]
-    _immutable_fields_ = ["pixelbuffer?"]
+               "_deferred_event", "bpp", "pitch"]
+    #_immutable_fields_ = ["pixelbuffer?"]
 
     def __init__(self, title):
         assert RSDL.Init(RSDL.INIT_VIDEO) >= 0
@@ -58,26 +59,27 @@
     def set_video_mode(self, w, h, d):
         assert w > 0 and h > 0
         assert d in [1, 2, 4, 8, 16, 32]
+        if d < MINIMUM_DEPTH:
+            d = MINIMUM_DEPTH
         self.width = w
         self.height = h
         self.depth = d
         flags = RSDL.HWPALETTE | RSDL.RESIZABLE | RSDL.ASYNCBLIT | RSDL.DOUBLEBUF
-        if d < 8:
-            d = 8
         self.screen = RSDL.SetVideoMode(w, h, d, flags)
         if not self.screen:
             print "Could not open display at depth %d" % d
             raise RuntimeError
-        elif d == 8:
+        elif d == MINIMUM_DEPTH:
             self.set_squeak_colormap(self.screen)
-        self.pixelbuffer = rffi.cast(rffi.UINTP, self.screen.c_pixels)
-
+        self.bpp = rffi.getintfield(self.screen.c_format, 'c_BytesPerPixel')
+        self.pitch = rffi.getintfield(self.screen, 'c_pitch')
+    
     def get_pixelbuffer(self):
-        return jit.promote(self.pixelbuffer)
-
+        return jit.promote(rffi.cast(RSDL.Uint32P, self.screen.c_pixels))
+    
     def defer_updates(self, flag):
         self._defer_updates = flag
-
+    
     def flip(self, force=False):
         if (not self._defer_updates) or force:
             RSDL.Flip(self.screen)
diff --git a/spyvm/interpreter.py b/spyvm/interpreter.py
--- a/spyvm/interpreter.py
+++ b/spyvm/interpreter.py
@@ -23,7 +23,7 @@
 
 
 class Interpreter(object):
-    _immutable_fields_ = ["space", "image", "image_name",
+    _immutable_fields_ = ["space", "image",
                           "interrupt_counter_size",
                           "startup_time", "evented", "interrupts"]
 
@@ -34,12 +34,11 @@
         get_printable_location=get_printable_location
     )
 
-    def __init__(self, space, image=None, image_name="",
+    def __init__(self, space, image=None,
                 trace=False, evented=True, interrupts=True):
         # === Initialize immutable variables
         self.space = space
         self.image = image
-        self.image_name = image_name
         if image:
             self.startup_time = image.startup_time
         else:
@@ -1009,9 +1008,8 @@
 # in order to enable tracing/jumping for message sends etc.
 def debugging():
     def stepping_debugger_init(original):
-        def meth(self, space, image=None, image_name="", trace=False):
-            return_value = original(self, space, image=image,
-                                    image_name=image_name, trace=trace)
+        def meth(self, space, image=None, trace=False):
+            return_value = original(self, space, image=image, trace=trace)
             # ##############################################################
 
             self.message_stepping = False
diff --git a/spyvm/interpreter_proxy.py b/spyvm/interpreter_proxy.py
--- a/spyvm/interpreter_proxy.py
+++ b/spyvm/interpreter_proxy.py
@@ -17,7 +17,7 @@
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib.unroll import unrolling_iterable
 
-from spyvm import error, model, objspace
+from spyvm import error, model, model_display, objspace, wrapper
 
 sqInt = rffi.INT
 sqLong = rffi.LONG
@@ -242,7 +242,7 @@
         return w_object.convert_to_c_layout()
     elif isinstance(w_object, model.W_BytesObject):
         return rffi.cast(sqIntArrayPtr, w_object.convert_to_c_layout())
-    elif isinstance(w_object, model.W_DisplayBitmap):
+    elif isinstance(w_object, model_display.W_DisplayBitmap):
         return rffi.cast(sqIntArrayPtr, w_object.convert_to_c_layout())
     else:
         raise ProxyFunctionFailed
@@ -529,7 +529,7 @@
 @expose_on_virtual_machine_proxy([], int)
 def fullDisplayUpdate():
     w_display = IProxy.space.objtable['w_display']
-    if isinstance(w_display, model.W_DisplayBitmap):
+    if isinstance(w_display, model_display.W_DisplayBitmap):
         w_display.update_from_buffer()
         w_display.flush_to_screen()
         return 0
@@ -559,16 +559,8 @@
     # display memory
     space = IProxy.space
     if w_dest_form.is_same_object(space.objtable['w_display']):
-        w_bitmap = w_dest_form.fetch(space, 0)
-        if not isinstance(w_bitmap, model.W_DisplayBitmap):
-            assert isinstance(w_bitmap, model.W_WordsObject)
-            w_display_bitmap = w_bitmap.as_display_bitmap(
-                w_dest_form,
-                IProxy.interp,
-                sdldisplay=None
-            )
-        else:
-            w_display_bitmap = w_bitmap
+        form = wrapper.FormWrapper(space, w_dest_form)
+        w_display_bitmap = form.get_display_bitmap()
         w_display_bitmap.update_from_buffer()
         w_display_bitmap.flush_to_screen()
     return 0
@@ -1008,7 +1000,7 @@
         self.argcount = 0
         self.w_method = None
         self.fail_reason = 0
-        self.trace_proxy.unset()
+        self.trace_proxy.deactivate()
 
     def call(self, signature, interp, s_frame, argcount, w_method):
         self.initialize_from_call(signature, interp, s_frame, argcount, w_method)
@@ -1050,7 +1042,7 @@
         self.argcount = argcount
         self.w_method = w_method
         self.space = interp.space
-        self.trace_proxy.set_to(interp.trace_proxy.is_set())
+        self.trace_proxy.set(interp.trace_proxy.is_set())
         # ensure that space.w_nil gets the first possible oop
         self.object_to_oop(self.space.w_nil)
 
diff --git a/spyvm/model.py b/spyvm/model.py
--- a/spyvm/model.py
+++ b/spyvm/model.py
@@ -987,26 +987,6 @@
                 c_words[i] = intmask(old_words[i])
             self.words = None
             return c_words
-
-    def as_display_bitmap(self, w_form, interp, sdldisplay=None):
-        width = interp.space.unwrap_int(w_form.fetch(interp.space, 1))
-        height = interp.space.unwrap_int(w_form.fetch(interp.space, 2))
-        depth = interp.space.unwrap_int(w_form.fetch(interp.space, 3))
-        if not sdldisplay:
-            from spyvm import display
-            sdldisplay = display.SDLDisplay(interp.image_name)
-            sdldisplay.set_video_mode(width, height, depth)
-        w_display_bitmap = W_DisplayBitmap.create(
-            interp.space,
-            self.getclass(interp.space),
-            self.size(),
-            depth,
-            sdldisplay
-        )
-        for idx in range(self.size()):
-            w_display_bitmap.setword(idx, self.getword(idx))
-        w_form.store(interp.space, 0, w_display_bitmap)
-        return w_display_bitmap
     
     def _become(self, w_other):
         assert isinstance(w_other, W_WordsObject)
@@ -1019,144 +999,6 @@
         if self.words is None:
             lltype.free(self.c_words, flavor='raw')
 
-class W_DisplayBitmap(W_AbstractObjectWithClassReference):
-    _attrs_ = ['pixelbuffer', '_realsize', '_real_depth_buffer', 'display', '_depth']
-    _immutable_fields_ = ['_realsize', 'display', '_depth']
-    repr_classname = "W_DisplayBitmap"
-    
-    pixelbuffer = None
-    
-    @staticmethod
-    def create(space, w_class, size, depth, display):
-        if depth < 8:
-            return W_MappingDisplayBitmap(space, w_class, size * (8 / depth), depth, display)
-        elif depth == 8:
-            return W_8BitDisplayBitmap(space, w_class, size, depth, display)
-        elif depth == 16:
-            return W_16BitDisplayBitmap(space, w_class, size, depth, display)
-        else:
-            return W_DisplayBitmap(space, w_class, size, depth, display)
-
-    def repr_content(self):
-        return "len=%d depth=%d %s" % (self.size(), self._depth, self.str_content())
-    
-    def __init__(self, space, w_class, size, depth, display):
-        W_AbstractObjectWithClassReference.__init__(self, space, w_class)
-        self._real_depth_buffer = lltype.malloc(rffi.CArray(rffi.UINT), size, flavor='raw')
-        self._realsize = size
-        self.display = display
-        self._depth = depth
-
-    def at0(self, space, index0):
-        val = self.getword(index0)
-        return space.wrap_uint(val)
-
-    def atput0(self, space, index0, w_value):
-        word = space.unwrap_uint(w_value)
-        self.setword(index0, word)
-
-    def flush_to_screen(self):
-        self.display.flip()
-
-    def size(self):
-        return self._realsize
-        
-    def invariant(self):
-        return False
-
-    def clone(self, space):
-        w_result = W_WordsObject(space, self.getclass(space), self._realsize)
-        n = 0
-        while n < self._realsize:
-            w_result.words[n] = self.getword(n)
-            n += 1
-        return w_result
-
-    def getword(self, n):
-        assert self.size() > n >= 0
-        return self._real_depth_buffer[n]
-
-    def setword(self, n, word):
-        self._real_depth_buffer[n] = word
-        self.display.get_pixelbuffer()[n] = word
-
-    def is_array_object(self):
-        return True
-
-    def update_from_buffer(self):
-        for i in range(self._realsize):
-            self.setword(i, self.getword(i))
-
-    def convert_to_c_layout(self):
-        return self._real_depth_buffer
-
-    def can_become(self, w_other):
-        # TODO - implement _become() for this class. Impossible due to _immutable_fields_?
-        return False
-    
-    def __del__(self):
-        lltype.free(self._real_depth_buffer, flavor='raw')
-
-
-class W_16BitDisplayBitmap(W_DisplayBitmap):
-    repr_classname = "W_16BitDisplayBitmap"
-    def setword(self, n, word):
-        self._real_depth_buffer[n] = word
-        mask = 0b11111
-        lsb = (r_uint(word) & r_uint(0xffff0000)) >> 16
-        msb = (r_uint(word) & r_uint(0x0000ffff))
-
-        lsb = (
-            ((lsb >> 10) & mask) |
-            (((lsb >> 5) & mask) << 6) |
-            ((lsb & mask) << 11)
-        )
-        msb = (
-            ((msb >> 10) & mask) |
-            (((msb >> 5) & mask) << 6) |
-            ((msb & mask) << 11)
-        )
-
-        self.display.get_pixelbuffer()[n] = r_uint(lsb | (msb << 16))
-
-
-class W_8BitDisplayBitmap(W_DisplayBitmap):
-    repr_classname = "W_8BitDisplayBitmap"
-    def setword(self, n, word):
-        self._real_depth_buffer[n] = word
-        self.display.get_pixelbuffer()[n] = r_uint(
-            (word >> 24) |
-            ((word >> 8) & 0x0000ff00) |
-            ((word << 8) & 0x00ff0000) |
-            (word << 24)
-        )
-
-NATIVE_DEPTH = 8
-class W_MappingDisplayBitmap(W_DisplayBitmap):
-    repr_classname = "W_MappingDisplayBitmap"
-    @jit.unroll_safe
-    def setword(self, n, word):
-        self._real_depth_buffer[n] = word
-        word = r_uint(word)
-        pos = self.compute_pos(n)
-        assert self._depth <= 4
-        rshift = 32 - self._depth
-        for i in xrange(8 / self._depth):
-            if pos >= self.size():
-                return
-            mapword = r_uint(0)
-            for i in xrange(4):
-                pixel = r_uint(word) >> rshift
-                mapword |= (r_uint(pixel) << (i * 8))
-                word <<= self._depth
-            self.display.get_pixelbuffer()[pos] = mapword
-            pos += 1
-
-    def compute_pos(self, n):
-        return n * (NATIVE_DEPTH / self._depth)
-        
-# XXX Shouldn't compiledmethod have class reference for subclassed compiled
-# methods?
 class W_CompiledMethod(W_AbstractObjectWithIdentityHash):
     """My instances are methods suitable for interpretation by the virtual machine.  This is the only class in the system whose instances intermix both indexable pointer fields and indexable integer fields.
 
@@ -1467,8 +1309,3 @@
             if isinstance(s_class, ClassShadow):
                 return "%s >> #%s" % (s_class.getname(), self.lookup_selector)
         return "#%s" % self.lookup_selector
-
-class DetachingShadowError(Exception):
-    def __init__(self, old_shadow, new_shadow_class):
-        self.old_shadow = old_shadow
-        self.new_shadow_class = new_shadow_class
diff --git a/spyvm/model_display.py b/spyvm/model_display.py
new file mode 100644
--- /dev/null
+++ b/spyvm/model_display.py
@@ -0,0 +1,196 @@
+
+from spyvm import model, constants, display
+from rpython.rlib import jit, objectmodel
+from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rlib.rarithmetic import r_uint
+
+def from_words_object(w_obj, form):
+    depth = form.depth()
+    space = form.space
+    size = w_obj.size()
+    w_class = w_obj.getclass(space)
+    
+    if depth < 8:
+        w_display_bitmap = W_MappingDisplayBitmap(space, w_class, size, depth)
+    elif depth == 8:
+        w_display_bitmap = W_8BitDisplayBitmap(space, w_class, size, depth)
+    elif depth == 16:
+        w_display_bitmap = W_16BitDisplayBitmap(space, w_class, size, depth)
+    else:
+        w_display_bitmap = W_DisplayBitmap(space, w_class, size, depth)
+    
+    for idx in range(size):
+        w_display_bitmap.setword(idx, w_obj.getword(idx))
+    
+    return w_display_bitmap
+
+class W_DisplayBitmap(model.W_AbstractObjectWithClassReference):
+    _attrs_ = ['pixelbuffer_words', '_real_depth_buffer', '_realsize', 'display', '_depth']
+    _immutable_fields_ = ['pixelbuffer_words?', '_real_depth_buffer', '_realsize', 'display', '_depth']
+    repr_classname = "W_DisplayBitmap"
+    
+    def __init__(self, space, w_class, size, depth):
+        model.W_AbstractObjectWithClassReference.__init__(self, space, w_class)
+        self._real_depth_buffer = lltype.malloc(rffi.CArray(rffi.UINT), size, flavor='raw')
+        self._realsize = size
+        self._depth = depth
+        self.display = space.display()
+        self.relinquish_display()
+    
+    # === Object access
+    
+    def at0(self, space, index0):
+        val = self.getword(index0)
+        return space.wrap_uint(val)
+    
+    def atput0(self, space, index0, w_value):
+        word = space.unwrap_uint(w_value)
+        self.setword(index0, word)
+    
+    def getword(self, n):
+        assert self.size() > n >= 0
+        return self._real_depth_buffer[n]
+    
+    def setword(self, n, word):
+        self._real_depth_buffer[n] = word
+        if self.pixelbuffer_words > 0:
+            self.set_pixelbuffer_word(n, word)
+    
+    def size(self):
+        return self._realsize
+    
+    # === Graphics
+    
+    def pixelbuffer(self):
+        return self.display.get_pixelbuffer()
+    
+    def pixelbuffer_UCHAR(self):
+        return self.display.get_pixelbuffer_UCHAR()
+    
+    def set_pixelbuffer_word(self, n, word):
+        self.pixelbuffer()[n] = word
+    
+    def take_over_display(self):
+        # Make sure FrameWrapper.take_over_display() is called first for the correct Frame object.
+        pixel_per_word = constants.BYTES_PER_WORD / (self.display.depth / 8)
+        self.pixelbuffer_words = self.display.width * self.display.height / pixel_per_word
+        self.update_from_buffer()
+    
+    def relinquish_display(self):
+        self.pixelbuffer_words = 0
+    
+    def flush_to_screen(self):
+        self.display.flip()
+    
+    def update_from_buffer(self):
+        if self.pixelbuffer_words > 0:
+            for i in range(self.size()):
+                self.set_pixelbuffer_word(i, self.getword(i))
+    
+    # === Misc
+    
+    def invariant(self):
+        return False
+
+    def clone(self, space):
+        w_result = model.W_WordsObject(space, self.getclass(space), self.size())
+        for n in range(self.size()):
+            w_result.setword(n, self.getword(n))
+        return w_result
+    
+    def is_array_object(self):
+        return True
+
+    def convert_to_c_layout(self):
+        return self._real_depth_buffer
+
+    def can_become(self, w_other):
+        # TODO - implement _become() for this class. Impossible due to _immutable_fields_?
+        return False
+    
+    def __del__(self):
+        lltype.free(self._real_depth_buffer, flavor='raw')
+    
+    def repr_content(self):
+        return "len=%d depth=%d %s" % (self.size(), self._depth, self.str_content())
+
+class W_16BitDisplayBitmap(W_DisplayBitmap):
+    
+    repr_classname = "W_16BitDisplayBitmap"
+    
+    def set_pixelbuffer_word(self, n, word):
+        mask = 0b11111
+        lsb = (r_uint(word) & r_uint(0xffff0000)) >> 16
+        msb = (r_uint(word) & r_uint(0x0000ffff))
+        
+        # Invert order of rgb-components
+        lsb = (
+            ((lsb >> 10) & mask) |
+            (((lsb >> 5) & mask) << 6) |
+            ((lsb & mask) << 11)
+        )
+        msb = (
+            ((msb >> 10) & mask) |
+            (((msb >> 5) & mask) << 6) |
+            ((msb & mask) << 11)
+        )
+        
+        self.pixelbuffer()[n] = r_uint(lsb | (msb << 16))
+
+class W_8BitDisplayBitmap(W_DisplayBitmap):
+    
+    repr_classname = "W_8BitDisplayBitmap"
+    
+    def set_pixelbuffer_word(self, n, word):
+        # Invert the byte-order.
+        self.pixelbuffer()[n] = r_uint(
+            (word >> 24) |
+            ((word >> 8) & 0x0000ff00) |
+            ((word << 8) & 0x00ff0000) |
+            (word << 24)
+        )
+
+BITS = r_uint(32)
+class W_MappingDisplayBitmap(W_DisplayBitmap):
+    
+    repr_classname = "W_MappingDisplayBitmap"
+    _attrs_ = ['words_per_line', 'bits_in_last_word', 'pitch']
+    _immutable_fields_ = ['words_per_line?', 'bits_in_last_word?', 'pitch?']
+    
+    def __init__(self, space, w_class, size, depth):
+        assert depth in [1, 2, 4]
+        W_DisplayBitmap.__init__(self, space, w_class, size, depth)
+    
+    def take_over_display(self):
+        pitch = r_uint(self.display.pitch) # The pitch is different from the width input to SDL!
+        self.pitch = pitch
+        self.bits_in_last_word = pitch % BITS
+        self.words_per_line = r_uint((pitch - self.bits_in_last_word) / BITS)
+        if self.bits_in_last_word > 0:
+            self.words_per_line += 1
+        W_DisplayBitmap.take_over_display(self)
+    
+    @jit.unroll_safe
+    def set_pixelbuffer_word(self, n, word):
+        n = r_uint(n)
+        if ((n+1) % self.words_per_line) == 0 and self.bits_in_last_word > 0:
+            # This is the last word on the line. A few bits are cut off.
+            bits = self.bits_in_last_word
+        else:
+            bits = BITS
+        
+        word = r_uint(word)
+        pos = self.compute_pos(n)
+        buf = rffi.ptradd(self.display.screen.c_pixels, pos)
+        depth = r_uint(self._depth)
+        rshift = BITS - depth
+        for i in range(bits / depth):
+            pixel = word >> rshift
+            buf[i] = rffi.cast(rffi.UCHAR, pixel)
+            word <<= depth
+    
+    def compute_pos(self, n):
+        word_on_line = n % self.words_per_line
+        y = r_uint((n - word_on_line) / self.words_per_line)
+        x = word_on_line * BITS / r_uint(self._depth)
+        return y * self.pitch + x
diff --git a/spyvm/objspace.py b/spyvm/objspace.py
--- a/spyvm/objspace.py
+++ b/spyvm/objspace.py
@@ -1,30 +1,47 @@
 import os
 
-from spyvm import constants, model, shadow, wrapper, version
+from spyvm import constants, model, model_display, shadow, wrapper, version, display
 from spyvm.error import UnwrappingError, WrappingError, PrimitiveFailedError
 from rpython.rlib import jit, rpath
-from rpython.rlib.objectmodel import instantiate, specialize
+from rpython.rlib.objectmodel import instantiate, specialize, import_from_mixin
 from rpython.rlib.rarithmetic import intmask, r_uint, int_between
 
-class ConstantFlag(object):
-    """Boolean flag that can be edited, but will be promoted
+class ConstantMixin(object):
+    """Mixin for constant values that can be edited, but will be promoted
     to a constant when jitting."""
     
-    def __init__(self, set_initially=False):
-        self.flag = [set_initially]
+    def __init__(self, initial_value = None):
+        if initial_value is None:
+            initial_value = self.default_value
+        self.value = [initial_value]
     
+    def set(self, value):
+        self.value[0] = value
+    
+    def get(self):
+        value = jit.promote(self.value[0])
+        return value
+
+class ConstantFlag(object):
+    import_from_mixin(ConstantMixin)
+    default_value = False
     def is_set(self):
-        flag = jit.promote(self.flag[0])
-        return flag
+        return self.get()
+    def activate(self):
+        self.set(True)
+    def deactivate(self):
+        self.set(False)
+
+class ConstantString(object):
+    import_from_mixin(ConstantMixin)
+    default_value = ""
+    def get(self):
+        # Promoting does not work on strings...
+        return self.value[0]
     
-    def set(self):
-        self.flag[0] = True
-    
-    def unset(self):
-        self.flag[0] = False
-    
-    def set_to(self, flag):
-        self.flag[0] = flag
+class ConstantObject(object):
+    import_from_mixin(ConstantMixin)
+    default_value = None
 
 class ObjSpace(object):
     def __init__(self):
@@ -33,12 +50,14 @@
         self.no_specialized_storage = ConstantFlag()
         # This is a hack; see compile_code() in targetimageloadingsmalltalk.py
         self.suppress_process_switch = ConstantFlag()
+        self.run_spy_hacks = ConstantFlag()
         self.headless = ConstantFlag()
         
         self.classtable = {}
         self.objtable = {}
-        self._executable_path = [""] # XXX: we cannot set the attribute
-                                  # directly on the frozen objectspace
+        self._executable_path = ConstantString()
+        self._image_name = ConstantString()
+        self._display = ConstantObject()
         
         # Create the nil object.
         # Circumvent the constructor because nil is already referenced there.
@@ -61,11 +80,12 @@
                     break
         return rpath.rabspath(executable)
 
-    def runtime_setup(self, executable):
+    def runtime_setup(self, executable, image_name):
         fullpath = rpath.rabspath(self.find_executable(executable))
         i = fullpath.rfind(os.path.sep) + 1
         assert i > 0
-        self._executable_path[0] = fullpath[:i]
+        self._executable_path.set(fullpath[:i])
+        self._image_name.set(image_name)
 
     def populate_special_objects(self, specials):
         for name, idx in constants.objects_in_special_object_table.items():
@@ -74,9 +94,6 @@
                 self.objtable[name] = specials[idx]
         # XXX this is kind of hacky, but I don't know where else to get Metaclass
         self.classtable["w_Metaclass"] = self.w_SmallInteger.w_class.w_class
-        
-    def executable_path(self):
-        return self._executable_path[0]
     
     def add_bootstrap_class(self, name, cls):
         self.classtable[name] = cls
@@ -124,14 +141,8 @@
             name = "w_" + name
             if not name in self.objtable:
                 self.add_bootstrap_object(name, None)
-    
-    @specialize.arg(1)
-    def get_special_selector(self, selector):
-        i0 = constants.find_selectorindex(selector)
-        self.w_special_selectors.as_cached_object_get_shadow(self)
-        return self.w_special_selectors.fetch(self, i0)
 
-    # methods for wrapping and unwrapping stuff
+    # ============= Methods for wrapping and unwrapping stuff =============
 
     def wrap_int(self, val):
         from spyvm import constants
@@ -233,14 +244,30 @@
 
         return [w_array.at0(self, i) for i in range(w_array.size())]
 
-    def get_display(self):
-        w_display = self.objtable['w_display']
-        if w_display:
-            w_bitmap = w_display.fetch(self, 0)
-            if isinstance(w_bitmap, model.W_DisplayBitmap):
-                return w_bitmap.display
-        raise PrimitiveFailedError("No display")
-
+    # ============= Access to static information =============
+    
+    @specialize.arg(1)
+    def get_special_selector(self, selector):
+        i0 = constants.find_selectorindex(selector)
+        self.w_special_selectors.as_cached_object_get_shadow(self)
+        return self.w_special_selectors.fetch(self, i0)
+    
+    def executable_path(self):
+        return self._executable_path.get()
+    
+    def image_name(self):
+        return self._image_name.get()
+    
+    def display(self):
+        disp = self._display.get()
+        if disp is None:
+            # Create lazy to allow headless execution.
+            disp = display.SDLDisplay(self.image_name())
+            self._display.set(disp)
+        return disp
+    
+    # ============= Other Methods =============
+    
     def _freeze_(self):
         return True
 
diff --git a/spyvm/plugins/bitblt.py b/spyvm/plugins/bitblt.py
--- a/spyvm/plugins/bitblt.py
+++ b/spyvm/plugins/bitblt.py
@@ -1,4 +1,4 @@
-from spyvm import model
+from spyvm import model_display, model
 from spyvm.error import PrimitiveFailedError
 from spyvm.shadow import AbstractCachingShadow
 from spyvm.plugins.plugin import Plugin
@@ -32,7 +32,7 @@
         s_frame.push(interp.space.wrap_int(s_bitblt.bitCount))
     elif w_dest_form.is_same_object(space.objtable['w_display']):
         w_bitmap = w_dest_form.fetch(space, 0)
-        assert isinstance(w_bitmap, model.W_DisplayBitmap)
+        assert isinstance(w_bitmap, model_display.W_DisplayBitmap)
         w_bitmap.flush_to_screen()
     return w_rcvr
 
@@ -741,7 +741,7 @@
         self.w_bits = self.fetch(0)
         if self.w_bits.is_nil(self.space):
             return
-        if not (isinstance(self.w_bits, model.W_WordsObject) or isinstance(self.w_bits, model.W_DisplayBitmap)):
+        if not (isinstance(self.w_bits, model.W_WordsObject) or isinstance(self.w_bits, model_display.W_DisplayBitmap)):
             return
         self.width = self.intOrIfNil(self.fetch(1), 0)
         self.height = self.intOrIfNil(self.fetch(2), 0)
diff --git a/spyvm/plugins/vmdebugging.py b/spyvm/plugins/vmdebugging.py
--- a/spyvm/plugins/vmdebugging.py
+++ b/spyvm/plugins/vmdebugging.py
@@ -20,12 +20,12 @@
 
 @DebuggingPlugin.expose_primitive(unwrap_spec=[object])
 def trace_proxy(interp, s_frame, w_rcvr):
-    interp.trace_proxy.set()
+    interp.trace_proxy.activate()
     return w_rcvr
 
 @DebuggingPlugin.expose_primitive(unwrap_spec=[object])
 def untrace_proxy(interp, s_frame, w_rcvr):
-    interp.trace_proxy.unset()
+    interp.trace_proxy.deactivate()
     return w_rcvr
 
 @DebuggingPlugin.expose_primitive(unwrap_spec=[object])
diff --git a/spyvm/primitives.py b/spyvm/primitives.py
--- a/spyvm/primitives.py
+++ b/spyvm/primitives.py
@@ -2,7 +2,7 @@
 import inspect
 import math
 import operator
-from spyvm import model, shadow, error, constants, display
+from spyvm import model, model_display, shadow, error, constants, display
 from spyvm.error import PrimitiveFailedError, PrimitiveNotYetWrittenError
 from spyvm import wrapper
 
@@ -644,7 +644,7 @@
 
 @expose_primitive(MOUSE_POINT, unwrap_spec=[object])
 def func(interp, s_frame, w_rcvr):
-    x, y = interp.space.get_display().mouse_point()
+    x, y = interp.space.display().mouse_point()
     w_point = model.W_PointersObject(interp.space, interp.space.w_Point, 2)
     w_point.store(interp.space, 0, interp.space.wrap_int(x))
     w_point.store(interp.space, 1, interp.space.wrap_int(y))
@@ -656,7 +656,7 @@
 def func(interp, s_frame, w_rcvr, w_into):
     if not interp.evented:
         raise PrimitiveFailedError()
-    ary = interp.space.get_display().get_next_event(time=interp.time_now())
+    ary = interp.space.display().get_next_event(time=interp.time_now())
     for i in range(8):
         w_into.store(interp.space, i, interp.space.wrap_int(ary[i]))
     # XXX - hack
@@ -676,7 +676,7 @@
         w_display = interp.space.objtable['w_display']
         if w_dest_form.is_same_object(w_display):
             w_bitmap = w_display.fetch(interp.space, 0)
-            assert isinstance(w_bitmap, model.W_DisplayBitmap)
+            assert isinstance(w_bitmap, model_display.W_DisplayBitmap)
             w_bitmap.flush_to_screen()
         return w_rcvr
     except shadow.MethodNotFound:
@@ -731,45 +731,23 @@
 def func(interp, s_frame, w_rcvr):
     if interp.space.headless.is_set():
         exitFromHeadlessExecution(s_frame)
-    
     if not isinstance(w_rcvr, model.W_PointersObject) or w_rcvr.size() < 4:
         raise PrimitiveFailedError
-    # the fields required are bits (a pointer to a Bitmap), width, height, depth
-
-    # XXX: TODO get the initial image TODO: figure out whether we
-    # should decide the width an report it in the other SCREEN_SIZE
-    w_bitmap = w_rcvr.fetch(interp.space, 0)
-    width = interp.space.unwrap_int(w_rcvr.fetch(interp.space, 1))
-    height = interp.space.unwrap_int(w_rcvr.fetch(interp.space, 2))
-    depth = interp.space.unwrap_int(w_rcvr.fetch(interp.space, 3))
-
-    sdldisplay = None
-
-    w_prev_display = interp.space.objtable['w_display']
-    if w_prev_display:
-        w_prev_bitmap = w_prev_display.fetch(interp.space, 0)
-        if isinstance(w_prev_bitmap, model.W_DisplayBitmap):
-            sdldisplay = w_prev_bitmap.display
-            sdldisplay.set_video_mode(width, height, depth)
-
-    if isinstance(w_bitmap, model.W_DisplayBitmap):
-        assert (sdldisplay is None) or (sdldisplay is w_bitmap.display)
-        sdldisplay = w_bitmap.display
-        sdldisplay.set_video_mode(width, height, depth)
-        w_display_bitmap = w_bitmap
-    else:
-        assert isinstance(w_bitmap, model.W_WordsObject)
-        w_display_bitmap = w_bitmap.as_display_bitmap(
-            w_rcvr,
-            interp,
-            sdldisplay=sdldisplay
-        )
-
+    
+    old_display = interp.space.objtable['w_display']
+    if isinstance(old_display, model_display.W_DisplayBitmap):
+        old_display.relinquish_display()
+    interp.space.objtable['w_display'] = w_rcvr
+    
+    # TODO: figure out whether we should decide the width an report it in the SCREEN_SIZE primitive
+    form = wrapper.FormWrapper(interp.space, w_rcvr)
+    form.take_over_display()
+    w_display_bitmap = form.get_display_bitmap()
+    w_display_bitmap.take_over_display()
     w_display_bitmap.flush_to_screen()
+    
     if interp.image:
-        interp.image.lastWindowSize = (width << 16)  + height
-    interp.space.objtable['w_display'] = w_rcvr
-
+        interp.image.lastWindowSize = (form.width() << 16) + form.height()
     return w_rcvr
 
 @expose_primitive(STRING_REPLACE, unwrap_spec=[object, index1_0, index1_0, object, index1_0])
@@ -812,12 +790,12 @@
 
 @expose_primitive(MOUSE_BUTTONS, unwrap_spec=[object])
 def func(interp, s_frame, w_rcvr):
-    btn = interp.space.get_display().mouse_button()
+    btn = interp.space.display().mouse_button()
     return interp.space.wrap_int(btn)
 
 @expose_primitive(KBD_NEXT, unwrap_spec=[object])
 def func(interp, s_frame, w_rcvr):
-    code = interp.space.get_display().next_keycode()
+    code = interp.space.display().next_keycode()
     if code & 0xFF == 0:
         return interp.space.w_nil
     else:
@@ -825,7 +803,7 @@
 
 @expose_primitive(KBD_PEEK, unwrap_spec=[object])
 def func(interp, s_frame, w_rcvr):
-    code = interp.space.get_display().peek_keycode()
+    code = interp.space.display().peek_keycode()
     if code & 0xFF == 0:
         return interp.space.w_nil
     else:
@@ -864,8 +842,8 @@
 
 @expose_primitive(EXIT_TO_DEBUGGER, unwrap_spec=[object])
 def func(interp, s_frame, w_rcvr):
-    if not objectmodel.we_are_translated():
-        import pdb; pdb.set_trace()
+    if interp.space.headless.is_set():
+        exitFromHeadlessExecution(s_frame, "EXIT_TO_DEBUGGER")
     raise PrimitiveNotYetWrittenError()
 
 @expose_primitive(CHANGE_CLASS, unwrap_spec=[object, object], no_result=True)
@@ -987,7 +965,7 @@
 def func(interp, s_frame, argument_count):
     if argument_count == 0:
         s_frame.pop()
-        return interp.space.wrap_string(interp.image_name)
+        return interp.space.wrap_string(interp.space.image_name())
     elif argument_count == 1:
         pass # XXX
     raise PrimitiveFailedError
@@ -1004,7 +982,7 @@
 
 @expose_primitive(DEFER_UPDATES, unwrap_spec=[object, bool])
 def func(interp, s_frame, w_receiver, flag):
-    sdldisplay = interp.space.get_display()
+    sdldisplay = interp.space.display()
     sdldisplay.defer_updates(flag)
     return w_receiver
 
@@ -1058,7 +1036,7 @@
 
 @expose_primitive(SET_INTERRUPT_KEY, unwrap_spec=[object, int])
 def func(interp, s_frame, w_rcvr, encoded_key):
-    interp.space.get_display().set_interrupt_key(interp.space, encoded_key)
+    interp.space.display().set_interrupt_key(interp.space, encoded_key)
     return w_rcvr
 
 @expose_primitive(INTERRUPT_SEMAPHORE, unwrap_spec=[object, object])
@@ -1142,7 +1120,7 @@
             raise PrimitiveFailedError
         for i in xrange(w_arg.size()):
             w_arg.setchar(i, chr(new_value))
-    elif isinstance(w_arg, model.W_WordsObject) or isinstance(w_arg, model.W_DisplayBitmap):
+    elif isinstance(w_arg, model.W_WordsObject) or isinstance(w_arg, model_display.W_DisplayBitmap):
         for i in xrange(w_arg.size()):
             w_arg.setword(i, new_value)
     else:
@@ -1531,7 +1509,7 @@
 
 @expose_primitive(FORCE_DISPLAY_UPDATE, unwrap_spec=[object])
 def func(interp, s_frame, w_rcvr):
-    interp.space.get_display().flip(force=True)
+    interp.space.display().flip(force=True)
     return w_rcvr
 
 # ___________________________________________________________________________
diff --git a/spyvm/squeakimage.py b/spyvm/squeakimage.py
--- a/spyvm/squeakimage.py
+++ b/spyvm/squeakimage.py
@@ -386,12 +386,13 @@
         self.startup_time = time.time()
 
     def run_spy_hacks(self, space):
-        pass
-        # w_display = space.objtable["w_display"]
-        # if w_display is not None and not w_display.is_nil(space):
-        #     if space.unwrap_int(w_display.fetch(space, 3)) < 8:
-        #         # non-native indexed color depth not well supported
-        #         w_display.store(space, 3, space.wrap_int(8))
+        if not space.run_spy_hacks.is_set():
+            return
+        w_display = space.objtable["w_display"]
+        if w_display is not None and not w_display.is_nil(space):
+            if space.unwrap_int(w_display.fetch(space, 3)) < 8:
+                # non-native indexed color depth not well supported
+                w_display.store(space, 3, space.wrap_int(8))
 
     def find_symbol(self, space, reader, symbol):
         w_dnu = self.special(constants.SO_DOES_NOT_UNDERSTAND)
diff --git a/spyvm/test/test_model.py b/spyvm/test/test_model.py
--- a/spyvm/test/test_model.py
+++ b/spyvm/test/test_model.py
@@ -1,5 +1,5 @@
 import py, math, socket
-from spyvm import model, shadow, objspace, error, display
+from spyvm import model, model_display, shadow, objspace, error, display
 from spyvm.shadow import MethodNotFound, WEAK_POINTERS
 from rpython.rlib.rarithmetic import intmask, r_uint
 from rpython.rtyper.lltypesystem import lltype, rffi
@@ -387,7 +387,7 @@
     d = display.SDLDisplay("test")
     d.set_video_mode(32, 10, 1)
 
-    target = model.W_DisplayBitmap.create(space, space.w_Array, 10, 1, d)
+    target = model_display.W_DisplayBitmap.create(space, space.w_Array, 10, 1, d)
     target.setword(0, r_uint(0xFF00))
     assert bin(target.getword(0)) == bin(0xFF00)
     target.setword(0, r_uint(0x00FF00FF))
@@ -404,15 +404,7 @@
         assert target.pixelbuffer[i] == 0x0
 
 def test_display_offset_computation():
-
-    def get_pixelbuffer(self):
-        return lltype.malloc(rffi.ULONGP.TO, self.width * self.height * 32, flavor='raw')
-    display.SDLDisplay.get_pixelbuffer = get_pixelbuffer
-    d = display.SDLDisplay("test")
-    d.set_video_mode(18, 5, 1)
-
-    dbitmap = model.W_DisplayBitmap.create(space, space.w_Array, 5, 1, d)
-
+    dbitmap = model_display.W_MappingDisplayBitmap(space, space.w_Array, 5, 1)
     assert dbitmap.compute_pos(0) == 0
     assert dbitmap.compute_pos(1) == 8
     assert dbitmap.size() == 5 * 8
diff --git a/spyvm/test/test_primitives.py b/spyvm/test/test_primitives.py
--- a/spyvm/test/test_primitives.py
+++ b/spyvm/test/test_primitives.py
@@ -1,5 +1,5 @@
 import py, os, math, time
-from spyvm import model, shadow, interpreter, constants, primitives, objspace, wrapper, display
+from spyvm import model, model_display, shadow, interpreter, constants, primitives, objspace, wrapper, display
 from spyvm.primitives import prim_table, PrimitiveFailedError
 from spyvm.plugins import bitblt
 from rpython.rlib.rfloat import INFINITY, NAN, isinf, isnan
@@ -47,7 +47,8 @@
         frame = context
         for i in range(len(stack)):
             frame.as_context_get_shadow(space).push(stack[i])
-    interp = TestInterpreter(space, image_name=IMAGENAME)
+    interp = TestInterpreter(space)
+    interp.space._image_name.set(IMAGENAME)
     return interp, frame, len(stack)
 
 def _prim(space, code, stack, context = None):
@@ -680,7 +681,7 @@
 
     closure = space.newClosure(w_frame, 4, 0, [])
     s_frame = w_frame.as_methodcontext_get_shadow(space)
-    interp = TestInterpreter(space, image_name=IMAGENAME)
+    interp = TestInterpreter(space)
     interp._loop = True
 
     try:
@@ -721,7 +722,7 @@
     assert space.objtable["w_display"] is mock_display
     w_bitmap = mock_display.fetch(space, 0)
     assert w_bitmap is not w_wordbmp
-    assert isinstance(w_bitmap, model.W_DisplayBitmap)
+    assert isinstance(w_bitmap, model_display.W_DisplayBitmap)
     sdldisplay = w_bitmap.display
     assert isinstance(sdldisplay, display.SDLDisplay)
 
@@ -733,7 +734,7 @@
     prim(primitives.BE_DISPLAY, [mock_display2])
     assert space.objtable["w_display"] is mock_display2
     w_bitmap2 = mock_display.fetch(space, 0)
-    assert isinstance(w_bitmap2, model.W_DisplayBitmap)
+    assert isinstance(w_bitmap2, model_display.W_DisplayBitmap)
     assert w_bitmap.display is w_bitmap2.display
     assert sdldisplay.width == 32
     assert sdldisplay.height == 10
@@ -764,7 +765,7 @@
         raise DisplayFlush
 
     try:
-        monkeypatch.setattr(space.get_display().__class__, "flip", flush_to_screen_mock)
+        monkeypatch.setattr(space.display().__class__, "flip", flush_to_screen_mock)
         with py.test.raises(DisplayFlush):
             prim(primitives.FORCE_DISPLAY_UPDATE, [mock_display])
     finally:
diff --git a/spyvm/wrapper.py b/spyvm/wrapper.py
--- a/spyvm/wrapper.py
+++ b/spyvm/wrapper.py
@@ -1,4 +1,4 @@
-from spyvm import model, constants
+from spyvm import model, model_display, constants
 from spyvm.error import FatalError, WrapperException, PrimitiveFailedError
 
 class Wrapper(object):
@@ -263,6 +263,30 @@
     def size(self):
         return self._w_self.size() - constants.BLKCLSR_SIZE
 
+class FormWrapper(Wrapper):
+    bits, store_bits = make_getter_setter(constants.FORM_BITS)
+    width, store_width = make_int_getter_setter(constants.FORM_WIDTH)
+    height, store_height = make_int_getter_setter(constants.FORM_HEIGHT)
+    depth, store_depth = make_int_getter_setter(constants.FORM_DEPTH)
+    
+    def create_display_bitmap(self):
+        w_display_bitmap = model_display.from_words_object(self.bits(), self)
+        self.store_bits(w_display_bitmap)
+        return w_display_bitmap
+    
+    def get_display_bitmap(self):
+        w_bitmap = self.bits()
+        if not isinstance(w_bitmap, model_display.W_DisplayBitmap):
+            w_display_bitmap = self.create_display_bitmap()
+        else:
+            w_display_bitmap = w_bitmap
+            if w_display_bitmap._depth != self.depth():
+                w_display_bitmap = self.create_display_bitmap()
+        return w_display_bitmap
+    
+    def take_over_display(self):
+        self.space.display().set_video_mode(self.width(), self.height(), self.depth())
+    
 # XXX Wrappers below are not used yet.
 class OffsetWrapper(Wrapper):
     offset_x  = make_int_getter(0)
diff --git a/targetimageloadingsmalltalk.py b/targetimageloadingsmalltalk.py
--- a/targetimageloadingsmalltalk.py
+++ b/targetimageloadingsmalltalk.py
@@ -36,6 +36,7 @@
             -p|--poll          - Actively poll for events. Try this if the image is not responding well.
             -i|--no-interrupts - Disable timer interrupt. Disables non-cooperative scheduling.
             -S                 - Disable specialized storage strategies; always use generic ListStorage
+            --hacks            - Enable Spy hacks. Set display color depth to 8.
             
           Logging parameters:
             -t|--trace                 - Output a trace of each message, primitive, return value and process switch.
@@ -69,7 +70,7 @@
     except error.Exit, e:
         print_error("Exited: %s" % e.msg)
         return -1
-    except Exception, e:
+    except BaseException, e:
         print_error("Exception: %s" % str(e))
         if not objectmodel.we_are_translated():
             import traceback
@@ -119,8 +120,10 @@
                 interrupts = False
             elif arg in ["-P", "--process"]:
                 headless = False
+            elif arg in ["--hacks"]:
+                space.run_spy_hacks.activate()
             elif arg in ["-S"]:
-                space.no_specialized_storage.set()
+                space.no_specialized_storage.activate()
             elif arg in ["-u"]:
                 from spyvm.plugins.vmdebugging import stop_ui_process
                 stop_ui_process()
@@ -158,10 +161,10 @@
     # Load & prepare image and environment
     image_reader = squeakimage.reader_for_image(space, squeakimage.Stream(data=imagedata))
     image = create_image(space, image_reader)
-    interp = interpreter.Interpreter(space, image, image_name=path,
+    interp = interpreter.Interpreter(space, image,
                 trace=trace, evented=not poll,
                 interrupts=interrupts)
-    space.runtime_setup(argv[0])
+    space.runtime_setup(argv[0], path)
     print_error("") # Line break after image-loading characters
     
     # Create context to be executed
@@ -174,7 +177,7 @@
             selector = compile_code(interp, w_receiver, code)
         s_frame = create_context(interp, w_receiver, selector, stringarg)
         if headless:
-            space.headless.set()
+            space.headless.activate()
             context = s_frame
         else:
             create_process(interp, s_frame)
@@ -203,7 +206,7 @@
     # registered (primitive 136 not called), so the idle process will never be left once it is entered.
     # TODO - Find a way to cleanly initialize the image, without executing the active_context of the image.
     # Instead, we want to execute our own context. Then remove this flag (and all references to it)
-    space.suppress_process_switch.set()
+    space.suppress_process_switch.activate()
     
     w_result = interp.perform(
         w_receiver_class,
@@ -215,7 +218,7 @@
     # TODO - is this expected in every image?
     if not isinstance(w_result, model.W_BytesObject) or w_result.as_string() != selector:
         raise error.Exit("Unexpected compilation result (probably failed to compile): %s" % result_string(w_result))
-    space.suppress_process_switch.unset()
+    space.suppress_process_switch.deactivate()
     
     w_receiver_class.as_class_get_shadow(space).s_methoddict().sync_method_cache()
     return selector

From noreply at buildbot.pypy.org  Tue Jul 22 23:27:24 2014
From: noreply at buildbot.pypy.org (anton_gulenko)
Date: Tue, 22 Jul 2014 23:27:24 +0200 (CEST)
Subject: [pypy-commit] lang-smalltalk storage-context-state: Merged.
Message-ID: <20140722212724.9FFF61D2875@cobra.cs.uni-duesseldorf.de>

Author: Anton Gulenko 
Branch: storage-context-state
Changeset: r941:4e7e88455fcc
Date: 2014-07-22 23:12 +0200
http://bitbucket.org/pypy/lang-smalltalk/changeset/4e7e88455fcc/

Log:	Merged.

diff --git a/images/Squeak4.5-noBitBlt.changes b/images/Squeak4.5-noBitBlt.changes
--- a/images/Squeak4.5-noBitBlt.changes
+++ b/images/Squeak4.5-noBitBlt.changes
@@ -12620,4 +12620,15 @@
 	
 	1 to: self splayTreeSize do: [:i |
 		self insertNewNode.
-	]! !

----SNAPSHOT----{15 July 2014 . 6:10:56 pm} Squeak4.5-noBitBlt.image priorSource: 15894330!
\ No newline at end of file
+	]! !

----SNAPSHOT----{15 July 2014 . 6:10:56 pm} Squeak4.5-noBitBlt.image priorSource: 15894330!
+
+----QUIT/NOSAVE----{21 July 2014 . 4:18:39 pm} Squeak4.5-noBitBlt.image priorSource: 15894825!

----STARTUP----{21 July 2014 . 6:19:06 pm} as C:\Dev\lang-smalltalk\images\Squeak4.5-noBitBlt.image!

!WeakMessageSend methodsFor: 'private' stamp: 'ag 7/21/2014 18:20' prior: 34321504!
withEnsuredReceiverAndArgumentsDo: aBlock otherwise: altBlock
	"Grab real references to receiver and arguments. If they still exist, evaluate aBlock."

	"Return if my receiver has gone away"
	| r a |
	r := self receiver.
	r ifNil: [ ^altBlock value ].

	"Make sure that my arguments haven't gone away"
	arguments ifNil: [ ^ altBlock value ].
	a := Array withAll: arguments.
	a with: shouldBeNil do: [ :arg :flag |
		arg ifNil: [ flag ifFalse: [ ^altBlock value ]]
	].

	^aBlock value: r value: a! !

----QUIT----{21 July 2014 . 6:20:43 pm} Squeak4.5-noBitBlt.image priorSource: 15894825!
+
+----QUIT/NOSAVE----{21 July 2014 . 4:21:36 pm} Squeak4.5-noBitBlt.image priorSource: 15895702!

----STARTUP----{21 July 2014 . 6:21:54 pm} as C:\Dev\lang-smalltalk\images\Squeak4.5-noBitBlt.image!

!WeakMessageSend methodsFor: 'comparing' stamp: 'ag 7/21/2014 18:22' prior: 33144463!
= anObject
	"Compare equal to equivalent MessageSend"
	^ anObject isMessageSend
		and: [self receiver == anObject receiver
		and: [selector == anObject selector
		and: [(Array withAll: self arguments) = (Array withAll: anObject arguments)]]]
! !
!WeakMessageSend methodsFor: 'private' stamp: 'ag 7/21/2014 18:23' prior: 49449636!
withEnsuredReceiverAndArgumentsDo: aBlock otherwise: altBlock
	"Grab real references to receiver and arguments. If they still exist, evaluate aBlock."

	"Return if my receiver has gone away"
	| r a |
	r := self receiver.
	r ifNil: [ ^altBlock value ].

	"Make sure that my arguments haven't gone away"
	a := Array withAll: self arguments.
	a with: shouldBeNil do: [ :arg :flag |
		arg ifNil: [ flag ifFalse: [ ^altBlock value ]]
	].

	^aBlock value: r value: a! !

----QUIT----{21 July 2014 . 6:23:49 pm} Squeak4.5-noBitBlt.image priorSource: 15895702!

----STARTUP----{21 July 2014 . 6:31:05 pm} as C:\Dev\lang-smalltalk\images\Squeak4.5-noBitBlt.image!

!WeakMessageSend methodsFor: 'accessing' stamp: 'ag 7/21/2014 18:31'!
shouldBeNil
	
	^ shouldBeNil ifNil: [ Array new ]! !
!WeakMessageSend methodsFor: 'private' stamp: 'ag 7/21/2014 18:31' prior: 33148869!
isAnyArgumentGarbage
	"Make sure that my arguments haven't gone away"
	arguments ifNotNil: [
		arguments with: self shouldBeNil do: [ :arg :flag |
			(flag not and: [arg isNil])
				ifTrue: [^true]
		]
	].
	^false
! !
!WeakMessageSend methodsFor: 'private' stamp: 'ag 7/21/2014 18:31' prior: 49450841!
withEnsuredReceiverAndArgumentsDo: aBlock otherwise: altBlock
	"Grab real references to receiver and arguments. If they still exist, evaluate aBlock."

	"Return if my receiver has gone away"
	| r a |
	r := self receiver.
	r ifNil: [ ^altBlock value ].

	"Make sure that my arguments haven't gone away"
	a := Array withAll: self arguments.
	a with: self shouldBeNil do: [ :arg :flag |
		arg ifNil: [ flag ifFalse: [ ^altBlock value ]]
	].

	^aBlock value: r value: a! !
!WeakMessageSend methodsFor: 'private' stamp: 'ag 7/21/2014 18:32' prior: 34360552!
withEnsuredReceiverAndArgumentsDo: aBlock withEnoughArguments: anArray otherwise: altBlock
	"call the selector with enough arguments from arguments and anArray"
	| r selfArgs enoughArgs |
	r := self receiver.
	r ifNil: [ ^altBlock value ].
	
	selfArgs := self arguments.
	selfArgs with: self shouldBeNil do: [ :arg :flag |
		arg ifNil: [ flag ifFalse: [ ^altBlock value ]]
	].

	enoughArgs := Array new: selector numArgs.
	enoughArgs replaceFrom: 1
		to: ( selfArgs size min: enoughArgs size)
		with: selfArgs
		startingAt: 1.
	enoughArgs size > selfArgs size ifTrue: [
		enoughArgs replaceFrom: selfArgs size + 1
			to: (selfArgs size + anArray size min: enoughArgs size)
			with: anArray
			startingAt: 1.
	].
	^aBlock value: r value: enoughArgs! !

----QUIT----{21 July 2014 . 6:32:32 pm} Squeak4.5-noBitBlt.image priorSource: 15896872!
+
+----STARTUP----{21 July 2014 . 4:32:52 pm} as C:\Dev\lang-smalltalk\images\Squeak4.5-noBitBlt.image!
+
+
+1+1!
+
+----QUIT/NOSAVE----{21 July 2014 . 4:33:15 pm} Squeak4.5-noBitBlt.image priorSource: 15898877!
\ No newline at end of file
diff --git a/images/Squeak4.5-noBitBlt.image b/images/Squeak4.5-noBitBlt.image
index ed92c78c940799d91bb94a8ed8527076db6816c7..00843c2c83f9c11e5dcfa3b9927bd415d0a22cd8
GIT binary patch

[cut]

diff --git a/spyvm/constants.py b/spyvm/constants.py
--- a/spyvm/constants.py
+++ b/spyvm/constants.py
@@ -54,6 +54,11 @@
 BLKCLSR_NUMARGS = 2
 BLKCLSR_SIZE = 3
 
+FORM_BITS = 0
+FORM_WIDTH = 1
+FORM_HEIGHT = 2
+FORM_DEPTH = 3
+
 # ___________________________________________________________________________
 # Miscellaneous constants
 
diff --git a/spyvm/display.py b/spyvm/display.py
--- a/spyvm/display.py
+++ b/spyvm/display.py
@@ -35,12 +35,13 @@
 WindowEventPaint = 5
 WindowEventStinks = 6
 
+MINIMUM_DEPTH = 8
 
 class SDLDisplay(object):
     _attrs_ = ["screen", "width", "height", "depth", "surface", "has_surface",
                "mouse_position", "button", "key", "interrupt_key", "_defer_updates",
-               "_deferred_event", "pixelbuffer"]
-    _immutable_fields_ = ["pixelbuffer?"]
+               "_deferred_event", "bpp", "pitch"]
+    #_immutable_fields_ = ["pixelbuffer?"]
 
     def __init__(self, title):
         assert RSDL.Init(RSDL.INIT_VIDEO) >= 0
@@ -58,26 +59,27 @@
     def set_video_mode(self, w, h, d):
         assert w > 0 and h > 0
         assert d in [1, 2, 4, 8, 16, 32]
+        if d < MINIMUM_DEPTH:
+            d = MINIMUM_DEPTH
         self.width = w
         self.height = h
         self.depth = d
         flags = RSDL.HWPALETTE | RSDL.RESIZABLE | RSDL.ASYNCBLIT | RSDL.DOUBLEBUF
-        if d < 8:
-            d = 8
         self.screen = RSDL.SetVideoMode(w, h, d, flags)
         if not self.screen:
             print "Could not open display at depth %d" % d
             raise RuntimeError
-        elif d == 8:
+        elif d == MINIMUM_DEPTH:
             self.set_squeak_colormap(self.screen)
-        self.pixelbuffer = rffi.cast(rffi.UINTP, self.screen.c_pixels)
-
+        self.bpp = rffi.getintfield(self.screen.c_format, 'c_BytesPerPixel')
+        self.pitch = rffi.getintfield(self.screen, 'c_pitch')
+    
     def get_pixelbuffer(self):
-        return jit.promote(self.pixelbuffer)
-
+        return jit.promote(rffi.cast(RSDL.Uint32P, self.screen.c_pixels))
+    
     def defer_updates(self, flag):
         self._defer_updates = flag
-
+    
     def flip(self, force=False):
         if (not self._defer_updates) or force:
             RSDL.Flip(self.screen)
diff --git a/spyvm/interpreter.py b/spyvm/interpreter.py
--- a/spyvm/interpreter.py
+++ b/spyvm/interpreter.py
@@ -23,7 +23,7 @@
 
 
 class Interpreter(object):
-    _immutable_fields_ = ["space", "image", "image_name",
+    _immutable_fields_ = ["space", "image",
                           "interrupt_counter_size",
                           "startup_time", "evented", "interrupts"]
 
@@ -34,12 +34,11 @@
         get_printable_location=get_printable_location
     )
 
-    def __init__(self, space, image=None, image_name="",
+    def __init__(self, space, image=None,
                 trace=False, evented=True, interrupts=True):
         # === Initialize immutable variables
         self.space = space
         self.image = image
-        self.image_name = image_name
         if image:
             self.startup_time = image.startup_time
         else:
@@ -1033,9 +1032,8 @@
 # in order to enable tracing/jumping for message sends etc.
 def debugging():
     def stepping_debugger_init(original):
-        def meth(self, space, image=None, image_name="", trace=False):
-            return_value = original(self, space, image=image,
-                                    image_name=image_name, trace=trace)
+        def meth(self, space, image=None, trace=False):
+            return_value = original(self, space, image=image, trace=trace)
             # ##############################################################
 
             self.message_stepping = False
diff --git a/spyvm/interpreter_proxy.py b/spyvm/interpreter_proxy.py
--- a/spyvm/interpreter_proxy.py
+++ b/spyvm/interpreter_proxy.py
@@ -17,7 +17,7 @@
 from rpython.rtyper.lltypesystem import lltype, rffi
 from rpython.rlib.unroll import unrolling_iterable
 
-from spyvm import error, model, objspace
+from spyvm import error, model, model_display, objspace, wrapper
 
 sqInt = rffi.INT
 sqLong = rffi.LONG
@@ -242,7 +242,7 @@
         return w_object.convert_to_c_layout()
     elif isinstance(w_object, model.W_BytesObject):
         return rffi.cast(sqIntArrayPtr, w_object.convert_to_c_layout())
-    elif isinstance(w_object, model.W_DisplayBitmap):
+    elif isinstance(w_object, model_display.W_DisplayBitmap):
         return rffi.cast(sqIntArrayPtr, w_object.convert_to_c_layout())
     else:
         raise ProxyFunctionFailed
@@ -529,7 +529,7 @@
 @expose_on_virtual_machine_proxy([], int)
 def fullDisplayUpdate():
     w_display = IProxy.space.objtable['w_display']
-    if isinstance(w_display, model.W_DisplayBitmap):
+    if isinstance(w_display, model_display.W_DisplayBitmap):
         w_display.update_from_buffer()
         w_display.flush_to_screen()
         return 0
@@ -559,16 +559,8 @@
     # display memory
     space = IProxy.space
     if w_dest_form.is_same_object(space.objtable['w_display']):
-        w_bitmap = w_dest_form.fetch(space, 0)
-        if not isinstance(w_bitmap, model.W_DisplayBitmap):
-            assert isinstance(w_bitmap, model.W_WordsObject)
-            w_display_bitmap = w_bitmap.as_display_bitmap(
-                w_dest_form,
-                IProxy.interp,
-                sdldisplay=None
-            )
-        else:
-            w_display_bitmap = w_bitmap
+        form = wrapper.FormWrapper(space, w_dest_form)
+        w_display_bitmap = form.get_display_bitmap()
         w_display_bitmap.update_from_buffer()
         w_display_bitmap.flush_to_screen()
     return 0
@@ -1008,7 +1000,7 @@
         self.argcount = 0
         self.w_method = None
         self.fail_reason = 0
-        self.trace_proxy.unset()
+        self.trace_proxy.deactivate()
 
     def call(self, signature, interp, s_frame, argcount, w_method):
         self.initialize_from_call(signature, interp, s_frame, argcount, w_method)
@@ -1050,7 +1042,7 @@
         self.argcount = argcount
         self.w_method = w_method
         self.space = interp.space
-        self.trace_proxy.set_to(interp.trace_proxy.is_set())
+        self.trace_proxy.set(interp.trace_proxy.is_set())
         # ensure that space.w_nil gets the first possible oop
         self.object_to_oop(self.space.w_nil)
 
diff --git a/spyvm/model.py b/spyvm/model.py
--- a/spyvm/model.py
+++ b/spyvm/model.py
@@ -987,26 +987,6 @@
                 c_words[i] = intmask(old_words[i])
             self.words = None
             return c_words
-
-    def as_display_bitmap(self, w_form, interp, sdldisplay=None):
-        width = interp.space.unwrap_int(w_form.fetch(interp.space, 1))
-        height = interp.space.unwrap_int(w_form.fetch(interp.space, 2))
-        depth = interp.space.unwrap_int(w_form.fetch(interp.space, 3))
-        if not sdldisplay:
-            from spyvm import display
-            sdldisplay = display.SDLDisplay(interp.image_name)
-            sdldisplay.set_video_mode(width, height, depth)
-        w_display_bitmap = W_DisplayBitmap.create(
-            interp.space,
-            self.getclass(interp.space),
-            self.size(),
-            depth,
-            sdldisplay
-        )
-        for idx in range(self.size()):
-            w_display_bitmap.setword(idx, self.getword(idx))
-        w_form.store(interp.space, 0, w_display_bitmap)
-        return w_display_bitmap
     
     def _become(self, w_other):
         assert isinstance(w_other, W_WordsObject)
@@ -1019,144 +999,6 @@
         if self.words is None:
             lltype.free(self.c_words, flavor='raw')
 
-class W_DisplayBitmap(W_AbstractObjectWithClassReference):
-    _attrs_ = ['pixelbuffer', '_realsize', '_real_depth_buffer', 'display', '_depth']
-    _immutable_fields_ = ['_realsize', 'display', '_depth']
-    repr_classname = "W_DisplayBitmap"
-    
-    pixelbuffer = None
-    
-    @staticmethod
-    def create(space, w_class, size, depth, display):
-        if depth < 8:
-            return W_MappingDisplayBitmap(space, w_class, size * (8 / depth), depth, display)
-        elif depth == 8:
-            return W_8BitDisplayBitmap(space, w_class, size, depth, display)
-        elif depth == 16:
-            return W_16BitDisplayBitmap(space, w_class, size, depth, display)
-        else:
-            return W_DisplayBitmap(space, w_class, size, depth, display)
-
-    def repr_content(self):
-        return "len=%d depth=%d %s" % (self.size(), self._depth, self.str_content())
-    
-    def __init__(self, space, w_class, size, depth, display):
-        W_AbstractObjectWithClassReference.__init__(self, space, w_class)
-        self._real_depth_buffer = lltype.malloc(rffi.CArray(rffi.UINT), size, flavor='raw')
-        self._realsize = size
-        self.display = display
-        self._depth = depth
-
-    def at0(self, space, index0):
-        val = self.getword(index0)
-        return space.wrap_uint(val)
-
-    def atput0(self, space, index0, w_value):
-        word = space.unwrap_uint(w_value)
-        self.setword(index0, word)
-
-    def flush_to_screen(self):
-        self.display.flip()
-
-    def size(self):
-        return self._realsize
-        
-    def invariant(self):
-        return False
-
-    def clone(self, space):
-        w_result = W_WordsObject(space, self.getclass(space), self._realsize)
-        n = 0
-        while n < self._realsize:
-            w_result.words[n] = self.getword(n)
-            n += 1
-        return w_result
-
-    def getword(self, n):
-        assert self.size() > n >= 0
-        return self._real_depth_buffer[n]
-
-    def setword(self, n, word):
-        self._real_depth_buffer[n] = word
-        self.display.get_pixelbuffer()[n] = word
-
-    def is_array_object(self):
-        return True
-
-    def update_from_buffer(self):
-        for i in range(self._realsize):
-            self.setword(i, self.getword(i))
-
-    def convert_to_c_layout(self):
-        return self._real_depth_buffer
-
-    def can_become(self, w_other):
-        # TODO - implement _become() for this class. Impossible due to _immutable_fields_?
-        return False
-    
-    def __del__(self):
-        lltype.free(self._real_depth_buffer, flavor='raw')
-
-
-class W_16BitDisplayBitmap(W_DisplayBitmap):
-    repr_classname = "W_16BitDisplayBitmap"
-    def setword(self, n, word):
-        self._real_depth_buffer[n] = word
-        mask = 0b11111
-        lsb = (r_uint(word) & r_uint(0xffff0000)) >> 16
-        msb = (r_uint(word) & r_uint(0x0000ffff))
-
-        lsb = (
-            ((lsb >> 10) & mask) |
-            (((lsb >> 5) & mask) << 6) |
-            ((lsb & mask) << 11)
-        )
-        msb = (
-            ((msb >> 10) & mask) |
-            (((msb >> 5) & mask) << 6) |
-            ((msb & mask) << 11)
-        )
-
-        self.display.get_pixelbuffer()[n] = r_uint(lsb | (msb << 16))
-
-
-class W_8BitDisplayBitmap(W_DisplayBitmap):
-    repr_classname = "W_8BitDisplayBitmap"
-    def setword(self, n, word):
-        self._real_depth_buffer[n] = word
-        self.display.get_pixelbuffer()[n] = r_uint(
-            (word >> 24) |
-            ((word >> 8) & 0x0000ff00) |
-            ((word << 8) & 0x00ff0000) |
-            (word << 24)
-        )
-
-NATIVE_DEPTH = 8
-class W_MappingDisplayBitmap(W_DisplayBitmap):
-    repr_classname = "W_MappingDisplayBitmap"
-    @jit.unroll_safe
-    def setword(self, n, word):
-        self._real_depth_buffer[n] = word
-        word = r_uint(word)
-        pos = self.compute_pos(n)
-        assert self._depth <= 4
-        rshift = 32 - self._depth
-        for i in xrange(8 / self._depth):
-            if pos >= self.size():
-                return
-            mapword = r_uint(0)
-            for i in xrange(4):
-                pixel = r_uint(word) >> rshift
-                mapword |= (r_uint(pixel) << (i * 8))
-                word <<= self._depth
-            self.display.get_pixelbuffer()[pos] = mapword
-            pos += 1
-
-    def compute_pos(self, n):
-        return n * (NATIVE_DEPTH / self._depth)
-        
-# XXX Shouldn't compiledmethod have class reference for subclassed compiled
-# methods?
 class W_CompiledMethod(W_AbstractObjectWithIdentityHash):
     """My instances are methods suitable for interpretation by the virtual machine.  This is the only class in the system whose instances intermix both indexable pointer fields and indexable integer fields.
 
@@ -1467,8 +1309,3 @@
             if isinstance(s_class, ClassShadow):
                 return "%s >> #%s" % (s_class.getname(), self.lookup_selector)
         return "#%s" % self.lookup_selector
-
-class DetachingShadowError(Exception):
-    def __init__(self, old_shadow, new_shadow_class):
-        self.old_shadow = old_shadow
-        self.new_shadow_class = new_shadow_class
diff --git a/spyvm/model_display.py b/spyvm/model_display.py
new file mode 100644
--- /dev/null
+++ b/spyvm/model_display.py
@@ -0,0 +1,196 @@
+
+from spyvm import model, constants, display
+from rpython.rlib import jit, objectmodel
+from rpython.rtyper.lltypesystem import lltype, rffi
+from rpython.rlib.rarithmetic import r_uint
+
+def from_words_object(w_obj, form):
+    depth = form.depth()
+    space = form.space
+    size = w_obj.size()
+    w_class = w_obj.getclass(space)
+    
+    if depth < 8:
+        w_display_bitmap = W_MappingDisplayBitmap(space, w_class, size, depth)
+    elif depth == 8:
+        w_display_bitmap = W_8BitDisplayBitmap(space, w_class, size, depth)
+    elif depth == 16:
+        w_display_bitmap = W_16BitDisplayBitmap(space, w_class, size, depth)
+    else:
+        w_display_bitmap = W_DisplayBitmap(space, w_class, size, depth)
+    
+    for idx in range(size):
+        w_display_bitmap.setword(idx, w_obj.getword(idx))
+    
+    return w_display_bitmap
+
+class W_DisplayBitmap(model.W_AbstractObjectWithClassReference):
+    _attrs_ = ['pixelbuffer_words', '_real_depth_buffer', '_realsize', 'display', '_depth']
+    _immutable_fields_ = ['pixelbuffer_words?', '_real_depth_buffer', '_realsize', 'display', '_depth']
+    repr_classname = "W_DisplayBitmap"
+    
+    def __init__(self, space, w_class, size, depth):
+        model.W_AbstractObjectWithClassReference.__init__(self, space, w_class)
+        self._real_depth_buffer = lltype.malloc(rffi.CArray(rffi.UINT), size, flavor='raw')
+        self._realsize = size
+        self._depth = depth
+        self.display = space.display()
+        self.relinquish_display()
+    
+    # === Object access
+    
+    def at0(self, space, index0):
+        val = self.getword(index0)
+        return space.wrap_uint(val)
+    
+    def atput0(self, space, index0, w_value):
+        word = space.unwrap_uint(w_value)
+        self.setword(index0, word)
+    
+    def getword(self, n):
+        assert self.size() > n >= 0
+        return self._real_depth_buffer[n]
+    
+    def setword(self, n, word):
+        self._real_depth_buffer[n] = word
+        if self.pixelbuffer_words > 0:
+            self.set_pixelbuffer_word(n, word)
+    
+    def size(self):
+        return self._realsize
+    
+    # === Graphics
+    
+    def pixelbuffer(self):
+        return self.display.get_pixelbuffer()
+    
+    def pixelbuffer_UCHAR(self):
+        return self.display.get_pixelbuffer_UCHAR()
+    
+    def set_pixelbuffer_word(self, n, word):
+        self.pixelbuffer()[n] = word
+    
+    def take_over_display(self):
+        # Make sure FrameWrapper.take_over_display() is called first for the correct Frame object.
+        pixel_per_word = constants.BYTES_PER_WORD / (self.display.depth / 8)
+        self.pixelbuffer_words = self.display.width * self.display.height / pixel_per_word
+        self.update_from_buffer()
+    
+    def relinquish_display(self):
+        self.pixelbuffer_words = 0
+    
+    def flush_to_screen(self):
+        self.display.flip()
+    
+    def update_from_buffer(self):
+        if self.pixelbuffer_words > 0:
+            for i in range(self.size()):
+                self.set_pixelbuffer_word(i, self.getword(i))
+    
+    # === Misc
+    
+    def invariant(self):
+        return False
+
+    def clone(self, space):
+        w_result = model.W_WordsObject(space, self.getclass(space), self.size())
+        for n in range(self.size()):
+            w_result.setword(n, self.getword(n))
+        return w_result
+    
+    def is_array_object(self):
+        return True
+
+    def convert_to_c_layout(self):
+        return self._real_depth_buffer
+
+    def can_become(self, w_other):
+        # TODO - implement _become() for this class. Impossible due to _immutable_fields_?
+        return False
+    
+    def __del__(self):
+        lltype.free(self._real_depth_buffer, flavor='raw')
+    
+    def repr_content(self):
+        return "len=%d depth=%d %s" % (self.size(), self._depth, self.str_content())
+
+class W_16BitDisplayBitmap(W_DisplayBitmap):
+    
+    repr_classname = "W_16BitDisplayBitmap"
+    
+    def set_pixelbuffer_word(self, n, word):
+        mask = 0b11111
+        lsb = (r_uint(word) & r_uint(0xffff0000)) >> 16
+        msb = (r_uint(word) & r_uint(0x0000ffff))
+        
+        # Invert order of rgb-components
+        lsb = (
+            ((lsb >> 10) & mask) |
+            (((lsb >> 5) & mask) << 6) |
+            ((lsb & mask) << 11)
+        )
+        msb = (
+            ((msb >> 10) & mask) |
+            (((msb >> 5) & mask) << 6) |
+            ((msb & mask) << 11)
+        )
+        
+        self.pixelbuffer()[n] = r_uint(lsb | (msb << 16))
+
+class W_8BitDisplayBitmap(W_DisplayBitmap):
+    
+    repr_classname = "W_8BitDisplayBitmap"
+    
+    def set_pixelbuffer_word(self, n, word):
+        # Invert the byte-order.
+        self.pixelbuffer()[n] = r_uint(
+            (word >> 24) |
+            ((word >> 8) & 0x0000ff00) |
+            ((word << 8) & 0x00ff0000) |
+            (word << 24)
+        )
+
+BITS = r_uint(32)
+class W_MappingDisplayBitmap(W_DisplayBitmap):
+    
+    repr_classname = "W_MappingDisplayBitmap"
+    _attrs_ = ['words_per_line', 'bits_in_last_word', 'pitch']
+    _immutable_fields_ = ['words_per_line?', 'bits_in_last_word?', 'pitch?']
+    
+    def __init__(self, space, w_class, size, depth):
+        assert depth in [1, 2, 4]
+        W_DisplayBitmap.__init__(self, space, w_class, size, depth)
+    
+    def take_over_display(self):
+        pitch = r_uint(self.display.pitch) # The pitch is different from the width input to SDL!
+        self.pitch = pitch
+        self.bits_in_last_word = pitch % BITS
+        self.words_per_line = r_uint((pitch - self.bits_in_last_word) / BITS)
+        if self.bits_in_last_word > 0:
+            self.words_per_line += 1
+        W_DisplayBitmap.take_over_display(self)
+    
+    @jit.unroll_safe
+    def set_pixelbuffer_word(self, n, word):
+        n = r_uint(n)
+        if ((n+1) % self.words_per_line) == 0 and self.bits_in_last_word > 0:
+            # This is the last word on the line. A few bits are cut off.
+            bits = self.bits_in_last_word
+        else:
+            bits = BITS
+        
+        word = r_uint(word)
+        pos = self.compute_pos(n)
+        buf = rffi.ptradd(self.display.screen.c_pixels, pos)
+        depth = r_uint(self._depth)
+        rshift = BITS - depth
+        for i in range(bits / depth):
+            pixel = word >> rshift
+            buf[i] = rffi.cast(rffi.UCHAR, pixel)
+            word <<= depth
+    
+    def compute_pos(self, n):
+        word_on_line = n % self.words_per_line
+        y = r_uint((n - word_on_line) / self.words_per_line)
+        x = word_on_line * BITS / r_uint(self._depth)
+        return y * self.pitch + x
diff --git a/spyvm/objspace.py b/spyvm/objspace.py
--- a/spyvm/objspace.py
+++ b/spyvm/objspace.py
@@ -1,30 +1,47 @@
 import os
 
-from spyvm import constants, model, shadow, wrapper, version
+from spyvm import constants, model, model_display, shadow, wrapper, version, display
 from spyvm.error import UnwrappingError, WrappingError, PrimitiveFailedError
 from rpython.rlib import jit, rpath
-from rpython.rlib.objectmodel import instantiate, specialize
+from rpython.rlib.objectmodel import instantiate, specialize, import_from_mixin
 from rpython.rlib.rarithmetic import intmask, r_uint, int_between
 
-class ConstantFlag(object):
-    """Boolean flag that can be edited, but will be promoted
+class ConstantMixin(object):
+    """Mixin for constant values that can be edited, but will be promoted
     to a constant when jitting."""
     
-    def __init__(self, set_initially=False):
-        self.flag = [set_initially]
+    def __init__(self, initial_value = None):
+        if initial_value is None:
+            initial_value = self.default_value
+        self.value = [initial_value]
     
+    def set(self, value):
+        self.value[0] = value
+    
+    def get(self):
+        value = jit.promote(self.value[0])
+        return value
+
+class ConstantFlag(object):
+    import_from_mixin(ConstantMixin)
+    default_value = False
     def is_set(self):
-        flag = jit.promote(self.flag[0])
-        return flag
+        return self.get()
+    def activate(self):
+        self.set(True)
+    def deactivate(self):
+        self.set(False)
+
+class ConstantString(object):
+    import_from_mixin(ConstantMixin)
+    default_value = ""
+    def get(self):
+        # Promoting does not work on strings...
+        return self.value[0]
     
-    def set(self):
-        self.flag[0] = True
-    
-    def unset(self):
-        self.flag[0] = False
-    
-    def set_to(self, flag):
-        self.flag[0] = flag
+class ConstantObject(object):
+    import_from_mixin(ConstantMixin)
+    default_value = None
 
 class ObjSpace(object):
     def __init__(self):
@@ -33,12 +50,14 @@
         self.no_specialized_storage = ConstantFlag()
         # This is a hack; see compile_code() in targetimageloadingsmalltalk.py
         self.suppress_process_switch = ConstantFlag()
+        self.run_spy_hacks = ConstantFlag()
         self.headless = ConstantFlag()
         
         self.classtable = {}
         self.objtable = {}
-        self._executable_path = [""] # XXX: we cannot set the attribute
-                                  # directly on the frozen objectspace
+        self._executable_path = ConstantString()
+        self._image_name = ConstantString()
+        self._display = ConstantObject()
         
         # Create the nil object.
         # Circumvent the constructor because nil is already referenced there.
@@ -61,11 +80,12 @@
                     break
         return rpath.rabspath(executable)
 
-    def runtime_setup(self, executable):
+    def runtime_setup(self, executable, image_name):
         fullpath = rpath.rabspath(self.find_executable(executable))
         i = fullpath.rfind(os.path.sep) + 1
         assert i > 0
-        self._executable_path[0] = fullpath[:i]
+        self._executable_path.set(fullpath[:i])
+        self._image_name.set(image_name)
 
     def populate_special_objects(self, specials):
         for name, idx in constants.objects_in_special_object_table.items():
@@ -74,9 +94,6 @@
                 self.objtable[name] = specials[idx]
         # XXX this is kind of hacky, but I don't know where else to get Metaclass
         self.classtable["w_Metaclass"] = self.w_SmallInteger.w_class.w_class
-        
-    def executable_path(self):
-        return self._executable_path[0]
     
     def add_bootstrap_class(self, name, cls):
         self.classtable[name] = cls
@@ -124,14 +141,8 @@
             name = "w_" + name
             if not name in self.objtable:
                 self.add_bootstrap_object(name, None)
-    
-    @specialize.arg(1)
-    def get_special_selector(self, selector):
-        i0 = constants.find_selectorindex(selector)
-        self.w_special_selectors.as_cached_object_get_shadow(self)
-        return self.w_special_selectors.fetch(self, i0)
 
-    # methods for wrapping and unwrapping stuff
+    # ============= Methods for wrapping and unwrapping stuff =============
 
     def wrap_int(self, val):
         from spyvm import constants
@@ -233,14 +244,30 @@
 
         return [w_array.at0(self, i) for i in range(w_array.size())]
 
-    def get_display(self):
-        w_display = self.objtable['w_display']
-        if w_display:
-            w_bitmap = w_display.fetch(self, 0)
-            if isinstance(w_bitmap, model.W_DisplayBitmap):
-                return w_bitmap.display
-        raise PrimitiveFailedError("No display")
-
+    # ============= Access to static information =============
+    
+    @specialize.arg(1)
+    def get_special_selector(self, selector):
+        i0 = constants.find_selectorindex(selector)
+        self.w_special_selectors.as_cached_object_get_shadow(self)
+        return self.w_special_selectors.fetch(self, i0)
+    
+    def executable_path(self):
+        return self._executable_path.get()
+    
+    def image_name(self):
+        return self._image_name.get()
+    
+    def display(self):
+        disp = self._display.get()
+        if disp is None:
+            # Create lazy to allow headless execution.
+            disp = display.SDLDisplay(self.image_name())
+            self._display.set(disp)
+        return disp
+    
+    # ============= Other Methods =============
+    
     def _freeze_(self):
         return True
 
diff --git a/spyvm/plugins/bitblt.py b/spyvm/plugins/bitblt.py
--- a/spyvm/plugins/bitblt.py
+++ b/spyvm/plugins/bitblt.py
@@ -1,4 +1,4 @@
-from spyvm import model
+from spyvm import model_display, model
 from spyvm.error import PrimitiveFailedError
 from spyvm.shadow import AbstractCachingShadow
 from spyvm.plugins.plugin import Plugin
@@ -32,7 +32,7 @@
         s_frame.push(interp.space.wrap_int(s_bitblt.bitCount))
     elif w_dest_form.is_same_object(space.objtable['w_display']):
         w_bitmap = w_dest_form.fetch(space, 0)
-        assert isinstance(w_bitmap, model.W_DisplayBitmap)
+        assert isinstance(w_bitmap, model_display.W_DisplayBitmap)
         w_bitmap.flush_to_screen()
     return w_rcvr
 
@@ -741,7 +741,7 @@
         self.w_bits = self.fetch(0)
         if self.w_bits.is_nil(self.space):
             return
-        if not (isinstance(self.w_bits, model.W_WordsObject) or isinstance(self.w_bits, model.W_DisplayBitmap)):
+        if not (isinstance(self.w_bits, model.W_WordsObject) or isinstance(self.w_bits, model_display.W_DisplayBitmap)):
             return
         self.width = self.intOrIfNil(self.fetch(1), 0)
         self.height = self.intOrIfNil(self.fetch(2), 0)
diff --git a/spyvm/plugins/vmdebugging.py b/spyvm/plugins/vmdebugging.py
--- a/spyvm/plugins/vmdebugging.py
+++ b/spyvm/plugins/vmdebugging.py
@@ -20,12 +20,12 @@
 
 @DebuggingPlugin.expose_primitive(unwrap_spec=[object])
 def trace_proxy(interp, s_frame, w_rcvr):
-    interp.trace_proxy.set()
+    interp.trace_proxy.activate()
     return w_rcvr
 
 @DebuggingPlugin.expose_primitive(unwrap_spec=[object])
 def untrace_proxy(interp, s_frame, w_rcvr):
-    interp.trace_proxy.unset()
+    interp.trace_proxy.deactivate()
     return w_rcvr
 
 @DebuggingPlugin.expose_primitive(unwrap_spec=[object])
diff --git a/spyvm/primitives.py b/spyvm/primitives.py
--- a/spyvm/primitives.py
+++ b/spyvm/primitives.py
@@ -2,7 +2,7 @@
 import inspect
 import math
 import operator
-from spyvm import model, shadow, error, constants, display
+from spyvm import model, model_display, shadow, error, constants, display
 from spyvm.error import PrimitiveFailedError, PrimitiveNotYetWrittenError
 from spyvm import wrapper
 
@@ -644,7 +644,7 @@
 
 @expose_primitive(MOUSE_POINT, unwrap_spec=[object])
 def func(interp, s_frame, w_rcvr):
-    x, y = interp.space.get_display().mouse_point()
+    x, y = interp.space.display().mouse_point()
     w_point = model.W_PointersObject(interp.space, interp.space.w_Point, 2)
     w_point.store(interp.space, 0, interp.space.wrap_int(x))
     w_point.store(interp.space, 1, interp.space.wrap_int(y))
@@ -656,7 +656,7 @@
 def func(interp, s_frame, w_rcvr, w_into):
     if not interp.evented:
         raise PrimitiveFailedError()
-    ary = interp.space.get_display().get_next_event(time=interp.time_now())
+    ary = interp.space.display().get_next_event(time=interp.time_now())
     for i in range(8):
         w_into.store(interp.space, i, interp.space.wrap_int(ary[i]))
     # XXX - hack
@@ -676,7 +676,7 @@
         w_display = interp.space.objtable['w_display']
         if w_dest_form.is_same_object(w_display):
             w_bitmap = w_display.fetch(interp.space, 0)
-            assert isinstance(w_bitmap, model.W_DisplayBitmap)
+            assert isinstance(w_bitmap, model_display.W_DisplayBitmap)
             w_bitmap.flush_to_screen()
         return w_rcvr
     except shadow.MethodNotFound:
@@ -731,45 +731,23 @@
 def func(interp, s_frame, w_rcvr):
     if interp.space.headless.is_set():
         exitFromHeadlessExecution(s_frame)
-    
     if not isinstance(w_rcvr, model.W_PointersObject) or w_rcvr.size() < 4:
         raise PrimitiveFailedError
-    # the fields required are bits (a pointer to a Bitmap), width, height, depth
-
-    # XXX: TODO get the initial image TODO: figure out whether we
-    # should decide the width an report it in the other SCREEN_SIZE
-    w_bitmap = w_rcvr.fetch(interp.space, 0)
-    width = interp.space.unwrap_int(w_rcvr.fetch(interp.space, 1))
-    height = interp.space.unwrap_int(w_rcvr.fetch(interp.space, 2))
-    depth = interp.space.unwrap_int(w_rcvr.fetch(interp.space, 3))
-
-    sdldisplay = None
-
-    w_prev_display = interp.space.objtable['w_display']
-    if w_prev_display:
-        w_prev_bitmap = w_prev_display.fetch(interp.space, 0)
-        if isinstance(w_prev_bitmap, model.W_DisplayBitmap):
-            sdldisplay = w_prev_bitmap.display
-            sdldisplay.set_video_mode(width, height, depth)
-
-    if isinstance(w_bitmap, model.W_DisplayBitmap):
-        assert (sdldisplay is None) or (sdldisplay is w_bitmap.display)
-        sdldisplay = w_bitmap.display
-        sdldisplay.set_video_mode(width, height, depth)
-        w_display_bitmap = w_bitmap
-    else:
-        assert isinstance(w_bitmap, model.W_WordsObject)
-        w_display_bitmap = w_bitmap.as_display_bitmap(
-            w_rcvr,
-            interp,
-            sdldisplay=sdldisplay
-        )
-
+    
+    old_display = interp.space.objtable['w_display']
+    if isinstance(old_display, model_display.W_DisplayBitmap):
+        old_display.relinquish_display()
+    interp.space.objtable['w_display'] = w_rcvr
+    
+    # TODO: figure out whether we should decide the width an report it in the SCREEN_SIZE primitive
+    form = wrapper.FormWrapper(interp.space, w_rcvr)
+    form.take_over_display()
+    w_display_bitmap = form.get_display_bitmap()
+    w_display_bitmap.take_over_display()
     w_display_bitmap.flush_to_screen()
+    
     if interp.image:
-        interp.image.lastWindowSize = (width << 16)  + height
-    interp.space.objtable['w_display'] = w_rcvr
-
+        interp.image.lastWindowSize = (form.width() << 16) + form.height()
     return w_rcvr
 
 @expose_primitive(STRING_REPLACE, unwrap_spec=[object, index1_0, index1_0, object, index1_0])
@@ -812,12 +790,12 @@
 
 @expose_primitive(MOUSE_BUTTONS, unwrap_spec=[object])
 def func(interp, s_frame, w_rcvr):
-    btn = interp.space.get_display().mouse_button()
+    btn = interp.space.display().mouse_button()
     return interp.space.wrap_int(btn)
 
 @expose_primitive(KBD_NEXT, unwrap_spec=[object])
 def func(interp, s_frame, w_rcvr):
-    code = interp.space.get_display().next_keycode()
+    code = interp.space.display().next_keycode()
     if code & 0xFF == 0:
         return interp.space.w_nil
     else:
@@ -825,7 +803,7 @@
 
 @expose_primitive(KBD_PEEK, unwrap_spec=[object])
 def func(interp, s_frame, w_rcvr):
-    code = interp.space.get_display().peek_keycode()
+    code = interp.space.display().peek_keycode()
     if code & 0xFF == 0:
         return interp.space.w_nil
     else:
@@ -864,8 +842,8 @@
 
 @expose_primitive(EXIT_TO_DEBUGGER, unwrap_spec=[object])
 def func(interp, s_frame, w_rcvr):
-    if not objectmodel.we_are_translated():
-        import pdb; pdb.set_trace()
+    if interp.space.headless.is_set():
+        exitFromHeadlessExecution(s_frame, "EXIT_TO_DEBUGGER")
     raise PrimitiveNotYetWrittenError()
 
 @expose_primitive(CHANGE_CLASS, unwrap_spec=[object, object], no_result=True)
@@ -987,7 +965,7 @@
 def func(interp, s_frame, argument_count):
     if argument_count == 0:
         s_frame.pop()
-        return interp.space.wrap_string(interp.image_name)
+        return interp.space.wrap_string(interp.space.image_name())
     elif argument_count == 1:
         pass # XXX
     raise PrimitiveFailedError
@@ -1004,7 +982,7 @@
 
 @expose_primitive(DEFER_UPDATES, unwrap_spec=[object, bool])
 def func(interp, s_frame, w_receiver, flag):
-    sdldisplay = interp.space.get_display()
+    sdldisplay = interp.space.display()
     sdldisplay.defer_updates(flag)
     return w_receiver
 
@@ -1058,7 +1036,7 @@
 
 @expose_primitive(SET_INTERRUPT_KEY, unwrap_spec=[object, int])
 def func(interp, s_frame, w_rcvr, encoded_key):
-    interp.space.get_display().set_interrupt_key(interp.space, encoded_key)
+    interp.space.display().set_interrupt_key(interp.space, encoded_key)
     return w_rcvr
 
 @expose_primitive(INTERRUPT_SEMAPHORE, unwrap_spec=[object, object])
@@ -1142,7 +1120,7 @@
             raise PrimitiveFailedError
         for i in xrange(w_arg.size()):
             w_arg.setchar(i, chr(new_value))
-    elif isinstance(w_arg, model.W_WordsObject) or isinstance(w_arg, model.W_DisplayBitmap):
+    elif isinstance(w_arg, model.W_WordsObject) or isinstance(w_arg, model_display.W_DisplayBitmap):
         for i in xrange(w_arg.size()):
             w_arg.setword(i, new_value)
     else:
@@ -1531,7 +1509,7 @@
 
 @expose_primitive(FORCE_DISPLAY_UPDATE, unwrap_spec=[object])
 def func(interp, s_frame, w_rcvr):
-    interp.space.get_display().flip(force=True)
+    interp.space.display().flip(force=True)
     return w_rcvr
 
 # ___________________________________________________________________________
diff --git a/spyvm/squeakimage.py b/spyvm/squeakimage.py
--- a/spyvm/squeakimage.py
+++ b/spyvm/squeakimage.py
@@ -386,12 +386,13 @@
         self.startup_time = time.time()
 
     def run_spy_hacks(self, space):
-        pass
-        # w_display = space.objtable["w_display"]
-        # if w_display is not None and not w_display.is_nil(space):
-        #     if space.unwrap_int(w_display.fetch(space, 3)) < 8:
-        #         # non-native indexed color depth not well supported
-        #         w_display.store(space, 3, space.wrap_int(8))
+        if not space.run_spy_hacks.is_set():
+            return
+        w_display = space.objtable["w_display"]
+        if w_display is not None and not w_display.is_nil(space):
+            if space.unwrap_int(w_display.fetch(space, 3)) < 8:
+                # non-native indexed color depth not well supported
+                w_display.store(space, 3, space.wrap_int(8))
 
     def find_symbol(self, space, reader, symbol):
         w_dnu = self.special(constants.SO_DOES_NOT_UNDERSTAND)
diff --git a/spyvm/test/test_model.py b/spyvm/test/test_model.py
--- a/spyvm/test/test_model.py
+++ b/spyvm/test/test_model.py
@@ -1,5 +1,5 @@
 import py, math, socket
-from spyvm import model, shadow, objspace, error, display
+from spyvm import model, model_display, shadow, objspace, error, display
 from spyvm.shadow import MethodNotFound, WEAK_POINTERS
 from rpython.rlib.rarithmetic import intmask, r_uint
 from rpython.rtyper.lltypesystem import lltype, rffi
@@ -387,7 +387,7 @@
     d = display.SDLDisplay("test")
     d.set_video_mode(32, 10, 1)
 
-    target = model.W_DisplayBitmap.create(space, space.w_Array, 10, 1, d)
+    target = model_display.W_DisplayBitmap.create(space, space.w_Array, 10, 1, d)
     target.setword(0, r_uint(0xFF00))
     assert bin(target.getword(0)) == bin(0xFF00)
     target.setword(0, r_uint(0x00FF00FF))
@@ -404,15 +404,7 @@
         assert target.pixelbuffer[i] == 0x0
 
 def test_display_offset_computation():
-
-    def get_pixelbuffer(self):
-        return lltype.malloc(rffi.ULONGP.TO, self.width * self.height * 32, flavor='raw')
-    display.SDLDisplay.get_pixelbuffer = get_pixelbuffer
-    d = display.SDLDisplay("test")
-    d.set_video_mode(18, 5, 1)
-
-    dbitmap = model.W_DisplayBitmap.create(space, space.w_Array, 5, 1, d)
-
+    dbitmap = model_display.W_MappingDisplayBitmap(space, space.w_Array, 5, 1)
     assert dbitmap.compute_pos(0) == 0
     assert dbitmap.compute_pos(1) == 8
     assert dbitmap.size() == 5 * 8
diff --git a/spyvm/test/test_primitives.py b/spyvm/test/test_primitives.py
--- a/spyvm/test/test_primitives.py
+++ b/spyvm/test/test_primitives.py
@@ -1,5 +1,5 @@
 import py, os, math, time
-from spyvm import model, shadow, interpreter, constants, primitives, objspace, wrapper, display
+from spyvm import model, model_display, shadow, interpreter, constants, primitives, objspace, wrapper, display
 from spyvm.primitives import prim_table, PrimitiveFailedError
 from spyvm.plugins import bitblt
 from rpython.rlib.rfloat import INFINITY, NAN, isinf, isnan
@@ -47,7 +47,8 @@
         frame = context
         for i in range(len(stack)):
             frame.as_context_get_shadow(space).push(stack[i])
-    interp = TestInterpreter(space, image_name=IMAGENAME)
+    interp = TestInterpreter(space)
+    interp.space._image_name.set(IMAGENAME)
     return interp, frame, len(stack)
 
 def _prim(space, code, stack, context = None):
@@ -680,7 +681,7 @@
 
     closure = space.newClosure(w_frame, 4, 0, [])
     s_frame = w_frame.as_methodcontext_get_shadow(space)
-    interp = TestInterpreter(space, image_name=IMAGENAME)
+    interp = TestInterpreter(space)
     interp._loop = True
 
     try:
@@ -721,7 +722,7 @@
     assert space.objtable["w_display"] is mock_display
     w_bitmap = mock_display.fetch(space, 0)
     assert w_bitmap is not w_wordbmp
-    assert isinstance(w_bitmap, model.W_DisplayBitmap)
+    assert isinstance(w_bitmap, model_display.W_DisplayBitmap)
     sdldisplay = w_bitmap.display
     assert isinstance(sdldisplay, display.SDLDisplay)
 
@@ -733,7 +734,7 @@
     prim(primitives.BE_DISPLAY, [mock_display2])
     assert space.objtable["w_display"] is mock_display2
     w_bitmap2 = mock_display.fetch(space, 0)
-    assert isinstance(w_bitmap2, model.W_DisplayBitmap)
+    assert isinstance(w_bitmap2, model_display.W_DisplayBitmap)
     assert w_bitmap.display is w_bitmap2.display
     assert sdldisplay.width == 32
     assert sdldisplay.height == 10
@@ -764,7 +765,7 @@
         raise DisplayFlush
 
     try:
-        monkeypatch.setattr(space.get_display().__class__, "flip", flush_to_screen_mock)
+        monkeypatch.setattr(space.display().__class__, "flip", flush_to_screen_mock)
         with py.test.raises(DisplayFlush):
             prim(primitives.FORCE_DISPLAY_UPDATE, [mock_display])
     finally:
diff --git a/spyvm/wrapper.py b/spyvm/wrapper.py
--- a/spyvm/wrapper.py
+++ b/spyvm/wrapper.py
@@ -1,4 +1,4 @@
-from spyvm import model, constants
+from spyvm import model, model_display, constants
 from spyvm.error import FatalError, WrapperException, PrimitiveFailedError
 
 class Wrapper(object):
@@ -263,6 +263,30 @@
     def size(self):
         return self._w_self.size() - constants.BLKCLSR_SIZE
 
+class FormWrapper(Wrapper):
+    bits, store_bits = make_getter_setter(constants.FORM_BITS)
+    width, store_width = make_int_getter_setter(constants.FORM_WIDTH)
+    height, store_height = make_int_getter_setter(constants.FORM_HEIGHT)
+    depth, store_depth = make_int_getter_setter(constants.FORM_DEPTH)
+    
+    def create_display_bitmap(self):
+        w_display_bitmap = model_display.from_words_object(self.bits(), self)
+        self.store_bits(w_display_bitmap)
+        return w_display_bitmap
+    
+    def get_display_bitmap(self):
+        w_bitmap = self.bits()
+        if not isinstance(w_bitmap, model_display.W_DisplayBitmap):
+            w_display_bitmap = self.create_display_bitmap()
+        else:
+            w_display_bitmap = w_bitmap
+            if w_display_bitmap._depth != self.depth():
+                w_display_bitmap = self.create_display_bitmap()
+        return w_display_bitmap
+    
+    def take_over_display(self):
+        self.space.display().set_video_mode(self.width(), self.height(), self.depth())
+    
 # XXX Wrappers below are not used yet.
 class OffsetWrapper(Wrapper):
     offset_x  = make_int_getter(0)
diff --git a/targetimageloadingsmalltalk.py b/targetimageloadingsmalltalk.py
--- a/targetimageloadingsmalltalk.py
+++ b/targetimageloadingsmalltalk.py
@@ -36,6 +36,7 @@
             -p|--poll          - Actively poll for events. Try this if the image is not responding well.
             -i|--no-interrupts - Disable timer interrupt. Disables non-cooperative scheduling.
             -S                 - Disable specialized storage strategies; always use generic ListStorage
+            --hacks            - Enable Spy hacks. Set display color depth to 8.
             
           Logging parameters:
             -t|--trace                 - Output a trace of each message, primitive, return value and process switch.
@@ -69,7 +70,7 @@
     except error.Exit, e:
         print_error("Exited: %s" % e.msg)
         return -1
-    except Exception, e:
+    except BaseException, e:
         print_error("Exception: %s" % str(e))
         if not objectmodel.we_are_translated():
             import traceback
@@ -119,8 +120,10 @@
                 interrupts = False
             elif arg in ["-P", "--process"]:
                 headless = False
+            elif arg in ["--hacks"]:
+                space.run_spy_hacks.activate()
             elif arg in ["-S"]:
-                space.no_specialized_storage.set()
+                space.no_specialized_storage.activate()
             elif arg in ["-u"]:
                 from spyvm.plugins.vmdebugging import stop_ui_process
                 stop_ui_process()
@@ -158,10 +161,10 @@
     # Load & prepare image and environment
     image_reader = squeakimage.reader_for_image(space, squeakimage.Stream(data=imagedata))
     image = create_image(space, image_reader)
-    interp = interpreter.Interpreter(space, image, image_name=path,
+    interp = interpreter.Interpreter(space, image,
                 trace=trace, evented=not poll,
                 interrupts=interrupts)
-    space.runtime_setup(argv[0])
+    space.runtime_setup(argv[0], path)
     print_error("") # Line break after image-loading characters
     
     # Create context to be executed
@@ -174,7 +177,7 @@
             selector = compile_code(interp, w_receiver, code)
         s_frame = create_context(interp, w_receiver, selector, stringarg)
         if headless:
-            space.headless.set()
+            space.headless.activate()
             context = s_frame
         else:
             create_process(interp, s_frame)
@@ -203,7 +206,7 @@
     # registered (primitive 136 not called), so the idle process will never be left once it is entered.
     # TODO - Find a way to cleanly initialize the image, without executing the active_context of the image.
     # Instead, we want to execute our own context. Then remove this flag (and all references to it)
-    space.suppress_process_switch.set()
+    space.suppress_process_switch.activate()
     
     w_result = interp.perform(
         w_receiver_class,
@@ -215,7 +218,7 @@
     # TODO - is this expected in every image?
     if not isinstance(w_result, model.W_BytesObject) or w_result.as_string() != selector:
         raise error.Exit("Unexpected compilation result (probably failed to compile): %s" % result_string(w_result))
-    space.suppress_process_switch.unset()
+    space.suppress_process_switch.deactivate()
     
     w_receiver_class.as_class_get_shadow(space).s_methoddict().sync_method_cache()
     return selector

From noreply at buildbot.pypy.org  Wed Jul 23 07:31:52 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 23 Jul 2014 07:31:52 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: in-progress
Message-ID: <20140723053152.1B8401D3562@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: extradoc
Changeset: r5371:59cb8a939bcc
Date: 2014-07-23 07:31 +0200
http://bitbucket.org/pypy/extradoc/changeset/59cb8a939bcc/

Log:	in-progress

diff --git a/talk/ep2014/stm/talk.rst b/talk/ep2014/stm/talk.rst
--- a/talk/ep2014/stm/talk.rst
+++ b/talk/ep2014/stm/talk.rst
@@ -138,6 +138,33 @@
   - but mostly still research-only
 
 
+PyPy-STM
+--------
+
+* implementation of a specially-tailored STM ("hard" part):
+    
+  - a reusable C library
+  - called STMGC-C7
+
+* used in PyPy to replace the GIL ("easy" part)
+
+* could also be used in CPython
+
+  - but refcounting needs replacing
+
+
+Commits
+---------
+
+.. image:: fig4.svg
+
+
+Demo
+------
+
+* counting primes
+
+
 Big Point
 ----------------------------
 
@@ -150,33 +177,31 @@
   - even if they both *acquire and release the same lock*
 
 
-Big Point
----------
+Long Transactions
+-----------------
 
 .. image:: fig4.svg
 
 
-Demo 1
+Demo
 ------
 
-* "Twisted apps made parallel out of the box"
-
 * Bottle web server
 
 
-PyPy-STM
---------
+PyPy-STM Programming Model
+---------------------------
 
-* implementation of a specially-tailored STM:
-    
-  - a reusable C library
-  - called STMGC-C7
+* threads-and-locks, fully compatible with the GIL
 
-* used in PyPy to replace the GIL
+* this is not "everybody should use careful explicit threading
+  with all the locking issues"
 
-* could also be used in CPython
+* instead, PyPy-STM pushes forward:
 
-  - but refcounting needs replacing
+  - use a thread pool library
+
+  - coarse locking, inside that library only
 
 
 PyPy-STM status
@@ -186,7 +211,7 @@
 
   - basics work
   - best case 25-40% overhead (much better than originally planned)
-  - parallelizing user locks not done yet
+  - parallelizing user locks not done yet (see ``with atomic``)
   - tons of things to improve
   - tons of things to improve
   - tons of things to improve
@@ -196,20 +221,14 @@
   - tons of things to improve
 
 
-Demo 2
-------
-
-* counting primes
-
-
-Benefits
---------
+Summary: Benefits
+-----------------
 
 * Keep locks coarse-grained
 
 * Potential to enable parallelism:
 
-  - in CPU-bound multithreaded programs
+  - in any CPU-bound multithreaded program
 
   - or as a replacement of ``multiprocessing``
 
@@ -218,10 +237,8 @@
   - as long as they do multiple things that are "often independent"
 
 
-Issues
-------
-
-* Performance hit: 25-40% everywhere (may be ok)
+Summary: Issues
+---------------
 
 * Keep locks coarse-grained:
 
@@ -229,13 +246,15 @@
 
   - need to track and fix them
 
-  - need tool support (debugger/profiler)
+  - need tool to support this (debugger/profiler)
 
+* Performance hit: 25-40% everywhere (may be ok)
 
-Summary
--------
 
-* Transactional Memory is still too researchy for production
+Summary: PyPy-STM
+-----------------
+
+* Not production-ready
 
 * But it has the potential to enable "easier parallelism"
 
@@ -248,3 +267,39 @@
 -----------------------
 
 **STMGC-C7**
+
+
+Overview
+--------
+
+* Say we want to run two threads
+
+* We reserve twice the memory
+
+* Thread 1 reads/writes "memory segment" 1
+
+* Thread 2 reads/writes "memory segment" 2
+
+* Upon commit, we (try to) copy the changes to the other segment
+
+
+Trick #1
+--------
+
+* Objects contain pointers to each other
+
+* These pointers are relative instead of absolute:
+
+  - 
+
+
+Trick #1
+--------
+
+* Most objects are the same in all segments:
+
+  - so we share the memory
+  
+  - ``mmap() MAP_SHARED`` trickery
+
+

From noreply at buildbot.pypy.org  Wed Jul 23 08:02:34 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 23 Jul 2014 08:02:34 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: Finish the slides
Message-ID: <20140723060234.C5F031D3633@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: extradoc
Changeset: r5372:3c5ba3e46ec5
Date: 2014-07-23 08:02 +0200
http://bitbucket.org/pypy/extradoc/changeset/3c5ba3e46ec5/

Log:	Finish the slides

diff --git a/talk/ep2014/stm/talk.html b/talk/ep2014/stm/talk.html
--- a/talk/ep2014/stm/talk.html
+++ b/talk/ep2014/stm/talk.html
@@ -502,40 +502,64 @@
 
 
 
-
-Big Point
+
+PyPy-STM
 
-application-level locks still needed...
+implementation of a specially-tailored STM ("hard" part):
+a reusable C library
+called STMGC-C7
+
+
+used in PyPy to replace the GIL ("easy" part)
+could also be used in CPython
+but refcounting needs replacing
+
+
+
+
+
+How does it work?
+
+
+
+Demo
+
+counting primes
+
+
+
+Long Transactions
+
+threads and application-level locks still needed...
 but can be very coarse:
-even two big transactions can optimistically run in parallel
+two transactions can optimistically run in parallel
 even if they both acquire and release the same lock
+internally, drive the transaction lengths by the locks we acquire
 
 
 
 
 
-Big Point
+Long Transactions
 
 
-
-Demo 1
+
+Demo
 
-"Twisted apps made parallel out of the box"
 Bottle web server
 
 
-
-PyPy-STM
+
+PyPy-STM Programming Model
 
-implementation of a specially-tailored STM:
-a reusable C library
-called STMGC-C7
-
-
-used in PyPy to replace the GIL
-could also be used in CPython
-but refcounting needs replacing
+threads-and-locks, fully compatible with the GIL
+this is not "everybody should use careful explicit threading
+with all the locking issues"
+instead, PyPy-STM pushes forward:
+use a thread pool library
+coarse locking, inside that library only
 
 
 
@@ -546,7 +570,7 @@
 
current status:
 basics work
 best case 25-40% overhead (much better than originally planned)
-parallelizing user locks not done yet
+parallelizing user locks not done yet (see "with atomic")
 tons of things to improve
 tons of things to improve
 tons of things to improve
@@ -558,52 +582,113 @@
 
 
 
-
-Demo 2
+
+Summary: Benefits
 
-counting primes
-
-
-
-Benefits
-
-Keep locks coarse-grained
 Potential to enable parallelism:
-in CPU-bound multithreaded programs
+in any CPU-bound multithreaded program
 or as a replacement of multiprocessing
 but also in existing applications not written for that
 as long as they do multiple things that are "often independent"
 
 
+Keep locks coarse-grained
 
 
-
-Issues
+
+Summary: Issues
 
-Performance hit: 25-40% everywhere (may be ok)
 Keep locks coarse-grained:
 but in case of systematic conflicts, performance is bad again
 need to track and fix them
-need tool support (debugger/profiler)
+need tool to support this (debugger/profiler)
 
 
+Performance hit: 25-40% over a plain PyPy-JIT (may be ok)
 
 
-
-Summary
+
+Summary: PyPy-STM
 
-Transactional Memory is still too researchy for production
-But it has the potential to enable "easier parallelism"
+Not production-ready
+But it has the potential to enable "easier parallelism for everybody"
 Still alpha but slowly getting there!
 see http://morepypy.blogspot.com/
 
 
+Crowdfunding!
+see http://pypy.org/
+
+
 
 
 
 Part 2 - Under The Hood
 STMGC-C7
 
+
+Overview
+
+Say we want to run N = 2 threads
+We reserve twice the memory
+Thread 1 reads/writes "memory segment" 1
+Thread 2 reads/writes "memory segment" 2
+Upon commit, we (try to) copy the changes to the other segment
+
+
+
+Trick #1
+
+Objects contain pointers to each other
+These pointers are relative instead of absolute:
+accessed as if they were "thread-local data"
+the x86 has a zero-cost way to do that (%fs, %gs)
+supported in clang (not gcc so far)
+
+
+
+
+
+Trick #2
+
+With Trick #1, most objects are exactly identical in all N segments:
+so we share the memory
+mmap() MAP_SHARED
+actual memory usage is multiplied by much less than N
+
+
+Newly allocated objects are directly in shared pages:
+we don't actually need to copy all new objects at commit,
+but only the few old objects modified
+
+
+
+
+
+Barriers
+
+Need to record all reads and writes done by a transaction
+Extremely cheap way to do that:
+Read: set a flag in thread-local memory (one byte)
+Write into a newly allocated object: nothing to do
+Write into an old object: add the object to a list
+
+
+Commit: check if each object from that list conflicts with
+a read flag set in some other thread
+
+
+
+...
+
+
+Thank You
+
+http://morepypy.blogspot.com/
+http://pypy.org/
+irc: #pypy on freenode.net
+
+
 
 
 
diff --git a/talk/ep2014/stm/talk.rst b/talk/ep2014/stm/talk.rst
--- a/talk/ep2014/stm/talk.rst
+++ b/talk/ep2014/stm/talk.rst
@@ -153,8 +153,8 @@
   - but refcounting needs replacing
 
 
-Commits
----------
+How does it work?
+-----------------
 
 .. image:: fig4.svg
 
@@ -165,17 +165,19 @@
 * counting primes
 
 
-Big Point
+Long Transactions
 ----------------------------
 
-* application-level locks still needed...
+* threads and application-level locks still needed...
 
 * but *can be very coarse:*
 
-  - even two big transactions can optimistically run in parallel
+  - two transactions can optimistically run in parallel
 
   - even if they both *acquire and release the same lock*
 
+  - internally, drive the transaction lengths by the locks we acquire
+
 
 Long Transactions
 -----------------
@@ -211,7 +213,7 @@
 
   - basics work
   - best case 25-40% overhead (much better than originally planned)
-  - parallelizing user locks not done yet (see ``with atomic``)
+  - parallelizing user locks not done yet (see "with atomic")
   - tons of things to improve
   - tons of things to improve
   - tons of things to improve
@@ -224,8 +226,6 @@
 Summary: Benefits
 -----------------
 
-* Keep locks coarse-grained
-
 * Potential to enable parallelism:
 
   - in any CPU-bound multithreaded program
@@ -236,6 +236,8 @@
 
   - as long as they do multiple things that are "often independent"
 
+* Keep locks coarse-grained
+
 
 Summary: Issues
 ---------------
@@ -248,7 +250,7 @@
 
   - need tool to support this (debugger/profiler)
 
-* Performance hit: 25-40% everywhere (may be ok)
+* Performance hit: 25-40% over a plain PyPy-JIT (may be ok)
 
 
 Summary: PyPy-STM
@@ -256,12 +258,16 @@
 
 * Not production-ready
 
-* But it has the potential to enable "easier parallelism"
+* But it has the potential to enable "easier parallelism for everybody"
 
 * Still alpha but slowly getting there!
 
   - see http://morepypy.blogspot.com/
 
+* Crowdfunding!
+
+  - see http://pypy.org/
+
 
 Part 2 - Under The Hood
 -----------------------
@@ -272,7 +278,7 @@
 Overview
 --------
 
-* Say we want to run two threads
+* Say we want to run N = 2 threads
 
 * We reserve twice the memory
 
@@ -290,16 +296,56 @@
 
 * These pointers are relative instead of absolute:
 
-  - 
+  - accessed as if they were "thread-local data"
 
+  - the x86 has a zero-cost way to do that (``%fs``, ``%gs``)
 
-Trick #1
+  - supported in clang (not gcc so far)
+
+
+Trick #2
 --------
 
-* Most objects are the same in all segments:
+* With Trick #1, most objects are exactly identical in all N segments:
 
   - so we share the memory
   
-  - ``mmap() MAP_SHARED`` trickery
+  - ``mmap() MAP_SHARED``
 
+  - actual memory usage is multiplied by much less than N
 
+* Newly allocated objects are directly in shared pages:
+    
+  - we don't actually need to copy *all new objects* at commit,
+    but only the few *old objects* modified
+
+
+Barriers
+--------
+
+* Need to record all reads and writes done by a transaction
+
+* Extremely cheap way to do that:
+
+  - *Read:* set a flag in thread-local memory (one byte)
+
+  - *Write* into a newly allocated object: nothing to do
+
+  - *Write* into an old object: add the object to a list
+
+* Commit: check if each object from that list conflicts with
+  a read flag set in some other thread
+
+
+...
+-------------------
+
+
+Thank You
+---------
+
+* http://morepypy.blogspot.com/
+
+* http://pypy.org/
+
+* irc: ``#pypy`` on freenode.net

From noreply at buildbot.pypy.org  Wed Jul 23 10:51:58 2014
From: noreply at buildbot.pypy.org (arigo)
Date: Wed, 23 Jul 2014 10:51:58 +0200 (CEST)
Subject: [pypy-commit] extradoc extradoc: tweaks
Message-ID: <20140723085158.172551C0325@cobra.cs.uni-duesseldorf.de>

Author: Armin Rigo 
Branch: extradoc
Changeset: r5373:8fe97fa6f212
Date: 2014-07-23 10:51 +0200
http://bitbucket.org/pypy/extradoc/changeset/8fe97fa6f212/

Log:	tweaks

diff --git a/talk/ep2014/stm/demo/bench-multiprocessing.py b/talk/ep2014/stm/demo/bench-multiprocessing.py
--- a/talk/ep2014/stm/demo/bench-multiprocessing.py
+++ b/talk/ep2014/stm/demo/bench-multiprocessing.py
@@ -8,7 +8,7 @@
             subtotal += 1
     return subtotal
 
-pool = Pool(4)
+pool = Pool(2)
 results = pool.map(process, xrange(0, 5000000, 20000))
 total = sum(results)
 
diff --git a/talk/ep2014/stm/talk.html b/talk/ep2014/stm/talk.html
--- a/talk/ep2014/stm/talk.html
+++ b/talk/ep2014/stm/talk.html
@@ -535,7 +535,6 @@
 but can be very coarse:
 two transactions can optimistically run in parallel
 even if they both acquire and release the same lock
-internally, drive the transaction lengths by the locks we acquire


@@ -558,19 +557,30 @@
 this is not "everybody should use careful explicit threading
 with all the locking issues"
 instead, PyPy-STM pushes forward:
-use a thread pool library
+make or use a thread pool library
 coarse locking, inside that library only

...

diff --git a/talk/ep2014/stm/talk.rst b/talk/ep2014/stm/talk.rst --- a/talk/ep2014/stm/talk.rst +++ b/talk/ep2014/stm/talk.rst @@ -176,8 +176,6 @@ - even if they both *acquire and release the same lock* - - internally, drive the transaction lengths by the locks we acquire - Long Transactions ----------------- @@ -201,11 +199,23 @@ * instead, PyPy-STM pushes forward: - - use a thread pool library + - make or use a thread pool library - coarse locking, inside that library only +PyPy-STM Programming Model +-------------------------- + +* e.g.: + + - ``multiprocessing``-like thread pool + + - Twisted/Tornado/Bottle extension + + - Stackless/greenlet/gevent extension + + PyPy-STM status --------------- @@ -213,7 +223,7 @@ - basics work - best case 25-40% overhead (much better than originally planned) - - parallelizing user locks not done yet (see "with atomic") + - app locks not done yet ("with atomic" workaround) - tons of things to improve - tons of things to improve - tons of things to improve @@ -250,7 +260,7 @@ - need tool to support this (debugger/profiler) -* Performance hit: 25-40% over a plain PyPy-JIT (may be ok) +* Performance hit: 25-40% slower than a plain PyPy-JIT (may be ok) Summary: PyPy-STM From noreply at buildbot.pypy.org Thu Jul 24 10:48:11 2014 From: noreply at buildbot.pypy.org (stefanor) Date: Thu, 24 Jul 2014 10:48:11 +0200 (CEST) Subject: [pypy-commit] cffi default: ARM64 support. Tests pass under qemu. Message-ID: <20140724084811.E2EB11C022A@cobra.cs.uni-duesseldorf.de> Author: Stefano Rivera Branch: Changeset: r1550:af4e381b5e99 Date: 2014-07-24 10:47 +0200 http://bitbucket.org/cffi/cffi/changeset/af4e381b5e99/ Log: ARM64 support. Tests pass under qemu. Fixes #136. Thanks Will Newton . diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c --- a/c/_cffi_backend.c +++ b/c/_cffi_backend.c @@ -3622,7 +3622,7 @@ #ifdef MS_WIN32 sflags |= SF_MSVC_BITFIELDS; #else -# ifdef __arm__ +# if defined(__arm__) || defined(__aarch64__) sflags |= SF_GCC_ARM_BITFIELDS; # else sflags |= SF_GCC_X86_BITFIELDS; diff --git a/testing/test_ffi_backend.py b/testing/test_ffi_backend.py --- a/testing/test_ffi_backend.py +++ b/testing/test_ffi_backend.py @@ -122,7 +122,7 @@ self.check("int a:2; short b:15; char c:2; char y;", 5, 4, 8) self.check("int a:2; char b:1; char c:1; char y;", 1, 4, 4) - @pytest.mark.skipif("platform.machine().startswith('arm')") + @pytest.mark.skipif("platform.machine().startswith(('arm', 'aarch64'))") def test_bitfield_anonymous_no_align(self): L = FFI().alignof("long long") self.check("char y; int :1;", 0, 1, 2) @@ -135,7 +135,8 @@ self.check("char x; long long z:57; char y;", L + 8, L, L + 8 + L) self.check("char x; long long :57; char y;", L + 8, 1, L + 9) - @pytest.mark.skipif("not platform.machine().startswith('arm')") + @pytest.mark.skipif( + "not platform.machine().startswith(('arm', 'aarch64'))") def test_bitfield_anonymous_align_arm(self): L = FFI().alignof("long long") self.check("char y; int :1;", 0, 4, 4) @@ -148,7 +149,7 @@ self.check("char x; long long z:57; char y;", L + 8, L, L + 8 + L) self.check("char x; long long :57; char y;", L + 8, L, L + 8 + L) - @pytest.mark.skipif("platform.machine().startswith('arm')") + @pytest.mark.skipif("platform.machine().startswith(('arm', 'aarch64'))") def test_bitfield_zero(self): L = FFI().alignof("long long") self.check("char y; int :0;", 0, 1, 4) @@ -159,7 +160,8 @@ self.check("char x; int :0; short b:1; char y;", 5, 2, 6) self.check("int a:1; int :0; int b:1; char y;", 5, 4, 8) - @pytest.mark.skipif("not platform.machine().startswith('arm')") + @pytest.mark.skipif( + "not platform.machine().startswith(('arm', 'aarch64'))") def test_bitfield_zero_arm(self): L = FFI().alignof("long long") self.check("char y; int :0;", 0, 4, 4) From noreply at buildbot.pypy.org Thu Jul 24 11:27:12 2014 From: noreply at buildbot.pypy.org (anton_gulenko) Date: Thu, 24 Jul 2014 11:27:12 +0200 (CEST) Subject: [pypy-commit] lang-smalltalk storage: Added --safe-trace flag which omits printing contents of byte objects. Message-ID: <20140724092712.C2DDD1C3273@cobra.cs.uni-duesseldorf.de> Author: Anton Gulenko Branch: storage Changeset: r942:adcfa00d78d1 Date: 2014-07-23 19:27 +0200 http://bitbucket.org/pypy/lang-smalltalk/changeset/adcfa00d78d1/ Log: Added --safe-trace flag which omits printing contents of byte objects. diff --git a/spyvm/interpreter.py b/spyvm/interpreter.py --- a/spyvm/interpreter.py +++ b/spyvm/interpreter.py @@ -648,12 +648,13 @@ def _mustBeBoolean(self, interp, receiver): return self._sendSpecialSelector(interp, receiver, "mustBeBoolean") - + def _call_primitive(self, code, interp, argcount, w_method, w_selector): # ################################################################## if interp.is_tracing(): - interp.print_padded("-> primitive %d \t(in %s, named #%s)" % ( - code, self.w_method().get_identifier_string(), w_selector.str_content())) + interp.print_padded("-> primitive %d \t(in %s, named %s)" % ( + code, self.w_method().get_identifier_string(), + w_selector.selector_string())) func = primitives.prim_holder.prim_table[code] try: # note: argcount does not include rcvr @@ -662,7 +663,7 @@ except primitives.PrimitiveFailedError, e: if interp.is_tracing(): interp.print_padded("-- primitive %d FAILED\t (in %s, named %s)" % ( - code, w_method.safe_identifier_string(), w_selector.str_content())) + code, w_method.safe_identifier_string(), w_selector.selector_string())) raise e def _return(self, return_value, interp, local_return=False): diff --git a/spyvm/model.py b/spyvm/model.py --- a/spyvm/model.py +++ b/spyvm/model.py @@ -194,6 +194,9 @@ def repr_content(self): return self.str_content() + + def selector_string(self): + return self.as_repr_string() class W_SmallInteger(W_Object): """Boxed integer value""" @@ -817,15 +820,22 @@ return self._size def str_content(self): - return "'%s'" % self.as_string() + if self.has_class() and self.w_class.has_space(): + if self.w_class.space().omit_printing_raw_bytes.is_set(): + return "" + else: + return "'%s'" % self.as_string().replace('\r', '\n') def as_string(self): if self.bytes is not None: string = "".join(self.bytes) else: string = "".join([self.c_bytes[i] for i in range(self.size())]) - return string.replace('\r', '\n') - + return string + + def selector_string(self): + return "#" + self.as_string() + def invariant(self): if not W_AbstractObjectWithClassReference.invariant(self): return False diff --git a/spyvm/objspace.py b/spyvm/objspace.py --- a/spyvm/objspace.py +++ b/spyvm/objspace.py @@ -52,6 +52,7 @@ self.suppress_process_switch = ConstantFlag() self.run_spy_hacks = ConstantFlag() self.headless = ConstantFlag() + self.omit_printing_raw_bytes = ConstantFlag() self.classtable = {} self.objtable = {} diff --git a/targetimageloadingsmalltalk.py b/targetimageloadingsmalltalk.py --- a/targetimageloadingsmalltalk.py +++ b/targetimageloadingsmalltalk.py @@ -12,7 +12,7 @@ def _usage(argv): print """ - Usage: %s [-r|-m|-h] [-naPu] [-jpiS] [-tlLE] + Usage: %s [-r|-m|-h] [-naPu] [-jpiS] [-tslLE] - image path (default: Squeak.image) Execution mode: @@ -40,6 +40,7 @@ Logging parameters: -t|--trace - Output a trace of each message, primitive, return value and process switch. + -s|--safe-trace - Like -t, but without printing contents of BytesObjects -l|--storage-log - Output a log of storage operations. -L|--storage-log-aggregate - Output an aggregated storage log at the end of execution. -E|--storage-log-elements - Include classnames of elements into the storage log. @@ -110,6 +111,9 @@ selector, idx = get_parameter(argv, idx, arg) elif arg in ["-t", "--trace"]: trace = True + elif arg in ["-s", "--safe-trace"]: + trace = True + space.omit_printing_raw_bytes.activate() elif arg in ["-p", "--poll"]: poll = True elif arg in ["-a", "--arg"]: From noreply at buildbot.pypy.org Thu Jul 24 11:27:25 2014 From: noreply at buildbot.pypy.org (anton_gulenko) Date: Thu, 24 Jul 2014 11:27:25 +0200 (CEST) Subject: [pypy-commit] lang-smalltalk storage-context-state: Updated test. Message-ID: <20140724092725.60D351C3273@cobra.cs.uni-duesseldorf.de> Author: Anton Gulenko Branch: storage-context-state Changeset: r943:369091d66f5d Date: 2014-07-23 15:30 +0200 http://bitbucket.org/pypy/lang-smalltalk/changeset/369091d66f5d/ Log: Updated test. diff --git a/spyvm/test/test_model.py b/spyvm/test/test_model.py --- a/spyvm/test/test_model.py +++ b/spyvm/test/test_model.py @@ -403,12 +403,23 @@ for i in xrange(6, 8): assert target.pixelbuffer[i] == 0x0 -def test_display_offset_computation(): - dbitmap = model_display.W_MappingDisplayBitmap(space, space.w_Array, 5, 1) +def test_display_offset_computation_even(): + dbitmap = model_display.W_MappingDisplayBitmap(space, space.w_Array, 200, 1) + dbitmap.pitch = 64 + dbitmap.words_per_line = 2 assert dbitmap.compute_pos(0) == 0 - assert dbitmap.compute_pos(1) == 8 - assert dbitmap.size() == 5 * 8 + assert dbitmap.compute_pos(1) == 32 + assert dbitmap.compute_pos(2) == 64 +def test_display_offset_computation_uneven(): + dbitmap = model_display.W_MappingDisplayBitmap(space, space.w_Array, 200, 1) + dbitmap.pitch = 67 + dbitmap.words_per_line = 2 + assert dbitmap.compute_pos(0) == 0 + assert dbitmap.compute_pos(1) == 32 + assert dbitmap.compute_pos(2) == 67 + assert dbitmap.compute_pos(3) == 67 + 32 + @py.test.mark.skipif("socket.gethostname() == 'precise32'") def test_weak_pointers(): w_cls = bootstrap_class(2) From noreply at buildbot.pypy.org Thu Jul 24 11:27:26 2014 From: noreply at buildbot.pypy.org (anton_gulenko) Date: Thu, 24 Jul 2014 11:27:26 +0200 (CEST) Subject: [pypy-commit] lang-smalltalk storage-context-state: Added --safe-trace flag which omits printing contents of byte objects. Message-ID: <20140724092726.860B51C3273@cobra.cs.uni-duesseldorf.de> Author: Anton Gulenko Branch: storage-context-state Changeset: r944:eca2665b8824 Date: 2014-07-23 19:27 +0200 http://bitbucket.org/pypy/lang-smalltalk/changeset/eca2665b8824/ Log: Added --safe-trace flag which omits printing contents of byte objects. diff --git a/spyvm/interpreter.py b/spyvm/interpreter.py --- a/spyvm/interpreter.py +++ b/spyvm/interpreter.py @@ -685,12 +685,13 @@ def _mustBeBoolean(self, interp, receiver): return self._sendSpecialSelector(interp, receiver, "mustBeBoolean") - + def _call_primitive(self, code, interp, argcount, w_method, w_selector): # ################################################################## if interp.is_tracing(): - interp.print_padded("-> primitive %d \t(in %s, named #%s)" % ( - code, self.w_method().get_identifier_string(), w_selector.str_content())) + interp.print_padded("-> primitive %d \t(in %s, named %s)" % ( + code, self.w_method().get_identifier_string(), + w_selector.selector_string())) func = primitives.prim_holder.prim_table[code] try: # note: argcount does not include rcvr @@ -699,7 +700,7 @@ except primitives.PrimitiveFailedError, e: if interp.is_tracing(): interp.print_padded("-- primitive %d FAILED\t (in %s, named %s)" % ( - code, w_method.safe_identifier_string(), w_selector.str_content())) + code, w_method.safe_identifier_string(), w_selector.selector_string())) raise e def _return(self, return_value, interp, local_return=False): diff --git a/spyvm/model.py b/spyvm/model.py --- a/spyvm/model.py +++ b/spyvm/model.py @@ -194,6 +194,9 @@ def repr_content(self): return self.str_content() + + def selector_string(self): + return self.as_repr_string() class W_SmallInteger(W_Object): """Boxed integer value""" @@ -817,15 +820,22 @@ return self._size def str_content(self): - return "'%s'" % self.as_string() + if self.has_class() and self.w_class.has_space(): + if self.w_class.space().omit_printing_raw_bytes.is_set(): + return "" + else: + return "'%s'" % self.as_string().replace('\r', '\n') def as_string(self): if self.bytes is not None: string = "".join(self.bytes) else: string = "".join([self.c_bytes[i] for i in range(self.size())]) - return string.replace('\r', '\n') - + return string + + def selector_string(self): + return "#" + self.as_string() + def invariant(self): if not W_AbstractObjectWithClassReference.invariant(self): return False diff --git a/spyvm/objspace.py b/spyvm/objspace.py --- a/spyvm/objspace.py +++ b/spyvm/objspace.py @@ -52,6 +52,7 @@ self.suppress_process_switch = ConstantFlag() self.run_spy_hacks = ConstantFlag() self.headless = ConstantFlag() + self.omit_printing_raw_bytes = ConstantFlag() self.classtable = {} self.objtable = {} diff --git a/targetimageloadingsmalltalk.py b/targetimageloadingsmalltalk.py --- a/targetimageloadingsmalltalk.py +++ b/targetimageloadingsmalltalk.py @@ -12,7 +12,7 @@ def _usage(argv): print """ - Usage: %s [-r|-m|-h] [-naPu] [-jpiS] [-tlLE] + Usage: %s [-r|-m|-h] [-naPu] [-jpiS] [-tslLE] - image path (default: Squeak.image) Execution mode: @@ -40,6 +40,7 @@ Logging parameters: -t|--trace - Output a trace of each message, primitive, return value and process switch. + -s|--safe-trace - Like -t, but without printing contents of BytesObjects -l|--storage-log - Output a log of storage operations. -L|--storage-log-aggregate - Output an aggregated storage log at the end of execution. -E|--storage-log-elements - Include classnames of elements into the storage log. @@ -110,6 +111,9 @@ selector, idx = get_parameter(argv, idx, arg) elif arg in ["-t", "--trace"]: trace = True + elif arg in ["-s", "--safe-trace"]: + trace = True + space.omit_printing_raw_bytes.activate() elif arg in ["-p", "--poll"]: poll = True elif arg in ["-a", "--arg"]: From noreply at buildbot.pypy.org Thu Jul 24 11:27:27 2014 From: noreply at buildbot.pypy.org (anton_gulenko) Date: Thu, 24 Jul 2014 11:27:27 +0200 (CEST) Subject: [pypy-commit] lang-smalltalk storage-context-state: Merged. Message-ID: <20140724092727.93CF81C3273@cobra.cs.uni-duesseldorf.de> Author: Anton Gulenko Branch: storage-context-state Changeset: r945:6f021854ceab Date: 2014-07-23 19:28 +0200 http://bitbucket.org/pypy/lang-smalltalk/changeset/6f021854ceab/ Log: Merged. From noreply at buildbot.pypy.org Thu Jul 24 14:12:15 2014 From: noreply at buildbot.pypy.org (arigo) Date: Thu, 24 Jul 2014 14:12:15 +0200 (CEST) Subject: [pypy-commit] pypy default: Issue #1817: missing support for virtual_refs pointing to several different types in the same RPython program Message-ID: <20140724121215.052931C1106@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r72480:35fdf446e439 Date: 2014-07-24 14:11 +0200 http://bitbucket.org/pypy/pypy/changeset/35fdf446e439/ Log: Issue #1817: missing support for virtual_refs pointing to several different types in the same RPython program diff --git a/rpython/jit/metainterp/test/test_virtualref.py b/rpython/jit/metainterp/test/test_virtualref.py --- a/rpython/jit/metainterp/test/test_virtualref.py +++ b/rpython/jit/metainterp/test/test_virtualref.py @@ -34,7 +34,7 @@ # def check_call(op, fname): assert op.opname == 'direct_call' - assert op.args[0].value._obj._name == fname + assert op.args[0].value._obj._name.startswith(fname) # ops = [op for block, op in graph.iterblockops()] check_call(ops[-3], 'virtual_ref') diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py --- a/rpython/rlib/jit.py +++ b/rpython/rlib/jit.py @@ -340,6 +340,7 @@ # ____________________________________________________________ # VRefs + at specialize.argtype(0) def virtual_ref(x): """Creates a 'vref' object that contains a reference to 'x'. Calls to virtual_ref/virtual_ref_finish must be properly nested. The idea @@ -351,6 +352,7 @@ return DirectJitVRef(x) virtual_ref.oopspec = 'virtual_ref(x)' + at specialize.argtype(1) def virtual_ref_finish(vref, x): """See docstring in virtual_ref(x)""" keepalive_until_here(x) # otherwise the whole function call is removed From noreply at buildbot.pypy.org Fri Jul 25 11:19:03 2014 From: noreply at buildbot.pypy.org (arigo) Date: Fri, 25 Jul 2014 11:19:03 +0200 (CEST) Subject: [pypy-commit] pypy default: Issue #1823: fix _m_ispad(), which always returned 0 Message-ID: <20140725091903.281091C024A@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r72481:71ab3dcb1309 Date: 2014-07-25 11:18 +0200 http://bitbucket.org/pypy/pypy/changeset/71ab3dcb1309/ Log: Issue #1823: fix _m_ispad(), which always returned 0 diff --git a/lib_pypy/_curses.py b/lib_pypy/_curses.py --- a/lib_pypy/_curses.py +++ b/lib_pypy/_curses.py @@ -309,11 +309,9 @@ #endif int _m_ispad(WINDOW *win) { -#if defined WINDOW_HAS_FLAGS + // may not have _flags (and possibly _ISPAD), + // but for now let's assume that always has it return (win->_flags & _ISPAD); -#else - return 0; -#endif } void _m_getsyx(int *yx) { From noreply at buildbot.pypy.org Fri Jul 25 21:17:11 2014 From: noreply at buildbot.pypy.org (pjenvey) Date: Fri, 25 Jul 2014 21:17:11 +0200 (CEST) Subject: [pypy-commit] pypy py3.3: merge py3k Message-ID: <20140725191711.554351C04B7@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3.3 Changeset: r72482:fc86a76771f4 Date: 2014-07-20 11:06 -0700 http://bitbucket.org/pypy/pypy/changeset/fc86a76771f4/ Log: merge py3k diff --git a/lib-python/3/json/__init__.py b/lib-python/3/json/__init__.py --- a/lib-python/3/json/__init__.py +++ b/lib-python/3/json/__init__.py @@ -104,6 +104,12 @@ __author__ = 'Bob Ippolito ' +try: + # PyPy speedup, the interface is different than CPython's _json + import _pypyjson +except ImportError: + _pypyjson = None + from .decoder import JSONDecoder from .encoder import JSONEncoder @@ -313,7 +319,7 @@ if (cls is None and object_hook is None and parse_int is None and parse_float is None and parse_constant is None and object_pairs_hook is None and not kw): - return _default_decoder.decode(s) + return _pypyjson.loads(s) if _pypyjson else _default_decoder.decode(s) if cls is None: cls = JSONDecoder if object_hook is not None: diff --git a/pypy/interpreter/test/test_generator.py b/pypy/interpreter/test/test_generator.py --- a/pypy/interpreter/test/test_generator.py +++ b/pypy/interpreter/test/test_generator.py @@ -307,13 +307,13 @@ w_co = space.appexec([], '''(): def g(x): yield x + 5 - return g.func_code + return g.__code__ ''') assert should_not_inline(w_co) == False w_co = space.appexec([], '''(): def g(x): yield x + 5 yield x + 6 - return g.func_code + return g.__code__ ''') assert should_not_inline(w_co) == True diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py --- a/pypy/module/_pypyjson/interp_decoder.py +++ b/pypy/module/_pypyjson/interp_decoder.py @@ -56,6 +56,7 @@ self.end_ptr = lltype.malloc(rffi.CCHARPP.TO, 1, flavor='raw') self.pos = 0 self.last_type = TYPE_UNKNOWN + self.memo = {} def close(self): rffi.free_charp(self.ll_chars) @@ -261,6 +262,8 @@ w_name = self.decode_any(i) if self.last_type != TYPE_STRING: self._raise("Key name must be string for object starting at char %d", start) + w_name = self.memo.setdefault(self.space.unicode_w(w_name), w_name) + i = self.skip_whitespace(self.pos) ch = self.ll_chars[i] if ch != ':': diff --git a/pypy/module/_pypyjson/test/test__pypyjson.py b/pypy/module/_pypyjson/test/test__pypyjson.py --- a/pypy/module/_pypyjson/test/test__pypyjson.py +++ b/pypy/module/_pypyjson/test/test__pypyjson.py @@ -187,4 +187,12 @@ import _pypyjson # http://json.org/JSON_checker/test/fail25.json s = '["\ttab\tcharacter\tin\tstring\t"]' - raises(ValueError, "_pypyjson.loads(s)") \ No newline at end of file + raises(ValueError, "_pypyjson.loads(s)") + + def test_keys_reuse(self): + import _pypyjson + s = '[{"a_key": 1, "b_\xe9": 2}, {"a_key": 3, "b_\xe9": 4}]' + rval = _pypyjson.loads(s) + (a, b), (c, d) = sorted(rval[0]), sorted(rval[1]) + assert a is c + assert b is d diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py --- a/pypy/module/array/interp_array.py +++ b/pypy/module/array/interp_array.py @@ -244,7 +244,7 @@ """ size = self.len if size == 0: - return space.wrap('') + return space.wrapbytes('') cbuf = self._charbuf_start() s = rffi.charpsize2str(cbuf, size * self.itemsize) self._charbuf_stop() diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py --- a/pypy/module/posix/interp_posix.py +++ b/pypy/module/posix/interp_posix.py @@ -488,8 +488,7 @@ cur = os.getlogin() except OSError, e: raise wrap_oserror(space, e) - else: - return space.wrap(cur) + return space.fsdecode(space.wrapbytes(cur)) # ____________________________________________________________ @@ -702,14 +701,21 @@ except OSError, e: raise wrap_oserror(space, e) - at unwrap_spec(path='fsencode') -def readlink(space, path): +def readlink(space, w_path): "Return a string representing the path to which the symbolic link points." + is_unicode = space.isinstance_w(w_path, space.w_unicode) + if is_unicode: + path = space.fsencode_w(w_path) + else: + path = space.bytes0_w(w_path) try: result = os.readlink(path) except OSError, e: - raise wrap_oserror(space, e, path) - return space.wrap(result) + raise wrap_oserror2(space, e, w_path) + w_result = space.wrapbytes(result) + if is_unicode: + return space.fsdecode(w_result) + return w_result before_fork_hooks = [] after_fork_child_hooks = [] @@ -899,7 +905,8 @@ r = os.uname() except OSError, e: raise wrap_oserror(space, e) - l_w = [space.wrap(i) for i in [r[0], r[1], r[2], r[3], r[4]]] + l_w = [space.fsdecode(space.wrapbytes(i)) + for i in [r[0], r[1], r[2], r[3], r[4]]] w_tuple = space.newtuple(l_w) w_uname_result = space.getattr(space.getbuiltinmodule(os.name), space.wrap('uname_result')) @@ -1229,7 +1236,7 @@ @unwrap_spec(fd=c_int) def ttyname(space, fd): try: - return space.wrap(os.ttyname(fd)) + return space.fsdecode(space.wrapbytes(os.ttyname(fd))) except OSError, e: raise wrap_oserror(space, e) @@ -1364,7 +1371,7 @@ Return the name of the controlling terminal for this process. """ - return space.wrap(os.ctermid()) + return space.fsdecode(space.wrapbytes(os.ctermid())) @unwrap_spec(fd=c_int) def device_encoding(space, fd): From noreply at buildbot.pypy.org Fri Jul 25 21:17:12 2014 From: noreply at buildbot.pypy.org (pjenvey) Date: Fri, 25 Jul 2014 21:17:12 +0200 (CEST) Subject: [pypy-commit] pypy py3.3: fix OSError aliases (Environment/IOError) causing an obscure clash of names w/ Message-ID: <20140725191712.9D4361C04B7@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3.3 Changeset: r72483:2fd763f1298c Date: 2014-07-25 12:15 -0700 http://bitbucket.org/pypy/pypy/changeset/2fd763f1298c/ Log: fix OSError aliases (Environment/IOError) causing an obscure clash of names w/ export_struct as used by cpyext. fixes translation but breaks their cpyext API names for now (+ a failing test for this) diff --git a/pypy/module/_io/interp_io.py b/pypy/module/_io/interp_io.py --- a/pypy/module/_io/interp_io.py +++ b/pypy/module/_io/interp_io.py @@ -4,7 +4,6 @@ from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.typedef import ( TypeDef, interp_attrproperty, generic_new_descr) -from pypy.module.exceptions.interp_exceptions import W_IOError from pypy.module._io.interp_fileio import W_FileIO from pypy.module._io.interp_textio import W_TextIOWrapper from rpython.rtyper.module.ll_os_stat import STAT_FIELD_TYPES diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -454,6 +454,10 @@ # PyExc_NameError, PyExc_MemoryError, PyExc_RuntimeError, # PyExc_UnicodeEncodeError, PyExc_UnicodeDecodeError, ... for exc_name in exceptions.Module.interpleveldefs.keys(): + if exc_name in ('EnvironmentError', 'IOError'): + # FIXME: aliases of OSError cause a clash of names via + # export_struct + continue GLOBALS['PyExc_' + exc_name] = ( 'PyTypeObject*', 'space.gettypeobject(interp_exceptions.W_%s.typedef)'% (exc_name, )) diff --git a/pypy/module/cpyext/test/test_exception.py b/pypy/module/cpyext/test/test_exception.py --- a/pypy/module/cpyext/test/test_exception.py +++ b/pypy/module/cpyext/test/test_exception.py @@ -1,4 +1,5 @@ from pypy.module.cpyext.test.test_api import BaseApiTest +from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from pypy.module.cpyext.pyobject import make_ref class TestExceptions(BaseApiTest): @@ -27,3 +28,16 @@ api.PyException_SetCause(w_exc, make_ref(space, w_cause)) assert space.is_w(api.PyException_GetCause(w_exc), w_cause) + +class AppTestExceptions(AppTestCpythonExtensionBase): + + def test_OSError_aliases(self): + module = self.import_extension('foo', [ + ("get_aliases", "METH_NOARGS", + """ + return PyTuple_Pack(2, + PyExc_EnvironmentError, + PyExc_IOError); + """), + ]) + assert module.get_aliases() == (OSError, OSError) diff --git a/pypy/module/exceptions/__init__.py b/pypy/module/exceptions/__init__.py --- a/pypy/module/exceptions/__init__.py +++ b/pypy/module/exceptions/__init__.py @@ -21,14 +21,14 @@ 'ConnectionResetError': 'interp_exceptions.W_ConnectionResetError', 'DeprecationWarning' : 'interp_exceptions.W_DeprecationWarning', 'EOFError' : 'interp_exceptions.W_EOFError', - 'EnvironmentError' : 'interp_exceptions.W_EnvironmentError', + 'EnvironmentError' : 'interp_exceptions.W_OSError', 'Exception' : 'interp_exceptions.W_Exception', 'FileExistsError': 'interp_exceptions.W_FileExistsError', 'FileNotFoundError': 'interp_exceptions.W_FileNotFoundError', 'FloatingPointError' : 'interp_exceptions.W_FloatingPointError', 'FutureWarning' : 'interp_exceptions.W_FutureWarning', 'GeneratorExit' : 'interp_exceptions.W_GeneratorExit', - 'IOError' : 'interp_exceptions.W_IOError', + 'IOError' : 'interp_exceptions.W_OSError', 'ImportError' : 'interp_exceptions.W_ImportError', 'ImportWarning' : 'interp_exceptions.W_ImportWarning', 'IndentationError' : 'interp_exceptions.W_IndentationError', diff --git a/pypy/module/exceptions/interp_exceptions.py b/pypy/module/exceptions/interp_exceptions.py --- a/pypy/module/exceptions/interp_exceptions.py +++ b/pypy/module/exceptions/interp_exceptions.py @@ -497,9 +497,6 @@ filename = readwrite_attrproperty_w('w_filename', W_OSError), ) -W_EnvironmentError = W_OSError -W_IOError = W_OSError - class W_WindowsError(W_OSError): """MS-Windows OS system call failed.""" From noreply at buildbot.pypy.org Fri Jul 25 22:12:43 2014 From: noreply at buildbot.pypy.org (pjenvey) Date: Fri, 25 Jul 2014 22:12:43 +0200 (CEST) Subject: [pypy-commit] pypy vendor/stdlib-3.3.5: fix the version Message-ID: <20140725201243.624951C0588@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: vendor/stdlib-3.3.5 Changeset: r72484:80b492d79663 Date: 2014-07-25 13:07 -0700 http://bitbucket.org/pypy/pypy/changeset/80b492d79663/ Log: fix the version diff --git a/lib-python/stdlib-version.txt b/lib-python/stdlib-version.txt --- a/lib-python/stdlib-version.txt +++ b/lib-python/stdlib-version.txt @@ -6,4 +6,4 @@ 2.7:: 3a1db0d2747e (2.7) v2.7.6 3:: - cef745775b65 (3.2) v3.2.5 + 62cf4e77f785 (3.3) v3.3.5 From noreply at buildbot.pypy.org Fri Jul 25 22:12:44 2014 From: noreply at buildbot.pypy.org (pjenvey) Date: Fri, 25 Jul 2014 22:12:44 +0200 (CEST) Subject: [pypy-commit] pypy py3.3: merge vendor/stdlib-3.3.5 Message-ID: <20140725201244.AFDF21C0588@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3.3 Changeset: r72485:78836101ec63 Date: 2014-07-25 13:08 -0700 http://bitbucket.org/pypy/pypy/changeset/78836101ec63/ Log: merge vendor/stdlib-3.3.5 diff --git a/lib-python/stdlib-version.txt b/lib-python/stdlib-version.txt --- a/lib-python/stdlib-version.txt +++ b/lib-python/stdlib-version.txt @@ -6,4 +6,4 @@ 2.7:: 3a1db0d2747e (2.7) v2.7.6 3:: - cef745775b65 (3.2) v3.2.5 + 62cf4e77f785 (3.3) v3.3.5 From noreply at buildbot.pypy.org Fri Jul 25 22:12:45 2014 From: noreply at buildbot.pypy.org (pjenvey) Date: Fri, 25 Jul 2014 22:12:45 +0200 (CEST) Subject: [pypy-commit] pypy py3.3: fix the version Message-ID: <20140725201245.E6F9F1C0588@cobra.cs.uni-duesseldorf.de> Author: Philip Jenvey Branch: py3.3 Changeset: r72486:ff5661747625 Date: 2014-07-25 13:11 -0700 http://bitbucket.org/pypy/pypy/changeset/ff5661747625/ Log: fix the version diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h --- a/pypy/module/cpyext/include/patchlevel.h +++ b/pypy/module/cpyext/include/patchlevel.h @@ -20,13 +20,13 @@ /* Version parsed out into numeric values */ #define PY_MAJOR_VERSION 3 -#define PY_MINOR_VERSION 2 +#define PY_MINOR_VERSION 3 #define PY_MICRO_VERSION 5 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_FINAL #define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "3.2.5" +#define PY_VERSION "3.3.5" /* PyPy version as a string */ #define PYPY_VERSION "2.4.0-alpha0" diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -6,7 +6,7 @@ from pypy.interpreter import gateway #XXX # the release serial 42 is not in range(16) -CPYTHON_VERSION = (3, 2, 5, "final", 0) +CPYTHON_VERSION = (3, 3, 5, "final", 0) #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py CPYTHON_API_VERSION = 1013 #XXX # sync with include/modsupport.h From noreply at buildbot.pypy.org Sat Jul 26 11:04:11 2014 From: noreply at buildbot.pypy.org (kvas) Date: Sat, 26 Jul 2014 11:04:11 +0200 (CEST) Subject: [pypy-commit] pypy py3.3: NoneType can be constructed. Message-ID: <20140726090411.62F051C085C@cobra.cs.uni-duesseldorf.de> Author: Vasily Kuznetsov Branch: py3.3 Changeset: r72487:032296946827 Date: 2014-07-26 11:00 +0200 http://bitbucket.org/pypy/pypy/changeset/032296946827/ Log: NoneType can be constructed. diff --git a/pypy/module/__builtin__/test/test_construct_singletons.py b/pypy/module/__builtin__/test/test_construct_singletons.py new file mode 100644 --- /dev/null +++ b/pypy/module/__builtin__/test/test_construct_singletons.py @@ -0,0 +1,7 @@ +class AppTestConstructSingletons: + + def test_construct_singletons(self): + none_type = type(None) + assert none_type() is None + raises(TypeError, none_type, 1, 2) + raises(TypeError, none_type, a=1, b=2) diff --git a/pypy/objspace/std/nonetype.py b/pypy/objspace/std/nonetype.py --- a/pypy/objspace/std/nonetype.py +++ b/pypy/objspace/std/nonetype.py @@ -1,8 +1,15 @@ from pypy.objspace.std.stdtypedef import StdTypeDef +from pypy.interpreter import gateway +def descr__new__(space, w_type): + return space.w_None + # ____________________________________________________________ none_typedef = StdTypeDef("NoneType", + __new__ = gateway.interp2app(descr__new__) ) none_typedef.acceptable_as_base_class = False + + From noreply at buildbot.pypy.org Sat Jul 26 12:32:29 2014 From: noreply at buildbot.pypy.org (Arjun Naik) Date: Sat, 26 Jul 2014 12:32:29 +0200 (CEST) Subject: [pypy-commit] pypy py3.3: Added the missing log2 in the math module. Message-ID: <20140726103229.36EE01D358B@cobra.cs.uni-duesseldorf.de> Author: Arjun Naik Branch: py3.3 Changeset: r72488:e54b7fe58ecf Date: 2014-07-26 10:41 +0200 http://bitbucket.org/pypy/pypy/changeset/e54b7fe58ecf/ Log: Added the missing log2 in the math module. diff --git a/pypy/module/math/__init__.py b/pypy/module/math/__init__.py --- a/pypy/module/math/__init__.py +++ b/pypy/module/math/__init__.py @@ -23,6 +23,7 @@ 'frexp' : 'interp_math.frexp', 'degrees' : 'interp_math.degrees', 'log' : 'interp_math.log', + 'log2' : 'interp_math.log2', 'log10' : 'interp_math.log10', 'fmod' : 'interp_math.fmod', 'atan' : 'interp_math.atan', diff --git a/pypy/module/math/interp_math.py b/pypy/module/math/interp_math.py --- a/pypy/module/math/interp_math.py +++ b/pypy/module/math/interp_math.py @@ -228,6 +228,11 @@ return math1(space, math.log, w_base) return _log_any(space, w_x, base) +def log2(space, w_x): + """log2(x) -> the base 2 logarithm of x. + """ + return _log_any(space, w_x, 2.0) + def log10(space, w_x): """log10(x) -> the base 10 logarithm of x. """ diff --git a/pypy/module/math/test/test_math.py b/pypy/module/math/test/test_math.py --- a/pypy/module/math/test/test_math.py +++ b/pypy/module/math/test/test_math.py @@ -148,6 +148,19 @@ raises(ValueError, math.log1p, -1) raises(ValueError, math.log1p, -100) + def test_log2(self): + import math + self.ftest(math.log2(0.125), -3) + self.ftest(math.log2(0.5), -1) + self.ftest(math.log2(4), 2) + + def test_log10(self): + import math + self.ftest(math.log10(0.1), -1) + self.ftest(math.log10(10), 1) + self.ftest(math.log10(100), 2) + self.ftest(math.log10(0.01), -2) + def test_acosh(self): import math self.ftest(math.acosh(1), 0) From noreply at buildbot.pypy.org Sat Jul 26 12:32:30 2014 From: noreply at buildbot.pypy.org (Arjun Naik) Date: Sat, 26 Jul 2014 12:32:30 +0200 (CEST) Subject: [pypy-commit] pypy py3.3: Added missing log2 in math module. Message-ID: <20140726103230.8E7FF1D358B@cobra.cs.uni-duesseldorf.de> Author: Arjun Naik Branch: py3.3 Changeset: r72489:342ebe4c4d7f Date: 2014-07-26 11:16 +0200 http://bitbucket.org/pypy/pypy/changeset/342ebe4c4d7f/ Log: Added missing log2 in math module. diff --git a/pypy/module/math/__init__.py b/pypy/module/math/__init__.py --- a/pypy/module/math/__init__.py +++ b/pypy/module/math/__init__.py @@ -23,6 +23,7 @@ 'frexp' : 'interp_math.frexp', 'degrees' : 'interp_math.degrees', 'log' : 'interp_math.log', + 'log2' : 'interp_math.log2', 'log10' : 'interp_math.log10', 'fmod' : 'interp_math.fmod', 'atan' : 'interp_math.atan', diff --git a/pypy/module/math/interp_math.py b/pypy/module/math/interp_math.py --- a/pypy/module/math/interp_math.py +++ b/pypy/module/math/interp_math.py @@ -228,6 +228,11 @@ return math1(space, math.log, w_base) return _log_any(space, w_x, base) +def log2(space, w_x): + """log2(x) -> the base 2 logarithm of x. + """ + return _log_any(space, w_x, 2.0) + def log10(space, w_x): """log10(x) -> the base 10 logarithm of x. """ diff --git a/pypy/module/math/test/test_math.py b/pypy/module/math/test/test_math.py --- a/pypy/module/math/test/test_math.py +++ b/pypy/module/math/test/test_math.py @@ -148,6 +148,19 @@ raises(ValueError, math.log1p, -1) raises(ValueError, math.log1p, -100) + def test_log2(self): + import math + self.ftest(math.log2(0.125), -3) + self.ftest(math.log2(0.5), -1) + self.ftest(math.log2(4), 2) + + def test_log10(self): + import math + self.ftest(math.log10(0.1), -1) + self.ftest(math.log10(10), 1) + self.ftest(math.log10(100), 2) + self.ftest(math.log10(0.01), -2) + def test_acosh(self): import math self.ftest(math.acosh(1), 0) From noreply at buildbot.pypy.org Sat Jul 26 12:32:31 2014 From: noreply at buildbot.pypy.org (kvas) Date: Sat, 26 Jul 2014 12:32:31 +0200 (CEST) Subject: [pypy-commit] pypy py3.3: __dir__ is allowed to return a tuple (it's converted to list like in cpython3). Message-ID: <20140726103231.BD3E91D358B@cobra.cs.uni-duesseldorf.de> Author: Vasily Kuznetsov Branch: py3.3 Changeset: r72490:71501209fc12 Date: 2014-07-26 11:22 +0200 http://bitbucket.org/pypy/pypy/changeset/71501209fc12/ Log: __dir__ is allowed to return a tuple (it's converted to list like in cpython3). diff --git a/pypy/module/__builtin__/app_inspect.py b/pypy/module/__builtin__/app_inspect.py --- a/pypy/module/__builtin__/app_inspect.py +++ b/pypy/module/__builtin__/app_inspect.py @@ -52,6 +52,8 @@ dir_meth = lookup_special(obj, "__dir__") if dir_meth is not None: result = dir_meth() + if isinstance(result, tuple): + result = list(result) if not isinstance(result, list): raise TypeError("__dir__() must return a list, not %r" % ( type(result),)) diff --git a/pypy/module/__builtin__/test/test_dir.py b/pypy/module/__builtin__/test/test_dir.py new file mode 100644 --- /dev/null +++ b/pypy/module/__builtin__/test/test_dir.py @@ -0,0 +1,12 @@ +class AppTestDir: + + def test_dir_obj__dir__tuple(self): + """When __dir__ method returns a tuple, python3 converts it to list.""" + + class Foo(object): + def __dir__(self): + return ("b", "c", "a") + + res = dir(Foo()) + assert isinstance(res, list) + assert res == ["a", "b", "c"] From noreply at buildbot.pypy.org Sat Jul 26 12:32:33 2014 From: noreply at buildbot.pypy.org (Boglarka Vezer) Date: Sat, 26 Jul 2014 12:32:33 +0200 (CEST) Subject: [pypy-commit] pypy py3.3: fixing typo in range hash function Message-ID: <20140726103233.01ED61D358B@cobra.cs.uni-duesseldorf.de> Author: Boglarka Vezer Branch: py3.3 Changeset: r72491:b8a56b701025 Date: 2014-07-26 11:26 +0200 http://bitbucket.org/pypy/pypy/changeset/b8a56b701025/ Log: fixing typo in range hash function diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py --- a/pypy/module/__builtin__/functional.py +++ b/pypy/module/__builtin__/functional.py @@ -482,7 +482,7 @@ def descr_hash(self, space): if space.eq_w(self.w_length, space.wrap(0)): w_tup = space.newtuple([self.w_length, space.w_None, space.w_None]) - elif space.eq_w(self.w_length, space.wrap(0)): + elif space.eq_w(self.w_length, space.wrap(1)): w_tup = space.newtuple([self.w_length, self.w_start, space.w_None]) else: w_tup = space.newtuple([self.w_length, self.w_start, self.w_step]) diff --git a/pypy/module/__builtin__/test/test_functional.py b/pypy/module/__builtin__/test/test_functional.py --- a/pypy/module/__builtin__/test/test_functional.py +++ b/pypy/module/__builtin__/test/test_functional.py @@ -484,7 +484,7 @@ for a in test_ranges: for b in test_ranges: if a == b: - assert (hash(a), hash(b)) + assert hash(a) == hash(b) # Ranges are unequal to other types (even sequence types) assert (range(0) == ()) is False From noreply at buildbot.pypy.org Sat Jul 26 12:32:34 2014 From: noreply at buildbot.pypy.org (kvas) Date: Sat, 26 Jul 2014 12:32:34 +0200 (CEST) Subject: [pypy-commit] pypy py3.3: __dir__ is allowed to return any iterable like in cpython3. Message-ID: <20140726103234.29F091D358B@cobra.cs.uni-duesseldorf.de> Author: Vasily Kuznetsov Branch: py3.3 Changeset: r72492:72a3a7810d5c Date: 2014-07-26 12:05 +0200 http://bitbucket.org/pypy/pypy/changeset/72a3a7810d5c/ Log: __dir__ is allowed to return any iterable like in cpython3. diff --git a/pypy/module/__builtin__/app_inspect.py b/pypy/module/__builtin__/app_inspect.py --- a/pypy/module/__builtin__/app_inspect.py +++ b/pypy/module/__builtin__/app_inspect.py @@ -52,11 +52,8 @@ dir_meth = lookup_special(obj, "__dir__") if dir_meth is not None: result = dir_meth() - if isinstance(result, tuple): - result = list(result) if not isinstance(result, list): - raise TypeError("__dir__() must return a list, not %r" % ( - type(result),)) + result = list(result) # Will throw TypeError if not iterable result.sort() return result elif isinstance(obj, types.ModuleType): diff --git a/pypy/module/__builtin__/test/test_dir.py b/pypy/module/__builtin__/test/test_dir.py --- a/pypy/module/__builtin__/test/test_dir.py +++ b/pypy/module/__builtin__/test/test_dir.py @@ -1,12 +1,26 @@ class AppTestDir: def test_dir_obj__dir__tuple(self): - """When __dir__ method returns a tuple, python3 converts it to list.""" - + """If __dir__ method returns a tuple, cpython3 converts it to list.""" class Foo(object): def __dir__(self): return ("b", "c", "a") - res = dir(Foo()) assert isinstance(res, list) assert res == ["a", "b", "c"] + + def test_dir_obj__dir__genexp(self): + """Generator expression is also converted to list by cpython3.""" + class Foo(object): + def __dir__(self): + return (i for i in ["b", "c", "a"]) + res = dir(Foo()) + assert isinstance(res, list) + assert res == ["a", "b", "c"] + + def test_dir_obj__dir__noniter(self): + """If result of __dir__ is not iterable, it's an error.""" + class Foo(object): + def __dir__(self): + return 42 + raises(TypeError, dir, Foo()) From noreply at buildbot.pypy.org Sat Jul 26 13:02:24 2014 From: noreply at buildbot.pypy.org (numerodix) Date: Sat, 26 Jul 2014 13:02:24 +0200 (CEST) Subject: [pypy-commit] pypy py3.3: patching cpython3 csv test as the exception message has bad grammar Message-ID: <20140726110224.970001D36A6@cobra.cs.uni-duesseldorf.de> Author: Martin Matusiak Branch: py3.3 Changeset: r72493:c94a608f5616 Date: 2014-07-26 11:11 +0200 http://bitbucket.org/pypy/pypy/changeset/c94a608f5616/ Log: patching cpython3 csv test as the exception message has bad grammar diff --git a/lib-python/3/test/test_csv.py b/lib-python/3/test/test_csv.py --- a/lib-python/3/test/test_csv.py +++ b/lib-python/3/test/test_csv.py @@ -766,8 +766,9 @@ mydialect.quotechar = "''" with self.assertRaises(csv.Error) as cm: mydialect() + # NOTE: Patched exception message since cpython uses bad grammar (cpython issue22076) self.assertEqual(str(cm.exception), - '"quotechar" must be an 1-character string') + '"quotechar" must be a 1-character string') mydialect.quotechar = 4 with self.assertRaises(csv.Error) as cm: @@ -789,14 +790,16 @@ mydialect.delimiter = ":::" with self.assertRaises(csv.Error) as cm: mydialect() + # NOTE: Patched exception message since cpython uses bad grammar (cpython issue22076) self.assertEqual(str(cm.exception), - '"delimiter" must be an 1-character string') + '"delimiter" must be a 1-character string') mydialect.delimiter = "" with self.assertRaises(csv.Error) as cm: mydialect() + # NOTE: Patched exception message since cpython uses bad grammar (cpython issue22076) self.assertEqual(str(cm.exception), - '"delimiter" must be an 1-character string') + '"delimiter" must be a 1-character string') mydialect.delimiter = b"," with self.assertRaises(csv.Error) as cm: From noreply at buildbot.pypy.org Sat Jul 26 13:02:25 2014 From: noreply at buildbot.pypy.org (numerodix) Date: Sat, 26 Jul 2014 13:02:25 +0200 (CEST) Subject: [pypy-commit] pypy py3.3: fixing _csv lineterminator exception message Message-ID: <20140726110225.BF46C1D36A6@cobra.cs.uni-duesseldorf.de> Author: Martin Matusiak Branch: py3.3 Changeset: r72494:cbfe89ead8c2 Date: 2014-07-26 12:12 +0200 http://bitbucket.org/pypy/pypy/changeset/cbfe89ead8c2/ Log: fixing _csv lineterminator exception message diff --git a/pypy/module/_csv/interp_csv.py b/pypy/module/_csv/interp_csv.py --- a/pypy/module/_csv/interp_csv.py +++ b/pypy/module/_csv/interp_csv.py @@ -34,10 +34,15 @@ return default return space.int_w(w_src) -def _get_str(space, w_src, default): +def _get_str(space, w_src, default, attrname): if w_src is None: return default - return space.unicode_w(w_src) + try: + return space.unicode_w(w_src) + except OperationError as e: + if e.match(space, space.w_TypeError): + raise oefmt(space.w_TypeError, '"%s" must be a string', attrname) + raise def _get_char(space, w_src, default, name): if w_src is None: @@ -91,7 +96,7 @@ dialect.delimiter = _get_char(space, w_delimiter, u',', 'delimiter') dialect.doublequote = _get_bool(space, w_doublequote, True) dialect.escapechar = _get_char(space, w_escapechar, u'\0', 'escapechar') - dialect.lineterminator = _get_str(space, w_lineterminator, u'\r\n') + dialect.lineterminator = _get_str(space, w_lineterminator, u'\r\n', 'lineterminator') dialect.quotechar = _get_char(space, w_quotechar, u'"', 'quotechar') tmp_quoting = _get_int(space, w_quoting, QUOTE_MINIMAL) dialect.skipinitialspace = _get_bool(space, w_skipinitialspace, False) diff --git a/pypy/module/_csv/test/test_dialect.py b/pypy/module/_csv/test/test_dialect.py --- a/pypy/module/_csv/test/test_dialect.py +++ b/pypy/module/_csv/test/test_dialect.py @@ -67,6 +67,9 @@ kwargs = {name: value} raises(TypeError, _csv.register_dialect, 'foo1', **kwargs) + exc_info = raises(TypeError, _csv.register_dialect, 'foo1', lineterminator=4) + assert exc_info.value.args[0] == '"lineterminator" must be a string' + def test_bool_arg(self): # boolean arguments take *any* object and use its truth-value import _csv From noreply at buildbot.pypy.org Sat Jul 26 13:02:26 2014 From: noreply at buildbot.pypy.org (numerodix) Date: Sat, 26 Jul 2014 13:02:26 +0200 (CEST) Subject: [pypy-commit] pypy py3.3: Merged pypy/pypy/py3.3 into py3.3 Message-ID: <20140726110226.E76B61D36A6@cobra.cs.uni-duesseldorf.de> Author: Martin Matusiak Branch: py3.3 Changeset: r72495:61fd7b56f3ed Date: 2014-07-26 12:41 +0200 http://bitbucket.org/pypy/pypy/changeset/61fd7b56f3ed/ Log: Merged pypy/pypy/py3.3 into py3.3 diff --git a/pypy/module/__builtin__/app_inspect.py b/pypy/module/__builtin__/app_inspect.py --- a/pypy/module/__builtin__/app_inspect.py +++ b/pypy/module/__builtin__/app_inspect.py @@ -53,8 +53,7 @@ if dir_meth is not None: result = dir_meth() if not isinstance(result, list): - raise TypeError("__dir__() must return a list, not %r" % ( - type(result),)) + result = list(result) # Will throw TypeError if not iterable result.sort() return result elif isinstance(obj, types.ModuleType): diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py --- a/pypy/module/__builtin__/functional.py +++ b/pypy/module/__builtin__/functional.py @@ -482,7 +482,7 @@ def descr_hash(self, space): if space.eq_w(self.w_length, space.wrap(0)): w_tup = space.newtuple([self.w_length, space.w_None, space.w_None]) - elif space.eq_w(self.w_length, space.wrap(0)): + elif space.eq_w(self.w_length, space.wrap(1)): w_tup = space.newtuple([self.w_length, self.w_start, space.w_None]) else: w_tup = space.newtuple([self.w_length, self.w_start, self.w_step]) diff --git a/pypy/module/__builtin__/test/test_construct_singletons.py b/pypy/module/__builtin__/test/test_construct_singletons.py new file mode 100644 --- /dev/null +++ b/pypy/module/__builtin__/test/test_construct_singletons.py @@ -0,0 +1,7 @@ +class AppTestConstructSingletons: + + def test_construct_singletons(self): + none_type = type(None) + assert none_type() is None + raises(TypeError, none_type, 1, 2) + raises(TypeError, none_type, a=1, b=2) diff --git a/pypy/module/__builtin__/test/test_dir.py b/pypy/module/__builtin__/test/test_dir.py new file mode 100644 --- /dev/null +++ b/pypy/module/__builtin__/test/test_dir.py @@ -0,0 +1,26 @@ +class AppTestDir: + + def test_dir_obj__dir__tuple(self): + """If __dir__ method returns a tuple, cpython3 converts it to list.""" + class Foo(object): + def __dir__(self): + return ("b", "c", "a") + res = dir(Foo()) + assert isinstance(res, list) + assert res == ["a", "b", "c"] + + def test_dir_obj__dir__genexp(self): + """Generator expression is also converted to list by cpython3.""" + class Foo(object): + def __dir__(self): + return (i for i in ["b", "c", "a"]) + res = dir(Foo()) + assert isinstance(res, list) + assert res == ["a", "b", "c"] + + def test_dir_obj__dir__noniter(self): + """If result of __dir__ is not iterable, it's an error.""" + class Foo(object): + def __dir__(self): + return 42 + raises(TypeError, dir, Foo()) diff --git a/pypy/module/__builtin__/test/test_functional.py b/pypy/module/__builtin__/test/test_functional.py --- a/pypy/module/__builtin__/test/test_functional.py +++ b/pypy/module/__builtin__/test/test_functional.py @@ -484,7 +484,7 @@ for a in test_ranges: for b in test_ranges: if a == b: - assert (hash(a), hash(b)) + assert hash(a) == hash(b) # Ranges are unequal to other types (even sequence types) assert (range(0) == ()) is False diff --git a/pypy/module/math/__init__.py b/pypy/module/math/__init__.py --- a/pypy/module/math/__init__.py +++ b/pypy/module/math/__init__.py @@ -23,6 +23,7 @@ 'frexp' : 'interp_math.frexp', 'degrees' : 'interp_math.degrees', 'log' : 'interp_math.log', + 'log2' : 'interp_math.log2', 'log10' : 'interp_math.log10', 'fmod' : 'interp_math.fmod', 'atan' : 'interp_math.atan', diff --git a/pypy/module/math/interp_math.py b/pypy/module/math/interp_math.py --- a/pypy/module/math/interp_math.py +++ b/pypy/module/math/interp_math.py @@ -228,6 +228,11 @@ return math1(space, math.log, w_base) return _log_any(space, w_x, base) +def log2(space, w_x): + """log2(x) -> the base 2 logarithm of x. + """ + return _log_any(space, w_x, 2.0) + def log10(space, w_x): """log10(x) -> the base 10 logarithm of x. """ diff --git a/pypy/module/math/test/test_math.py b/pypy/module/math/test/test_math.py --- a/pypy/module/math/test/test_math.py +++ b/pypy/module/math/test/test_math.py @@ -148,6 +148,19 @@ raises(ValueError, math.log1p, -1) raises(ValueError, math.log1p, -100) + def test_log2(self): + import math + self.ftest(math.log2(0.125), -3) + self.ftest(math.log2(0.5), -1) + self.ftest(math.log2(4), 2) + + def test_log10(self): + import math + self.ftest(math.log10(0.1), -1) + self.ftest(math.log10(10), 1) + self.ftest(math.log10(100), 2) + self.ftest(math.log10(0.01), -2) + def test_acosh(self): import math self.ftest(math.acosh(1), 0) diff --git a/pypy/objspace/std/nonetype.py b/pypy/objspace/std/nonetype.py --- a/pypy/objspace/std/nonetype.py +++ b/pypy/objspace/std/nonetype.py @@ -1,8 +1,15 @@ from pypy.objspace.std.stdtypedef import StdTypeDef +from pypy.interpreter import gateway +def descr__new__(space, w_type): + return space.w_None + # ____________________________________________________________ none_typedef = StdTypeDef("NoneType", + __new__ = gateway.interp2app(descr__new__) ) none_typedef.acceptable_as_base_class = False + + From noreply at buildbot.pypy.org Sat Jul 26 13:05:44 2014 From: noreply at buildbot.pypy.org (mswart) Date: Sat, 26 Jul 2014 13:05:44 +0200 (CEST) Subject: [pypy-commit] lang-smalltalk stmgc-c7: End image loading with \n Message-ID: <20140726110544.BD4F31D36A6@cobra.cs.uni-duesseldorf.de> Author: Malte Swart Branch: stmgc-c7 Changeset: r946:a772ee2447d9 Date: 2014-07-26 13:05 +0200 http://bitbucket.org/pypy/lang-smalltalk/changeset/a772ee2447d9/ Log: End image loading with \n For better reading of further output, end image loading with \n. By this the ouput is separated by a new line. diff --git a/spyvm/squeakimage.py b/spyvm/squeakimage.py --- a/spyvm/squeakimage.py +++ b/spyvm/squeakimage.py @@ -231,6 +231,7 @@ self.init_w_objects() self.fillin_w_objects() self.synchronize_shadows() + os.write(2, "\n") def read_version(self): # 1 word version From noreply at buildbot.pypy.org Sat Jul 26 13:25:47 2014 From: noreply at buildbot.pypy.org (amauryfa) Date: Sat, 26 Jul 2014 13:25:47 +0200 (CEST) Subject: [pypy-commit] pypy py3.3: _posixsubprocess is not optional anymore. Message-ID: <20140726112547.C3D881C04B7@cobra.cs.uni-duesseldorf.de> Author: Amaury Forgeot d'Arc Branch: py3.3 Changeset: r72496:2e4e787ecfa6 Date: 2014-07-26 13:22 +0200 http://bitbucket.org/pypy/pypy/changeset/2e4e787ecfa6/ Log: _posixsubprocess is not optional anymore. diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -17,7 +17,7 @@ usemodules = ['binascii', 'posix', 'signal', 'struct', 'rctime'] # py3k os.open uses subprocess, requiring the following per platform if os.name != 'nt': - usemodules += ['fcntl', 'select'] + usemodules += ['fcntl', 'select', '_posixsubprocess'] else: usemodules += ['_rawffi', 'thread'] mod.space = gettestobjspace(usemodules=usemodules) From noreply at buildbot.pypy.org Sat Jul 26 13:55:40 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 26 Jul 2014 13:55:40 +0200 (CEST) Subject: [pypy-commit] pypy default: Oops. If, during translation, we execute app-level code that contains a Message-ID: <20140726115540.B8B211C06AE@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r72497:c97bfecbfc61 Date: 2014-07-26 13:54 +0200 http://bitbucket.org/pypy/pypy/changeset/c97bfecbfc61/ Log: Oops. If, during translation, we execute app-level code that contains a "continue" in a "try:" block, and if we're translating with "-Ojit", then crash. This case occurs in the py3.3 branch. diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -200,7 +200,7 @@ elif opcode == opcodedesc.BREAK_LOOP.index: next_instr = self.BREAK_LOOP(oparg, next_instr) elif opcode == opcodedesc.CONTINUE_LOOP.index: - next_instr = self.CONTINUE_LOOP(oparg, next_instr) + return self.CONTINUE_LOOP(oparg, next_instr) elif opcode == opcodedesc.FOR_ITER.index: next_instr = self.FOR_ITER(oparg, next_instr) elif opcode == opcodedesc.JUMP_FORWARD.index: From noreply at buildbot.pypy.org Sat Jul 26 13:57:46 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 26 Jul 2014 13:57:46 +0200 (CEST) Subject: [pypy-commit] pypy py3.3: Oops. If, during translation, we execute app-level code that contains a Message-ID: <20140726115746.9C0BD1C085C@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: py3.3 Changeset: r72498:5fbb8de72550 Date: 2014-07-26 13:54 +0200 http://bitbucket.org/pypy/pypy/changeset/5fbb8de72550/ Log: Oops. If, during translation, we execute app-level code that contains a "continue" in a "try:" block, and if we're translating with "-Ojit", then crash. This case occurs in the py3.3 branch. diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -204,7 +204,7 @@ elif opcode == opcodedesc.BREAK_LOOP.index: next_instr = self.BREAK_LOOP(oparg, next_instr) elif opcode == opcodedesc.CONTINUE_LOOP.index: - next_instr = self.CONTINUE_LOOP(oparg, next_instr) + return self.CONTINUE_LOOP(oparg, next_instr) elif opcode == opcodedesc.FOR_ITER.index: next_instr = self.FOR_ITER(oparg, next_instr) elif opcode == opcodedesc.JUMP_FORWARD.index: From noreply at buildbot.pypy.org Sat Jul 26 13:57:47 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 26 Jul 2014 13:57:47 +0200 (CEST) Subject: [pypy-commit] pypy py3.3: Remove unused return value Message-ID: <20140726115747.C5B3C1C085C@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: py3.3 Changeset: r72499:900ace194cd5 Date: 2014-07-26 13:56 +0200 http://bitbucket.org/pypy/pypy/changeset/900ace194cd5/ Log: Remove unused return value diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -1022,7 +1022,6 @@ raise w_value = space.w_None self.pushvalue(w_value) - return next_instr else: # iter remains on stack, w_retval is value to be yielded. self.pushvalue(w_retval) From noreply at buildbot.pypy.org Sat Jul 26 13:57:48 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 26 Jul 2014 13:57:48 +0200 (CEST) Subject: [pypy-commit] pypy py3.3: merge heads Message-ID: <20140726115748.EB3921C085C@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: py3.3 Changeset: r72500:38d2f98adca7 Date: 2014-07-26 13:57 +0200 http://bitbucket.org/pypy/pypy/changeset/38d2f98adca7/ Log: merge heads diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -17,7 +17,7 @@ usemodules = ['binascii', 'posix', 'signal', 'struct', 'rctime'] # py3k os.open uses subprocess, requiring the following per platform if os.name != 'nt': - usemodules += ['fcntl', 'select'] + usemodules += ['fcntl', 'select', '_posixsubprocess'] else: usemodules += ['_rawffi', 'thread'] mod.space = gettestobjspace(usemodules=usemodules) From noreply at buildbot.pypy.org Sat Jul 26 13:59:58 2014 From: noreply at buildbot.pypy.org (agobi) Date: Sat, 26 Jul 2014 13:59:58 +0200 (CEST) Subject: [pypy-commit] pypy py3.3: creating the RTLD_* constants in the module posix Message-ID: <20140726115958.1DC2B1C085C@cobra.cs.uni-duesseldorf.de> Author: Attila Gobi Branch: py3.3 Changeset: r72501:2e121b41a4c3 Date: 2014-07-26 13:47 +0200 http://bitbucket.org/pypy/pypy/changeset/2e121b41a4c3/ Log: creating the RTLD_* constants in the module posix diff --git a/pypy/module/posix/__init__.py b/pypy/module/posix/__init__.py --- a/pypy/module/posix/__init__.py +++ b/pypy/module/posix/__init__.py @@ -1,5 +1,6 @@ from pypy.interpreter.mixedmodule import MixedModule from rpython.rtyper.module.ll_os import RegisterOs +from rpython.rlib import rdynload import os exec 'import %s as posix' % os.name @@ -173,6 +174,12 @@ if hasattr(os, name): interpleveldefs[name] = 'interp_posix.' + name + for _name in ["RTLD_LAZY", "RTLD_NOW", "RTLD_GLOBAL", "RTLD_LOCAL", + "RTLD_NODELETE", "RTLD_NOLOAD", "RTLD_DEEPBIND"]: + if getattr(rdynload.cConfig, _name) is not None: + interpleveldefs[_name] = 'space.wrap(%d)' % ( + getattr(rdynload.cConfig, _name),) + # os.py uses this list to build os.supports_dir_fd() and os.supports_fd(). # Fill with e.g. HAVE_FCHDIR, when os.chdir() supports file descriptors. interpleveldefs['_have_functions'] = 'space.newlist([])' diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -1042,6 +1042,12 @@ # just ensure it returns something reasonable assert encoding is None or type(encoding) is str + def test_rtld_constants(self): + # check presence of major RTLD_* constants + self.posix.RTLD_LAZY + self.posix.RTLD_NOW + self.posix.RTLD_GLOBAL + self.posix.RTLD_LOCAL class AppTestEnvironment(object): def setup_class(cls): From noreply at buildbot.pypy.org Sat Jul 26 13:59:59 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 26 Jul 2014 13:59:59 +0200 (CEST) Subject: [pypy-commit] pypy py3.3: merge heads Message-ID: <20140726115959.796191C085C@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: py3.3 Changeset: r72502:2ff4e3b542b6 Date: 2014-07-26 13:59 +0200 http://bitbucket.org/pypy/pypy/changeset/2ff4e3b542b6/ Log: merge heads diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -204,7 +204,7 @@ elif opcode == opcodedesc.BREAK_LOOP.index: next_instr = self.BREAK_LOOP(oparg, next_instr) elif opcode == opcodedesc.CONTINUE_LOOP.index: - next_instr = self.CONTINUE_LOOP(oparg, next_instr) + return self.CONTINUE_LOOP(oparg, next_instr) elif opcode == opcodedesc.FOR_ITER.index: next_instr = self.FOR_ITER(oparg, next_instr) elif opcode == opcodedesc.JUMP_FORWARD.index: @@ -1022,7 +1022,6 @@ raise w_value = space.w_None self.pushvalue(w_value) - return next_instr else: # iter remains on stack, w_retval is value to be yielded. self.pushvalue(w_retval) From noreply at buildbot.pypy.org Sat Jul 26 14:49:54 2014 From: noreply at buildbot.pypy.org (Manuel Jacob) Date: Sat, 26 Jul 2014 14:49:54 +0200 (CEST) Subject: [pypy-commit] pypy improve-docs: hg merge default Message-ID: <20140726124954.EC2AC1C050D@cobra.cs.uni-duesseldorf.de> Author: Manuel Jacob Branch: improve-docs Changeset: r72503:9ca2657fe663 Date: 2014-07-26 14:49 +0200 http://bitbucket.org/pypy/pypy/changeset/9ca2657fe663/ Log: hg merge default diff too long, truncating to 2000 out of 33446 lines diff --git a/.hgtags b/.hgtags --- a/.hgtags +++ b/.hgtags @@ -10,3 +10,7 @@ 20e51c4389ed4469b66bb9d6289ce0ecfc82c4b9 release-2.3.0 0000000000000000000000000000000000000000 release-2.3.0 394146e9bb673514c61f0150ab2013ccf78e8de7 release-2.3 +32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.2=3.1 +32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.3.1 +32f35069a16d819b58c1b6efb17c44e3e53397b2 release-2.2=3.1 +0000000000000000000000000000000000000000 release-2.2=3.1 diff --git a/LICENSE b/LICENSE --- a/LICENSE +++ b/LICENSE @@ -128,6 +128,7 @@ Stian Andreassen Laurence Tratt Wanja Saatkamp + Ivan Sichmann Freitas Gerald Klix Mike Blume Oscar Nierstrasz @@ -212,7 +213,9 @@ Alejandro J. Cura Jacob Oscarson Travis Francis Athougies + Ryan Gonzalez Kristjan Valur Jonsson + Sebastian Pawluś Neil Blakey-Milner anatoly techtonik Lutz Paelike @@ -245,6 +248,7 @@ Michael Hudson-Doyle Anders Sigfridsson Yasir Suhail + rafalgalczynski at gmail.com Floris Bruynooghe Laurens Van Houtven Akira Li @@ -274,6 +278,8 @@ Zooko Wilcox-O Hearn Tomer Chachamu Christopher Groskopf + Asmo Soinio + Stefan Marr jiaaro opassembler.py Antony Lee diff --git a/lib-python/2.7/ctypes/__init__.py b/lib-python/2.7/ctypes/__init__.py --- a/lib-python/2.7/ctypes/__init__.py +++ b/lib-python/2.7/ctypes/__init__.py @@ -389,12 +389,13 @@ func.__name__ = name_or_ordinal return func -class PyDLL(CDLL): - """This class represents the Python library itself. It allows to - access Python API functions. The GIL is not released, and - Python exceptions are handled correctly. - """ - _func_flags_ = _FUNCFLAG_CDECL | _FUNCFLAG_PYTHONAPI +# Not in PyPy +#class PyDLL(CDLL): +# """This class represents the Python library itself. It allows to +# access Python API functions. The GIL is not released, and +# Python exceptions are handled correctly. +# """ +# _func_flags_ = _FUNCFLAG_CDECL | _FUNCFLAG_PYTHONAPI if _os.name in ("nt", "ce"): @@ -447,15 +448,8 @@ return self._dlltype(name) cdll = LibraryLoader(CDLL) -pydll = LibraryLoader(PyDLL) - -if _os.name in ("nt", "ce"): - pythonapi = PyDLL("python dll", None, _sys.dllhandle) -elif _sys.platform == "cygwin": - pythonapi = PyDLL("libpython%d.%d.dll" % _sys.version_info[:2]) -else: - pythonapi = PyDLL(None) - +# not on PyPy +#pydll = LibraryLoader(PyDLL) if _os.name in ("nt", "ce"): windll = LibraryLoader(WinDLL) diff --git a/lib-python/2.7/ctypes/test/test_values.py b/lib-python/2.7/ctypes/test/test_values.py --- a/lib-python/2.7/ctypes/test/test_values.py +++ b/lib-python/2.7/ctypes/test/test_values.py @@ -4,6 +4,7 @@ import unittest from ctypes import * +from ctypes.test import xfail import _ctypes_test @@ -23,7 +24,8 @@ class Win_ValuesTestCase(unittest.TestCase): """This test only works when python itself is a dll/shared library""" - + + @xfail def test_optimizeflag(self): # This test accesses the Py_OptimizeFlag intger, which is # exported by the Python dll. @@ -40,6 +42,7 @@ else: self.assertEqual(opt, 2) + @xfail def test_frozentable(self): # Python exports a PyImport_FrozenModules symbol. This is a # pointer to an array of struct _frozen entries. The end of the @@ -75,6 +78,7 @@ from ctypes import _pointer_type_cache del _pointer_type_cache[struct_frozen] + @xfail def test_undefined(self): self.assertRaises(ValueError, c_int.in_dll, pydll, "Undefined_Symbol") diff --git a/lib-python/2.7/imputil.py b/lib-python/2.7/imputil.py --- a/lib-python/2.7/imputil.py +++ b/lib-python/2.7/imputil.py @@ -422,7 +422,8 @@ saved back to the filesystem for future imports. The source file's modification timestamp must be provided as a Long value. """ - codestring = open(pathname, 'rU').read() + with open(pathname, 'rU') as fp: + codestring = fp.read() if codestring and codestring[-1] != '\n': codestring = codestring + '\n' code = __builtin__.compile(codestring, pathname, 'exec') @@ -603,8 +604,8 @@ self.desc = desc def import_file(self, filename, finfo, fqname): - fp = open(filename, self.desc[1]) - module = imp.load_module(fqname, fp, filename, self.desc) + with open(filename, self.desc[1]) as fp: + module = imp.load_module(fqname, fp, filename, self.desc) module.__file__ = filename return 0, module, { } diff --git a/lib-python/2.7/modulefinder.py b/lib-python/2.7/modulefinder.py --- a/lib-python/2.7/modulefinder.py +++ b/lib-python/2.7/modulefinder.py @@ -109,16 +109,16 @@ def run_script(self, pathname): self.msg(2, "run_script", pathname) - fp = open(pathname, READ_MODE) - stuff = ("", "r", imp.PY_SOURCE) - self.load_module('__main__', fp, pathname, stuff) + with open(pathname, READ_MODE) as fp: + stuff = ("", "r", imp.PY_SOURCE) + self.load_module('__main__', fp, pathname, stuff) def load_file(self, pathname): dir, name = os.path.split(pathname) name, ext = os.path.splitext(name) - fp = open(pathname, READ_MODE) - stuff = (ext, "r", imp.PY_SOURCE) - self.load_module(name, fp, pathname, stuff) + with open(pathname, READ_MODE) as fp: + stuff = (ext, "r", imp.PY_SOURCE) + self.load_module(name, fp, pathname, stuff) def import_hook(self, name, caller=None, fromlist=None, level=-1): self.msg(3, "import_hook", name, caller, fromlist, level) @@ -461,6 +461,8 @@ fp, buf, stuff = self.find_module("__init__", m.__path__) self.load_module(fqname, fp, buf, stuff) self.msgout(2, "load_package ->", m) + if fp: + fp.close() return m def add_module(self, fqname): diff --git a/lib-python/2.7/test/test_argparse.py b/lib-python/2.7/test/test_argparse.py --- a/lib-python/2.7/test/test_argparse.py +++ b/lib-python/2.7/test/test_argparse.py @@ -48,6 +48,9 @@ def tearDown(self): os.chdir(self.old_dir) + import gc + # Force a collection which should close FileType() options + gc.collect() for root, dirs, files in os.walk(self.temp_dir, topdown=False): for name in files: os.chmod(os.path.join(self.temp_dir, name), stat.S_IWRITE) diff --git a/lib-python/2.7/test/test_gdbm.py b/lib-python/2.7/test/test_gdbm.py --- a/lib-python/2.7/test/test_gdbm.py +++ b/lib-python/2.7/test/test_gdbm.py @@ -98,6 +98,17 @@ self.assertTrue(key in self.g) self.assertTrue(self.g.has_key(key)) + def test_unicode_key(self): + key = u'ab' + value = u'cd' + self.g = gdbm.open(filename, 'cf') + self.g[key] = value + self.g.close() + self.g = gdbm.open(filename, 'r') + self.assertEquals(self.g[key], value) + self.assertTrue(key in self.g) + self.assertTrue(self.g.has_key(key)) + def test_main(): run_unittest(TestGdbm) diff --git a/lib-python/2.7/timeit.py b/lib-python/2.7/timeit.py --- a/lib-python/2.7/timeit.py +++ b/lib-python/2.7/timeit.py @@ -55,11 +55,6 @@ import gc import sys import time -try: - import itertools -except ImportError: - # Must be an older Python version (see timeit() below) - itertools = None __all__ = ["Timer"] @@ -81,7 +76,8 @@ def inner(_it, _timer): %(setup)s _t0 = _timer() - for _i in _it: + while _it > 0: + _it -= 1 %(stmt)s _t1 = _timer() return _t1 - _t0 @@ -96,7 +92,8 @@ def inner(_it, _timer, _func=func): setup() _t0 = _timer() - for _i in _it: + while _it > 0: + _it -= 1 _func() _t1 = _timer() return _t1 - _t0 @@ -133,9 +130,19 @@ else: raise ValueError("setup is neither a string nor callable") self.src = src # Save for traceback display - code = compile(src, dummy_src_name, "exec") - exec code in globals(), ns - self.inner = ns["inner"] + def make_inner(): + # PyPy tweak: recompile the source code each time before + # calling inner(). There are situations like Issue #1776 + # where PyPy tries to reuse the JIT code from before, + # but that's not going to work: the first thing the + # function does is the "-s" statement, which may declare + # new classes (here a namedtuple). We end up with + # bridges from the inner loop; more and more of them + # every time we call inner(). + code = compile(src, dummy_src_name, "exec") + exec code in globals(), ns + return ns["inner"] + self.make_inner = make_inner elif hasattr(stmt, '__call__'): self.src = None if isinstance(setup, basestring): @@ -144,7 +151,8 @@ exec _setup in globals(), ns elif not hasattr(setup, '__call__'): raise ValueError("setup is neither a string nor callable") - self.inner = _template_func(setup, stmt) + inner = _template_func(setup, stmt) + self.make_inner = lambda: inner else: raise ValueError("stmt is neither a string nor callable") @@ -185,15 +193,12 @@ to one million. The main statement, the setup statement and the timer function to be used are passed to the constructor. """ - if itertools: - it = itertools.repeat(None, number) - else: - it = [None] * number + inner = self.make_inner() gcold = gc.isenabled() if '__pypy__' not in sys.builtin_module_names: gc.disable() # only do that on CPython try: - timing = self.inner(it, self.timer) + timing = inner(number, self.timer) finally: if gcold: gc.enable() diff --git a/lib-python/2.7/xml/sax/saxutils.py b/lib-python/2.7/xml/sax/saxutils.py --- a/lib-python/2.7/xml/sax/saxutils.py +++ b/lib-python/2.7/xml/sax/saxutils.py @@ -98,13 +98,14 @@ except AttributeError: pass # wrap a binary writer with TextIOWrapper - class UnbufferedTextIOWrapper(io.TextIOWrapper): - def write(self, s): - super(UnbufferedTextIOWrapper, self).write(s) - self.flush() - return UnbufferedTextIOWrapper(buffer, encoding=encoding, + return _UnbufferedTextIOWrapper(buffer, encoding=encoding, errors='xmlcharrefreplace', newline='\n') +# PyPy: moved this class outside the function above +class _UnbufferedTextIOWrapper(io.TextIOWrapper): + def write(self, s): + super(_UnbufferedTextIOWrapper, self).write(s) + self.flush() class XMLGenerator(handler.ContentHandler): diff --git a/lib_pypy/_ctypes/function.py b/lib_pypy/_ctypes/function.py --- a/lib_pypy/_ctypes/function.py +++ b/lib_pypy/_ctypes/function.py @@ -219,6 +219,8 @@ if restype is None: import ctypes restype = ctypes.c_int + if self._argtypes_ is None: + self._argtypes_ = [] self._ptr = self._getfuncptr_fromaddress(self._argtypes_, restype) self._check_argtypes_for_fastpath() return diff --git a/lib_pypy/_curses.py b/lib_pypy/_curses.py --- a/lib_pypy/_curses.py +++ b/lib_pypy/_curses.py @@ -309,11 +309,9 @@ #endif int _m_ispad(WINDOW *win) { -#if defined WINDOW_HAS_FLAGS + // may not have _flags (and possibly _ISPAD), + // but for now let's assume that always has it return (win->_flags & _ISPAD); -#else - return 0; -#endif } void _m_getsyx(int *yx) { diff --git a/lib_pypy/_pypy_testcapi.py b/lib_pypy/_pypy_testcapi.py --- a/lib_pypy/_pypy_testcapi.py +++ b/lib_pypy/_pypy_testcapi.py @@ -13,7 +13,15 @@ k1 = k1.lstrip('0x').rstrip('L') k2 = hex(binascii.crc32(key[1::2]) & 0xffffffff) k2 = k2.lstrip('0').rstrip('L') - output_dir = tempfile.gettempdir() + os.path.sep + 'tmp_%s%s' %(k1, k2) + try: + username = os.environ['USER'] #linux, et al + except KeyError: + try: + username = os.environ['USERNAME'] #windows + except KeyError: + username = os.getuid() + output_dir = tempfile.gettempdir() + os.path.sep + 'tmp_%s_%s%s' % ( + username, k1, k2) if not os.path.exists(output_dir): os.mkdir(output_dir) return output_dir diff --git a/lib_pypy/_tkinter/license.terms b/lib_pypy/_tkinter/license.terms new file mode 100644 --- /dev/null +++ b/lib_pypy/_tkinter/license.terms @@ -0,0 +1,39 @@ +This software is copyrighted by the Regents of the University of +California, Sun Microsystems, Inc., and other parties. The following +terms apply to all files associated with the software unless explicitly +disclaimed in individual files. + +The authors hereby grant permission to use, copy, modify, distribute, +and license this software and its documentation for any purpose, provided +that existing copyright notices are retained in all copies and that this +notice is included verbatim in any distributions. No written agreement, +license, or royalty fee is required for any of the authorized uses. +Modifications to this software may be copyrighted by their authors +and need not follow the licensing terms described here, provided that +the new terms are clearly indicated on the first page of each file where +they apply. + +IN NO EVENT SHALL THE AUTHORS OR DISTRIBUTORS BE LIABLE TO ANY PARTY +FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES +ARISING OUT OF THE USE OF THIS SOFTWARE, ITS DOCUMENTATION, OR ANY +DERIVATIVES THEREOF, EVEN IF THE AUTHORS HAVE BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +THE AUTHORS AND DISTRIBUTORS SPECIFICALLY DISCLAIM ANY WARRANTIES, +INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, AND NON-INFRINGEMENT. THIS SOFTWARE +IS PROVIDED ON AN "AS IS" BASIS, AND THE AUTHORS AND DISTRIBUTORS HAVE +NO OBLIGATION TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR +MODIFICATIONS. + +GOVERNMENT USE: If you are acquiring this software on behalf of the +U.S. government, the Government shall have only "Restricted Rights" +in the software and related documentation as defined in the Federal +Acquisition Regulations (FARs) in Clause 52.227.19 (c) (2). If you +are acquiring the software on behalf of the Department of Defense, the +software shall be classified as "Commercial Computer Software" and the +Government shall have only "Restricted Rights" as defined in Clause +252.227-7013 (c) (1) of DFARs. Notwithstanding the foregoing, the +authors grant the U.S. Government and others acting in its behalf +permission to use and distribute the software in accordance with the +terms specified in this license. diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,5 +4,5 @@ from .api import FFI, CDefError, FFIError from .ffiplatform import VerificationError, VerificationMissing -__version__ = "0.8.2" -__version_info__ = (0, 8, 2) +__version__ = "0.8.6" +__version_info__ = (0, 8, 6) diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py --- a/lib_pypy/cffi/api.py +++ b/lib_pypy/cffi/api.py @@ -55,8 +55,7 @@ # _cffi_backend.so compiled. import _cffi_backend as backend from . import __version__ - assert (backend.__version__ == __version__ or - backend.__version__ == __version__[:3]) + assert backend.__version__ == __version__ # (If you insist you can also try to pass the option # 'backend=backend_ctypes.CTypesBackend()', but don't # rely on it! It's probably not going to work well.) @@ -443,6 +442,10 @@ for enumname, enumval in zip(tp.enumerators, tp.enumvalues): if enumname not in library.__dict__: library.__dict__[enumname] = enumval + for key, val in ffi._parser._int_constants.items(): + if key not in library.__dict__: + library.__dict__[key] = val + copied_enums.append(True) if name in library.__dict__: return diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py --- a/lib_pypy/cffi/cparser.py +++ b/lib_pypy/cffi/cparser.py @@ -24,6 +24,7 @@ _r_partial_array = re.compile(r"\[\s*\.\.\.\s*\]") _r_words = re.compile(r"\w+|\S") _parser_cache = None +_r_int_literal = re.compile(r"^0?x?[0-9a-f]+u?l?$", re.IGNORECASE) def _get_parser(): global _parser_cache @@ -99,6 +100,7 @@ self._structnode2type = weakref.WeakKeyDictionary() self._override = False self._packed = False + self._int_constants = {} def _parse(self, csource): csource, macros = _preprocess(csource) @@ -128,9 +130,10 @@ finally: if lock is not None: lock.release() - return ast, macros + # csource will be used to find buggy source text + return ast, macros, csource - def convert_pycparser_error(self, e, csource): + def _convert_pycparser_error(self, e, csource): # xxx look for ":NUM:" at the start of str(e) and try to interpret # it as a line number line = None @@ -142,6 +145,12 @@ csourcelines = csource.splitlines() if 1 <= linenum <= len(csourcelines): line = csourcelines[linenum-1] + return line + + def convert_pycparser_error(self, e, csource): + line = self._convert_pycparser_error(e, csource) + + msg = str(e) if line: msg = 'cannot parse "%s"\n%s' % (line.strip(), msg) else: @@ -160,14 +169,9 @@ self._packed = prev_packed def _internal_parse(self, csource): - ast, macros = self._parse(csource) + ast, macros, csource = self._parse(csource) # add the macros - for key, value in macros.items(): - value = value.strip() - if value != '...': - raise api.CDefError('only supports the syntax "#define ' - '%s ..." for now (literally)' % key) - self._declare('macro ' + key, value) + self._process_macros(macros) # find the first "__dotdotdot__" and use that as a separator # between the repeated typedefs and the real csource iterator = iter(ast.ext) @@ -175,27 +179,61 @@ if decl.name == '__dotdotdot__': break # - for decl in iterator: - if isinstance(decl, pycparser.c_ast.Decl): - self._parse_decl(decl) - elif isinstance(decl, pycparser.c_ast.Typedef): - if not decl.name: - raise api.CDefError("typedef does not declare any name", - decl) - if (isinstance(decl.type.type, pycparser.c_ast.IdentifierType) - and decl.type.type.names == ['__dotdotdot__']): - realtype = model.unknown_type(decl.name) - elif (isinstance(decl.type, pycparser.c_ast.PtrDecl) and - isinstance(decl.type.type, pycparser.c_ast.TypeDecl) and - isinstance(decl.type.type.type, - pycparser.c_ast.IdentifierType) and - decl.type.type.type.names == ['__dotdotdot__']): - realtype = model.unknown_ptr_type(decl.name) + try: + for decl in iterator: + if isinstance(decl, pycparser.c_ast.Decl): + self._parse_decl(decl) + elif isinstance(decl, pycparser.c_ast.Typedef): + if not decl.name: + raise api.CDefError("typedef does not declare any name", + decl) + if (isinstance(decl.type.type, pycparser.c_ast.IdentifierType) + and decl.type.type.names == ['__dotdotdot__']): + realtype = model.unknown_type(decl.name) + elif (isinstance(decl.type, pycparser.c_ast.PtrDecl) and + isinstance(decl.type.type, pycparser.c_ast.TypeDecl) and + isinstance(decl.type.type.type, + pycparser.c_ast.IdentifierType) and + decl.type.type.type.names == ['__dotdotdot__']): + realtype = model.unknown_ptr_type(decl.name) + else: + realtype = self._get_type(decl.type, name=decl.name) + self._declare('typedef ' + decl.name, realtype) else: - realtype = self._get_type(decl.type, name=decl.name) - self._declare('typedef ' + decl.name, realtype) + raise api.CDefError("unrecognized construct", decl) + except api.FFIError as e: + msg = self._convert_pycparser_error(e, csource) + if msg: + e.args = (e.args[0] + "\n *** Err: %s" % msg,) + raise + + def _add_constants(self, key, val): + if key in self._int_constants: + raise api.FFIError( + "multiple declarations of constant: %s" % (key,)) + self._int_constants[key] = val + + def _process_macros(self, macros): + for key, value in macros.items(): + value = value.strip() + match = _r_int_literal.search(value) + if match is not None: + int_str = match.group(0).lower().rstrip("ul") + + # "010" is not valid oct in py3 + if (int_str.startswith("0") and + int_str != "0" and + not int_str.startswith("0x")): + int_str = "0o" + int_str[1:] + + pyvalue = int(int_str, 0) + self._add_constants(key, pyvalue) + elif value == '...': + self._declare('macro ' + key, value) else: - raise api.CDefError("unrecognized construct", decl) + raise api.CDefError('only supports the syntax "#define ' + '%s ..." (literally) or "#define ' + '%s 0x1FF" for now' % (key, key)) def _parse_decl(self, decl): node = decl.type @@ -227,7 +265,7 @@ self._declare('variable ' + decl.name, tp) def parse_type(self, cdecl): - ast, macros = self._parse('void __dummy(\n%s\n);' % cdecl) + ast, macros = self._parse('void __dummy(\n%s\n);' % cdecl)[:2] assert not macros exprnode = ast.ext[-1].type.args.params[0] if isinstance(exprnode, pycparser.c_ast.ID): @@ -306,7 +344,8 @@ if ident == 'void': return model.void_type if ident == '__dotdotdot__': - raise api.FFIError('bad usage of "..."') + raise api.FFIError(':%d: bad usage of "..."' % + typenode.coord.line) return resolve_common_type(ident) # if isinstance(type, pycparser.c_ast.Struct): @@ -333,7 +372,8 @@ return self._get_struct_union_enum_type('union', typenode, name, nested=True) # - raise api.FFIError("bad or unsupported type declaration") + raise api.FFIError(":%d: bad or unsupported type declaration" % + typenode.coord.line) def _parse_function_type(self, typenode, funcname=None): params = list(getattr(typenode.args, 'params', [])) @@ -499,6 +539,10 @@ if (isinstance(exprnode, pycparser.c_ast.UnaryOp) and exprnode.op == '-'): return -self._parse_constant(exprnode.expr) + # load previously defined int constant + if (isinstance(exprnode, pycparser.c_ast.ID) and + exprnode.name in self._int_constants): + return self._int_constants[exprnode.name] # if partial_length_ok: if (isinstance(exprnode, pycparser.c_ast.ID) and @@ -506,8 +550,8 @@ self._partial_length = True return '...' # - raise api.FFIError("unsupported expression: expected a " - "simple numeric constant") + raise api.FFIError(":%d: unsupported expression: expected a " + "simple numeric constant" % exprnode.coord.line) def _build_enum_type(self, explicit_name, decls): if decls is not None: @@ -522,6 +566,7 @@ if enum.value is not None: nextenumvalue = self._parse_constant(enum.value) enumvalues.append(nextenumvalue) + self._add_constants(enum.name, nextenumvalue) nextenumvalue += 1 enumvalues = tuple(enumvalues) tp = model.EnumType(explicit_name, enumerators, enumvalues) @@ -535,3 +580,5 @@ kind = name.split(' ', 1)[0] if kind in ('typedef', 'struct', 'union', 'enum'): self._declare(name, tp) + for k, v in other._int_constants.items(): + self._add_constants(k, v) diff --git a/lib_pypy/cffi/ffiplatform.py b/lib_pypy/cffi/ffiplatform.py --- a/lib_pypy/cffi/ffiplatform.py +++ b/lib_pypy/cffi/ffiplatform.py @@ -38,6 +38,7 @@ import distutils.errors # dist = Distribution({'ext_modules': [ext]}) + dist.parse_config_files() options = dist.get_option_dict('build_ext') options['force'] = ('ffiplatform', True) options['build_lib'] = ('ffiplatform', tmpdir) diff --git a/lib_pypy/cffi/vengine_cpy.py b/lib_pypy/cffi/vengine_cpy.py --- a/lib_pypy/cffi/vengine_cpy.py +++ b/lib_pypy/cffi/vengine_cpy.py @@ -89,43 +89,54 @@ # by generate_cpy_function_method(). prnt('static PyMethodDef _cffi_methods[] = {') self._generate("method") - prnt(' {"_cffi_setup", _cffi_setup, METH_VARARGS},') - prnt(' {NULL, NULL} /* Sentinel */') + prnt(' {"_cffi_setup", _cffi_setup, METH_VARARGS, NULL},') + prnt(' {NULL, NULL, 0, NULL} /* Sentinel */') prnt('};') prnt() # # standard init. modname = self.verifier.get_module_name() - if sys.version_info >= (3,): - prnt('static struct PyModuleDef _cffi_module_def = {') - prnt(' PyModuleDef_HEAD_INIT,') - prnt(' "%s",' % modname) - prnt(' NULL,') - prnt(' -1,') - prnt(' _cffi_methods,') - prnt(' NULL, NULL, NULL, NULL') - prnt('};') - prnt() - initname = 'PyInit_%s' % modname - createmod = 'PyModule_Create(&_cffi_module_def)' - errorcase = 'return NULL' - finalreturn = 'return lib' - else: - initname = 'init%s' % modname - createmod = 'Py_InitModule("%s", _cffi_methods)' % modname - errorcase = 'return' - finalreturn = 'return' + constants = self._chained_list_constants[False] + prnt('#if PY_MAJOR_VERSION >= 3') + prnt() + prnt('static struct PyModuleDef _cffi_module_def = {') + prnt(' PyModuleDef_HEAD_INIT,') + prnt(' "%s",' % modname) + prnt(' NULL,') + prnt(' -1,') + prnt(' _cffi_methods,') + prnt(' NULL, NULL, NULL, NULL') + prnt('};') + prnt() prnt('PyMODINIT_FUNC') - prnt('%s(void)' % initname) + prnt('PyInit_%s(void)' % modname) prnt('{') prnt(' PyObject *lib;') - prnt(' lib = %s;' % createmod) - prnt(' if (lib == NULL || %s < 0)' % ( - self._chained_list_constants[False],)) - prnt(' %s;' % errorcase) - prnt(' _cffi_init();') - prnt(' %s;' % finalreturn) + prnt(' lib = PyModule_Create(&_cffi_module_def);') + prnt(' if (lib == NULL)') + prnt(' return NULL;') + prnt(' if (%s < 0 || _cffi_init() < 0) {' % (constants,)) + prnt(' Py_DECREF(lib);') + prnt(' return NULL;') + prnt(' }') + prnt(' return lib;') prnt('}') + prnt() + prnt('#else') + prnt() + prnt('PyMODINIT_FUNC') + prnt('init%s(void)' % modname) + prnt('{') + prnt(' PyObject *lib;') + prnt(' lib = Py_InitModule("%s", _cffi_methods);' % modname) + prnt(' if (lib == NULL)') + prnt(' return;') + prnt(' if (%s < 0 || _cffi_init() < 0)' % (constants,)) + prnt(' return;') + prnt(' return;') + prnt('}') + prnt() + prnt('#endif') def load_library(self): # XXX review all usages of 'self' here! @@ -394,7 +405,7 @@ meth = 'METH_O' else: meth = 'METH_VARARGS' - self._prnt(' {"%s", _cffi_f_%s, %s},' % (name, name, meth)) + self._prnt(' {"%s", _cffi_f_%s, %s, NULL},' % (name, name, meth)) _loading_cpy_function = _loaded_noop @@ -481,8 +492,8 @@ if tp.fldnames is None: return # nothing to do with opaque structs layoutfuncname = '_cffi_layout_%s_%s' % (prefix, name) - self._prnt(' {"%s", %s, METH_NOARGS},' % (layoutfuncname, - layoutfuncname)) + self._prnt(' {"%s", %s, METH_NOARGS, NULL},' % (layoutfuncname, + layoutfuncname)) def _loading_struct_or_union(self, tp, prefix, name, module): if tp.fldnames is None: @@ -589,13 +600,7 @@ 'variable type'),)) assert delayed else: - prnt(' if (LONG_MIN <= (%s) && (%s) <= LONG_MAX)' % (name, name)) - prnt(' o = PyInt_FromLong((long)(%s));' % (name,)) - prnt(' else if ((%s) <= 0)' % (name,)) - prnt(' o = PyLong_FromLongLong((long long)(%s));' % (name,)) - prnt(' else') - prnt(' o = PyLong_FromUnsignedLongLong(' - '(unsigned long long)(%s));' % (name,)) + prnt(' o = _cffi_from_c_int_const(%s);' % name) prnt(' if (o == NULL)') prnt(' return -1;') if size_too: @@ -632,13 +637,18 @@ # ---------- # enums + def _enum_funcname(self, prefix, name): + # "$enum_$1" => "___D_enum____D_1" + name = name.replace('$', '___D_') + return '_cffi_e_%s_%s' % (prefix, name) + def _generate_cpy_enum_decl(self, tp, name, prefix='enum'): if tp.partial: for enumerator in tp.enumerators: self._generate_cpy_const(True, enumerator, delayed=False) return # - funcname = '_cffi_e_%s_%s' % (prefix, name) + funcname = self._enum_funcname(prefix, name) prnt = self._prnt prnt('static int %s(PyObject *lib)' % funcname) prnt('{') @@ -760,17 +770,30 @@ #include #include -#ifdef MS_WIN32 -#include /* for alloca() */ -typedef __int8 int8_t; -typedef __int16 int16_t; -typedef __int32 int32_t; -typedef __int64 int64_t; -typedef unsigned __int8 uint8_t; -typedef unsigned __int16 uint16_t; -typedef unsigned __int32 uint32_t; -typedef unsigned __int64 uint64_t; -typedef unsigned char _Bool; +/* this block of #ifs should be kept exactly identical between + c/_cffi_backend.c, cffi/vengine_cpy.py, cffi/vengine_gen.py */ +#if defined(_MSC_VER) +# include /* for alloca() */ +# if _MSC_VER < 1600 /* MSVC < 2010 */ + typedef __int8 int8_t; + typedef __int16 int16_t; + typedef __int32 int32_t; + typedef __int64 int64_t; + typedef unsigned __int8 uint8_t; + typedef unsigned __int16 uint16_t; + typedef unsigned __int32 uint32_t; + typedef unsigned __int64 uint64_t; +# else +# include +# endif +# if _MSC_VER < 1800 /* MSVC < 2013 */ + typedef unsigned char _Bool; +# endif +#else +# include +# if (defined (__SVR4) && defined (__sun)) || defined(_AIX) +# include +# endif #endif #if PY_MAJOR_VERSION < 3 @@ -795,6 +818,15 @@ #define _cffi_to_c_double PyFloat_AsDouble #define _cffi_to_c_float PyFloat_AsDouble +#define _cffi_from_c_int_const(x) \ + (((x) > 0) ? \ + ((unsigned long long)(x) <= (unsigned long long)LONG_MAX) ? \ + PyInt_FromLong((long)(x)) : \ + PyLong_FromUnsignedLongLong((unsigned long long)(x)) : \ + ((long long)(x) >= (long long)LONG_MIN) ? \ + PyInt_FromLong((long)(x)) : \ + PyLong_FromLongLong((long long)(x))) + #define _cffi_from_c_int(x, type) \ (((type)-1) > 0 ? /* unsigned */ \ (sizeof(type) < sizeof(long) ? PyInt_FromLong(x) : \ @@ -804,14 +836,14 @@ PyLong_FromLongLong(x))) #define _cffi_to_c_int(o, type) \ - (sizeof(type) == 1 ? (((type)-1) > 0 ? _cffi_to_c_u8(o) \ - : _cffi_to_c_i8(o)) : \ - sizeof(type) == 2 ? (((type)-1) > 0 ? _cffi_to_c_u16(o) \ - : _cffi_to_c_i16(o)) : \ - sizeof(type) == 4 ? (((type)-1) > 0 ? _cffi_to_c_u32(o) \ - : _cffi_to_c_i32(o)) : \ - sizeof(type) == 8 ? (((type)-1) > 0 ? _cffi_to_c_u64(o) \ - : _cffi_to_c_i64(o)) : \ + (sizeof(type) == 1 ? (((type)-1) > 0 ? (type)_cffi_to_c_u8(o) \ + : (type)_cffi_to_c_i8(o)) : \ + sizeof(type) == 2 ? (((type)-1) > 0 ? (type)_cffi_to_c_u16(o) \ + : (type)_cffi_to_c_i16(o)) : \ + sizeof(type) == 4 ? (((type)-1) > 0 ? (type)_cffi_to_c_u32(o) \ + : (type)_cffi_to_c_i32(o)) : \ + sizeof(type) == 8 ? (((type)-1) > 0 ? (type)_cffi_to_c_u64(o) \ + : (type)_cffi_to_c_i64(o)) : \ (Py_FatalError("unsupported size for type " #type), 0)) #define _cffi_to_c_i8 \ @@ -885,25 +917,32 @@ return PyBool_FromLong(was_alive); } -static void _cffi_init(void) +static int _cffi_init(void) { - PyObject *module = PyImport_ImportModule("_cffi_backend"); - PyObject *c_api_object; + PyObject *module, *c_api_object = NULL; + module = PyImport_ImportModule("_cffi_backend"); if (module == NULL) - return; + goto failure; c_api_object = PyObject_GetAttrString(module, "_C_API"); if (c_api_object == NULL) - return; + goto failure; if (!PyCapsule_CheckExact(c_api_object)) { - Py_DECREF(c_api_object); PyErr_SetNone(PyExc_ImportError); - return; + goto failure; } memcpy(_cffi_exports, PyCapsule_GetPointer(c_api_object, "cffi"), _CFFI_NUM_EXPORTS * sizeof(void *)); + + Py_DECREF(module); Py_DECREF(c_api_object); + return 0; + + failure: + Py_XDECREF(module); + Py_XDECREF(c_api_object); + return -1; } #define _cffi_type(num) ((CTypeDescrObject *)PyList_GET_ITEM(_cffi_types, num)) diff --git a/lib_pypy/cffi/vengine_gen.py b/lib_pypy/cffi/vengine_gen.py --- a/lib_pypy/cffi/vengine_gen.py +++ b/lib_pypy/cffi/vengine_gen.py @@ -249,10 +249,10 @@ prnt(' /* %s */' % str(e)) # cannot verify it, ignore prnt('}') self.export_symbols.append(layoutfuncname) - prnt('ssize_t %s(ssize_t i)' % (layoutfuncname,)) + prnt('intptr_t %s(intptr_t i)' % (layoutfuncname,)) prnt('{') prnt(' struct _cffi_aligncheck { char x; %s y; };' % cname) - prnt(' static ssize_t nums[] = {') + prnt(' static intptr_t nums[] = {') prnt(' sizeof(%s),' % cname) prnt(' offsetof(struct _cffi_aligncheck, y),') for fname, ftype, fbitsize in tp.enumfields(): @@ -276,7 +276,7 @@ return # nothing to do with opaque structs layoutfuncname = '_cffi_layout_%s_%s' % (prefix, name) # - BFunc = self.ffi._typeof_locked("ssize_t(*)(ssize_t)")[0] + BFunc = self.ffi._typeof_locked("intptr_t(*)(intptr_t)")[0] function = module.load_function(BFunc, layoutfuncname) layout = [] num = 0 @@ -410,13 +410,18 @@ # ---------- # enums + def _enum_funcname(self, prefix, name): + # "$enum_$1" => "___D_enum____D_1" + name = name.replace('$', '___D_') + return '_cffi_e_%s_%s' % (prefix, name) + def _generate_gen_enum_decl(self, tp, name, prefix='enum'): if tp.partial: for enumerator in tp.enumerators: self._generate_gen_const(True, enumerator) return # - funcname = '_cffi_e_%s_%s' % (prefix, name) + funcname = self._enum_funcname(prefix, name) self.export_symbols.append(funcname) prnt = self._prnt prnt('int %s(char *out_error)' % funcname) @@ -430,14 +435,14 @@ enumerator, enumerator, enumvalue)) prnt(' char buf[64];') prnt(' if ((%s) < 0)' % enumerator) - prnt(' snprintf(buf, 63, "%%ld", (long)(%s));' % enumerator) + prnt(' sprintf(buf, "%%ld", (long)(%s));' % enumerator) prnt(' else') - prnt(' snprintf(buf, 63, "%%lu", (unsigned long)(%s));' % + prnt(' sprintf(buf, "%%lu", (unsigned long)(%s));' % enumerator) - prnt(' snprintf(out_error, 255,' + prnt(' sprintf(out_error,' ' "%s has the real value %s, not %s",') prnt(' "%s", buf, "%d");' % ( - enumerator, enumvalue)) + enumerator[:100], enumvalue)) prnt(' return -1;') prnt(' }') prnt(' return 0;') @@ -453,7 +458,7 @@ else: BType = self.ffi._typeof_locked("char[]")[0] BFunc = self.ffi._typeof_locked("int(*)(char*)")[0] - funcname = '_cffi_e_%s_%s' % (prefix, name) + funcname = self._enum_funcname(prefix, name) function = module.load_function(BFunc, funcname) p = self.ffi.new(BType, 256) if function(p) < 0: @@ -547,20 +552,29 @@ #include #include /* XXX for ssize_t on some platforms */ -#ifdef _WIN32 -# include -# define snprintf _snprintf -typedef __int8 int8_t; -typedef __int16 int16_t; -typedef __int32 int32_t; -typedef __int64 int64_t; -typedef unsigned __int8 uint8_t; -typedef unsigned __int16 uint16_t; -typedef unsigned __int32 uint32_t; -typedef unsigned __int64 uint64_t; -typedef SSIZE_T ssize_t; -typedef unsigned char _Bool; +/* this block of #ifs should be kept exactly identical between + c/_cffi_backend.c, cffi/vengine_cpy.py, cffi/vengine_gen.py */ +#if defined(_MSC_VER) +# include /* for alloca() */ +# if _MSC_VER < 1600 /* MSVC < 2010 */ + typedef __int8 int8_t; + typedef __int16 int16_t; + typedef __int32 int32_t; + typedef __int64 int64_t; + typedef unsigned __int8 uint8_t; + typedef unsigned __int16 uint16_t; + typedef unsigned __int32 uint32_t; + typedef unsigned __int64 uint64_t; +# else +# include +# endif +# if _MSC_VER < 1800 /* MSVC < 2013 */ + typedef unsigned char _Bool; +# endif #else -# include +# include +# if (defined (__SVR4) && defined (__sun)) || defined(_AIX) +# include +# endif #endif ''' diff --git a/lib_pypy/gdbm.py b/lib_pypy/gdbm.py --- a/lib_pypy/gdbm.py +++ b/lib_pypy/gdbm.py @@ -50,6 +50,8 @@ pass def _fromstr(key): + if isinstance(key, unicode): + key = key.encode("ascii") if not isinstance(key, str): raise TypeError("gdbm mappings have string indices only") return {'dptr': ffi.new("char[]", key), 'dsize': len(key)} @@ -71,8 +73,8 @@ def _raise_from_errno(self): if ffi.errno: - raise error(os.strerror(ffi.errno)) - raise error(lib.gdbm_strerror(lib.gdbm_errno)) + raise error(ffi.errno, os.strerror(ffi.errno)) + raise error(lib.gdbm_errno, lib.gdbm_strerror(lib.gdbm_errno)) def __len__(self): if self.size < 0: @@ -141,7 +143,7 @@ def _check_closed(self): if not self.ll_dbm: - raise error("GDBM object has already been closed") + raise error(0, "GDBM object has already been closed") __del__ = close @@ -159,7 +161,7 @@ elif flags[0] == 'n': iflags = lib.GDBM_NEWDB else: - raise error("First flag must be one of 'r', 'w', 'c' or 'n'") + raise error(0, "First flag must be one of 'r', 'w', 'c' or 'n'") for flag in flags[1:]: if flag == 'f': iflags |= lib.GDBM_FAST @@ -168,7 +170,7 @@ elif flag == 'u': iflags |= lib.GDBM_NOLOCK else: - raise error("Flag '%s' not supported" % flag) + raise error(0, "Flag '%s' not supported" % flag) return gdbm(filename, iflags, mode) open_flags = "rwcnfsu" diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py --- a/pypy/config/pypyoption.py +++ b/pypy/config/pypyoption.py @@ -113,7 +113,7 @@ try: for name in modlist: __import__(name) - except (ImportError, CompilationError, py.test.skip.Exception), e: + except (ImportError, CompilationError, py.test.skip.Exception) as e: errcls = e.__class__.__name__ raise Exception( "The module %r is disabled\n" % (modname,) + diff --git a/pypy/doc/coding-guide.rst b/pypy/doc/coding-guide.rst --- a/pypy/doc/coding-guide.rst +++ b/pypy/doc/coding-guide.rst @@ -101,7 +101,7 @@ while True: try: w_key = space.next(w_iter) - except OperationError, e: + except OperationError as e: if not e.match(space, space.w_StopIteration): raise # re-raise other app-level exceptions break @@ -271,7 +271,7 @@ try: ... - except OperationError, e: + except OperationError as e: if not e.match(space, space.w_XxxError): raise ... diff --git a/pypy/doc/conf.py b/pypy/doc/conf.py --- a/pypy/doc/conf.py +++ b/pypy/doc/conf.py @@ -25,13 +25,13 @@ on_rtd = os.environ.get('READTHEDOCS', None) == 'True' if not on_rtd: # only import and set the theme if we're building docs locally - try: - import sphinx_rtd_theme - html_theme = 'sphinx_rtd_theme' - html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] - except ImportError: - print('sphinx_rtd_theme is not installed') - html_theme = 'default' + try: + import sphinx_rtd_theme + html_theme = 'sphinx_rtd_theme' + html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] + except ImportError: + print('sphinx_rtd_theme is not installed') + html_theme = 'default' # otherwise, readthedocs.org uses their theme by default, so no need to specify it diff --git a/pypy/doc/config/translation.log.txt b/pypy/doc/config/translation.log.txt --- a/pypy/doc/config/translation.log.txt +++ b/pypy/doc/config/translation.log.txt @@ -2,4 +2,4 @@ These must be enabled by setting the PYPYLOG environment variable. The exact set of features supported by PYPYLOG is described in -pypy/translation/c/src/debug_print.h. +rpython/translator/c/src/debug_print.h. diff --git a/pypy/doc/contributor.rst b/pypy/doc/contributor.rst --- a/pypy/doc/contributor.rst +++ b/pypy/doc/contributor.rst @@ -98,6 +98,7 @@ Stian Andreassen Laurence Tratt Wanja Saatkamp + Ivan Sichmann Freitas Gerald Klix Mike Blume Oscar Nierstrasz @@ -182,7 +183,9 @@ Alejandro J. Cura Jacob Oscarson Travis Francis Athougies + Ryan Gonzalez Kristjan Valur Jonsson + Sebastian Pawluś Neil Blakey-Milner anatoly techtonik Lutz Paelike @@ -215,6 +218,7 @@ Michael Hudson-Doyle Anders Sigfridsson Yasir Suhail + rafalgalczynski at gmail.com Floris Bruynooghe Laurens Van Houtven Akira Li @@ -244,6 +248,8 @@ Zooko Wilcox-O Hearn Tomer Chachamu Christopher Groskopf + Asmo Soinio + Stefan Marr jiaaro opassembler.py Antony Lee diff --git a/pypy/doc/ctypes-implementation.rst b/pypy/doc/ctypes-implementation.rst --- a/pypy/doc/ctypes-implementation.rst +++ b/pypy/doc/ctypes-implementation.rst @@ -74,13 +74,11 @@ Here is a list of the limitations and missing features of the current implementation: -* ``ctypes.pythonapi`` lets you access the CPython C API emulation layer - of PyPy, at your own risks and without doing anything sensible about - the GIL. Since PyPy 2.3, these functions are also named with an extra - "Py", for example ``PyPyInt_FromLong()``. Basically, don't use this, - but it might more or less work in simple cases if you do. (Obviously, - assuming the PyObject pointers you get have any particular fields in - any particular order is just going to crash.) +* ``ctypes.pythonapi`` is missing. In previous versions, it was present + and redirected to the `cpyext` C API emulation layer, but our + implementation did not do anything sensible about the GIL and the + functions were named with an extra "Py", for example + ``PyPyInt_FromLong()``. It was removed for being unhelpful. * We copy Python strings instead of having pointers to raw buffers diff --git a/pypy/doc/extradoc.rst b/pypy/doc/extradoc.rst --- a/pypy/doc/extradoc.rst +++ b/pypy/doc/extradoc.rst @@ -6,6 +6,9 @@ *Articles about PyPy published so far, most recent first:* (bibtex_ file) +* `A Way Forward in Parallelising Dynamic Languages`_, + R. Meier, A. Rigo + * `Runtime Feedback in a Meta-Tracing JIT for Efficient Dynamic Languages`_, C.F. Bolz, A. Cuni, M. Fijalkowski, M. Leuschel, S. Pedroni, A. Rigo @@ -69,6 +72,7 @@ .. _bibtex: https://bitbucket.org/pypy/extradoc/raw/tip/talk/bibtex.bib +.. _A Way Forward in Parallelising Dynamic Languages: https://bitbucket.org/pypy/extradoc/raw/extradoc/talk/icooolps2014/position-paper.pdf .. _Runtime Feedback in a Meta-Tracing JIT for Efficient Dynamic Languages: https://bitbucket.org/pypy/extradoc/raw/extradoc/talk/icooolps2011/jit-hints.pdf .. _Allocation Removal by Partial Evaluation in a Tracing JIT: https://bitbucket.org/pypy/extradoc/raw/extradoc/talk/pepm2011/bolz-allocation-removal.pdf .. _Towards a Jitting VM for Prolog Execution: http://www.stups.uni-duesseldorf.de/mediawiki/images/a/a7/Pub-BoLeSch2010.pdf @@ -91,6 +95,11 @@ Talks and Presentations ----------------------- +*This part is no longer updated.* The complete list is here__ (in +alphabetical order). + +.. __: https://bitbucket.org/pypy/extradoc/src/extradoc/talk/ + Talks in 2010 ~~~~~~~~~~~~~ diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst --- a/pypy/doc/faq.rst +++ b/pypy/doc/faq.rst @@ -150,14 +150,18 @@ You might be interested in our `benchmarking site`_ and our :doc:`jit documentation `. -Note that the JIT has a very high warm-up cost, meaning that the -programs are slow at the beginning. If you want to compare the timings -with CPython, even relatively simple programs need to run *at least* one -second, preferrably at least a few seconds. Large, complicated programs -need even more time to warm-up the JIT. +`Your tests are not a benchmark`_: tests tend to be slow under PyPy +because they run exactly once; if they are good tests, they exercise +various corner cases in your code. This is a bad case for JIT +compilers. Note also that our JIT has a very high warm-up cost, meaning +that any program is slow at the beginning. If you want to compare the +timings with CPython, even relatively simple programs need to run *at +least* one second, preferrably at least a few seconds. Large, +complicated programs need even more time to warm-up the JIT. .. _benchmarking site: http://speed.pypy.org +.. _your tests are not a benchmark: http://alexgaynor.net/2013/jul/15/your-tests-are-not-benchmark/ Couldn't the JIT dump and reload already-compiled machine code? --------------------------------------------------------------- diff --git a/pypy/doc/jit-hooks.rst b/pypy/doc/jit-hooks.rst --- a/pypy/doc/jit-hooks.rst +++ b/pypy/doc/jit-hooks.rst @@ -34,7 +34,7 @@ aborted due to some reason. The hook will be invoked with the siagnture: - ``hook(jitdriver_name, greenkey, reason)`` + ``hook(jitdriver_name, greenkey, reason, oplist)`` Reason is a string, the meaning of other arguments is the same as attributes on JitLoopInfo object diff --git a/pypy/doc/man/pypy.1.rst b/pypy/doc/man/pypy.1.rst --- a/pypy/doc/man/pypy.1.rst +++ b/pypy/doc/man/pypy.1.rst @@ -95,13 +95,12 @@ ``PYPYLOG`` If set to a non-empty value, enable logging, the format is: - *fname* + *fname* or *+fname* logging for profiling: includes all ``debug_start``/``debug_stop`` but not any nested ``debug_print``. *fname* can be ``-`` to log to *stderr*. - Note that using a : in fname is a bad idea, Windows - users, beware. + The *+fname* form can be used if there is a *:* in fname ``:``\ *fname* Full logging, including ``debug_print``. diff --git a/pypy/doc/objspace-proxies.rst b/pypy/doc/objspace-proxies.rst --- a/pypy/doc/objspace-proxies.rst +++ b/pypy/doc/objspace-proxies.rst @@ -24,6 +24,16 @@ Transparent Proxies ------------------- +.. warning:: + + This is a feature that was tried experimentally long ago, and we + found no really good use cases. The basic functionality is still + there, but we don't recommend using it. Some of the examples below + might not work any more (e.g. you can't tproxy a list object any + more). The rest can be done by hacking in standard Python. If + anyone is interested in working on tproxy again, he is welcome, but + we don't regard this as an interesting extension. + PyPy's Transparent Proxies allow routing of operations on objects to a callable. Application-level code can customize objects without interfering with the type system - ``type(proxied_list) is list`` holds true diff --git a/pypy/doc/release-2.3.1.rst b/pypy/doc/release-2.3.1.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-2.3.1.rst @@ -0,0 +1,81 @@ +================================================= +PyPy 2.3.1 - Terrestrial Arthropod Trap Revisited +================================================= + +We're pleased to announce PyPy 2.3.1, a feature-and-bugfix improvement over our +recent release last month. + +This release contains several bugfixes and enhancements. + +You can download the PyPy 2.3.1 release here: + + http://pypy.org/download.html + +We would like to thank our donors for the continued support of the PyPy +project, and for those who donate to our three sub-projects. +We've shown quite a bit of progress +but we're slowly running out of funds. +Please consider donating more, or even better convince your employer to donate, +so we can finish those projects! The three sub-projects are: + +* `Py3k`_ (supporting Python 3.x): the release PyPy3 2.3 is imminent. + +* `STM`_ (software transactional memory): a preview will be released very soon, + once we fix a few bugs + +* `NumPy`_ which requires installation of our fork of upstream numpy, available `on bitbucket`_ + +.. _`Py3k`: http://pypy.org/py3donate.html +.. _`STM`: http://pypy.org/tmdonate2.html +.. _`NumPy`: http://pypy.org/numpydonate.html +.. _`on bitbucket`: https://www.bitbucket.org/pypy/numpy + +What is PyPy? +============= + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7. It's fast (`pypy 2.3 and cpython 2.7.x`_ performance comparison; +note that cpython's speed has not changed since 2.7.2) +due to its integrated tracing JIT compiler. + +This release supports x86 machines running Linux 32/64, Mac OS X 64, Windows, +and OpenBSD, +as well as newer ARM hardware (ARMv6 or ARMv7, with VFPv3) running Linux. + +While we support 32 bit python on Windows, work on the native Windows 64 +bit python is still stalling, we would welcome a volunteer +to `handle that`_. + +.. _`pypy 2.3 and cpython 2.7.x`: http://speed.pypy.org +.. _`handle that`: http://doc.pypy.org/en/latest/windows.html#what-is-missing-for-a-full-64-bit-translation + +Highlights +========== + +Issues with the 2.3 release were resolved after being reported by users to +our new issue tracker at https://bitbucket.org/pypy/pypy/issues or on IRC at +#pypy. Here is a summary of the user-facing changes; +for more information see `whats-new`_: + +* The built-in ``struct`` module was renamed to ``_struct``, solving issues + with IDLE and other modules. + +* Support for compilation with gcc-4.9 + +* A rewrite of packaging.py which produces our downloadable packages to + modernize command line argument handling and to document third-party + contributions in our LICENSE file + +* A CFFI-based version of the gdbm module is now included in our downloads + +* Many issues were resolved_ since the 2.3 release on May 8 + +.. _`whats-new`: http://doc.pypy.org/en/latest/whatsnew-2.3.1.html +.. _resolved: https://bitbucket.org/pypy/pypy/issues?status=resolved +Please try it out and let us know what you think. We especially welcome +success stories, we know you are using PyPy, please tell us about it! + +Cheers + +The PyPy Team + diff --git a/pypy/doc/release-pypy3-2.3.1.rst b/pypy/doc/release-pypy3-2.3.1.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/release-pypy3-2.3.1.rst @@ -0,0 +1,69 @@ +===================== +PyPy3 2.3.1 - Fulcrum +===================== + +We're pleased to announce the first stable release of PyPy3. PyPy3 +targets Python 3 (3.2.5) compatibility. + +We would like to thank all of the people who donated_ to the `py3k proposal`_ +for supporting the work that went into this. + +You can download the PyPy3 2.3.1 release here: + + http://pypy.org/download.html#pypy3-2-3-1 + +Highlights +========== + +* The first stable release of PyPy3: support for Python 3! + +* The stdlib has been updated to Python 3.2.5 + +* Additional support for the u'unicode' syntax (`PEP 414`_) from Python 3.3 + +* Updates from the default branch, such as incremental GC and various JIT + improvements + +* Resolved some notable JIT performance regressions from PyPy2: + + - Re-enabled the previously disabled collection (list/dict/set) strategies + + - Resolved performance of iteration over range objects + + - Resolved handling of Python 3's exception __context__ unnecessarily forcing + frame object overhead + +.. _`PEP 414`: http://legacy.python.org/dev/peps/pep-0414/ + +What is PyPy? +============== + +PyPy is a very compliant Python interpreter, almost a drop-in replacement for +CPython 2.7.6 or 3.2.5. It's fast due to its integrated tracing JIT compiler. + +This release supports x86 machines running Linux 32/64, Mac OS X 64, Windows, +and OpenBSD, +as well as newer ARM hardware (ARMv6 or ARMv7, with VFPv3) running Linux. + +While we support 32 bit python on Windows, work on the native Windows 64 +bit python is still stalling, we would welcome a volunteer +to `handle that`_. + +.. _`handle that`: http://doc.pypy.org/en/latest/windows.html#what-is-missing-for-a-full-64-bit-translation + +How to use PyPy? +================= + +We suggest using PyPy from a `virtualenv`_. Once you have a virtualenv +installed, you can follow instructions from `pypy documentation`_ on how +to proceed. This document also covers other `installation schemes`_. + +.. _donated: http://morepypy.blogspot.com/2012/01/py3k-and-numpy-first-stage-thanks-to.html +.. _`py3k proposal`: http://pypy.org/py3donate.html +.. _`pypy documentation`: http://doc.pypy.org/en/latest/getting-started.html#installing-using-virtualenv +.. _`virtualenv`: http://www.virtualenv.org/en/latest/ +.. _`installation schemes`: http://doc.pypy.org/en/latest/getting-started.html#installing-pypy + + +Cheers, +the PyPy team diff --git a/pypy/doc/stm.rst b/pypy/doc/stm.rst --- a/pypy/doc/stm.rst +++ b/pypy/doc/stm.rst @@ -28,7 +28,8 @@ Introduction ============ -``pypy-stm`` is a variant of the regular PyPy interpreter. With caveats_ +``pypy-stm`` is a variant of the regular PyPy interpreter. (This +version supports Python 2.7; see below for `Python 3`_.) With caveats_ listed below, it should be in theory within 20%-50% slower than a regular PyPy, comparing the JIT version in both cases (but see below!). It is called @@ -92,9 +93,9 @@ We're busy fixing them as we find them; feel free to `report bugs`_. * It runs with an overhead as low as 20% on examples like "richards". - There are also other examples with higher overheads --up to 10x for - "translate.py"-- which we are still trying to understand. One suspect - is our partial GC implementation, see below. + There are also other examples with higher overheads --currently up to + 2x for "translate.py"-- which we are still trying to understand. + One suspect is our partial GC implementation, see below. * Currently limited to 1.5 GB of RAM (this is just a parameter in `core.h`__). Memory overflows are not correctly handled; they cause @@ -111,9 +112,8 @@ * The GC is new; although clearly inspired by PyPy's regular GC, it misses a number of optimizations for now. Programs allocating large - numbers of small objects that don't immediately die, as well as - programs that modify large lists or dicts, suffer from these missing - optimizations. + numbers of small objects that don't immediately die (surely a common + situation) suffer from these missing optimizations. * The GC has no support for destructors: the ``__del__`` method is never called (including on file objects, which won't be closed for you). @@ -138,6 +138,25 @@ +Python 3 +======== + +In this document I describe "pypy-stm", which is based on PyPy's Python +2.7 interpreter. Supporting Python 3 should take about half an +afternoon of work. Obviously, what I *don't* mean is that by tomorrow +you can have a finished and polished "pypy3-stm" product. General py3k +work is still missing; and general stm work is also still missing. But +they are rather independent from each other, as usual in PyPy. The +required afternoon of work will certainly be done one of these days now +that the internal interfaces seem to stabilize. + +The same is true for other languages implemented in the RPython +framework, although the amount of work to put there might vary, because +the STM framework within RPython is currently targeting the PyPy +interpreter and other ones might have slightly different needs. + + + User Guide ========== @@ -490,8 +509,6 @@ The last two lines are special; they are an internal marker read by ``transactional_memory.print_abort_info()``. -These statistics are not printed out for the main thread, for now. - Reference to implementation details ----------------------------------- diff --git a/pypy/doc/whatsnew-2.3.1.rst b/pypy/doc/whatsnew-2.3.1.rst --- a/pypy/doc/whatsnew-2.3.1.rst +++ b/pypy/doc/whatsnew-2.3.1.rst @@ -9,5 +9,16 @@ Support compilation with gcc-4.9 -Fixes for issues #1769, #1764, #1762, #1752 +Added support for the stdlib gdbm module via cffi +Annotator cleanups + +.. branch: release-2.3.x + +.. branch: unify-call-ops + +.. branch packaging +Use argparse for packaging.py, and add third-party components to LICENSE file. +Also mention that gdbm is GPL. +Do not crash the packaging process on failure in CFFI or license-building, +rather complete the build step and return -1. diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -3,11 +3,50 @@ ======================= .. this is a revision shortly after release-2.3.x -.. startrev: b2cc67adbaad +.. startrev: ca9b7cf02cf4 -Added support for the stdlib gdbm module via cffi +.. branch: fix-bytearray-complexity +Bytearray operations no longer copy the bytearray unnecessarily -Annotator cleanups +Added support for ``__getitem__``, ``__setitem__``, ``__getslice__``, +``__setslice__``, and ``__len__`` to RPython -.. branch: release-2.3.x +.. branch: stringbuilder2-perf +Give the StringBuilder a more flexible internal structure, with a +chained list of strings instead of just one string. This make it +more efficient when building large strings, e.g. with cStringIO(). +Also, use systematically jit.conditional_call() instead of regular +branches. This lets the JIT make more linear code, at the cost of +forcing a bit more data (to be passed as arguments to +conditional_calls). I would expect the net result to be a slight +slow-down on some simple benchmarks and a speed-up on bigger +programs. + +.. branch: ec-threadlocal +Change the executioncontext's lookup to be done by reading a thread- +local variable (which is implemented in C using '__thread' if +possible, and pthread_getspecific() otherwise). On Linux x86 and +x86-64, the JIT backend has a special optimization that lets it emit +directly a single MOV from a %gs- or %fs-based address. It seems +actually to give a good boost in performance. + +.. branch: fast-gil +A faster way to handle the GIL, particularly in JIT code. The GIL is +now a composite of two concepts: a global number (it's just set from +1 to 0 and back around CALL_RELEASE_GIL), and a real mutex. If there +are threads waiting to acquire the GIL, one of them is actively +checking the global number every 0.1 ms to 1 ms. Overall, JIT loops +full of external function calls now run a bit faster (if no thread was +started yet), or a *lot* faster (if threads were started already). + +.. branch: jit-get-errno +Optimize the errno handling in the JIT, notably around external +function calls. Linux-only. + +.. branch: disable_pythonapi +Remove non-functioning ctypes.pyhonapi and ctypes.PyDLL, document this +incompatibility with cpython. Recast sys.dllhandle to an int. + +.. branch: scalar-operations +Fix performance regression on ufunc(, ) in numpy. diff --git a/pypy/doc/whatsnew-pypy3-2.3.1.rst b/pypy/doc/whatsnew-pypy3-2.3.1.rst new file mode 100644 --- /dev/null +++ b/pypy/doc/whatsnew-pypy3-2.3.1.rst @@ -0,0 +1,6 @@ +========================= +What's new in PyPy3 2.3.1 +========================= + +.. this is a revision shortly after pypy3-release-2.3.x +.. startrev: 0137d8e6657d diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py --- a/pypy/goal/targetpypystandalone.py +++ b/pypy/goal/targetpypystandalone.py @@ -30,8 +30,6 @@ if w_dict is not None: # for tests w_entry_point = space.getitem(w_dict, space.wrap('entry_point')) w_run_toplevel = space.getitem(w_dict, space.wrap('run_toplevel')) - w_call_finish_gateway = space.wrap(gateway.interp2app(call_finish)) - w_call_startup_gateway = space.wrap(gateway.interp2app(call_startup)) withjit = space.config.objspace.usemodules.pypyjit def entry_point(argv): @@ -53,7 +51,7 @@ argv = argv[:1] + argv[3:] try: try: - space.call_function(w_run_toplevel, w_call_startup_gateway) + space.startup() w_executable = space.wrap(argv[0]) w_argv = space.newlist([space.wrap(s) for s in argv[1:]]) w_exitcode = space.call_function(w_entry_point, w_executable, w_argv) @@ -69,7 +67,7 @@ return 1 finally: try: - space.call_function(w_run_toplevel, w_call_finish_gateway) + space.finish() except OperationError, e: debug("OperationError:") debug(" operror-type: " + e.w_type.getname(space)) @@ -184,11 +182,6 @@ 'pypy_thread_attach': pypy_thread_attach, 'pypy_setup_home': pypy_setup_home} -def call_finish(space): - space.finish() - -def call_startup(space): - space.startup() # _____ Define and setup target ___ diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -395,6 +395,7 @@ def startup(self): # To be called before using the space + self.threadlocals.enter_thread(self) # Initialize already imported builtin modules from pypy.interpreter.module import Module @@ -639,30 +640,36 @@ """NOT_RPYTHON: Abstract method that should put some minimal content into the w_builtins.""" - @jit.loop_invariant def getexecutioncontext(self): "Return what we consider to be the active execution context." # Important: the annotator must not see a prebuilt ExecutionContext: # you should not see frames while you translate # so we make sure that the threadlocals never *have* an # ExecutionContext during translation. - if self.config.translating and not we_are_translated(): - assert self.threadlocals.getvalue() is None, ( - "threadlocals got an ExecutionContext during translation!") - try: - return self._ec_during_translation - except AttributeError: - ec = self.createexecutioncontext() - self._ec_during_translation = ec + if not we_are_translated(): + if self.config.translating: + assert self.threadlocals.get_ec() is None, ( + "threadlocals got an ExecutionContext during translation!") + try: + return self._ec_during_translation + except AttributeError: + ec = self.createexecutioncontext() + self._ec_during_translation = ec + return ec + else: + ec = self.threadlocals.get_ec() + if ec is None: + self.threadlocals.enter_thread(self) + ec = self.threadlocals.get_ec() return ec - # normal case follows. The 'thread' module installs a real - # thread-local object in self.threadlocals, so this builds - # and caches a new ec in each thread. - ec = self.threadlocals.getvalue() - if ec is None: - ec = self.createexecutioncontext() - self.threadlocals.setvalue(ec) - return ec + else: + # translated case follows. self.threadlocals is either from + # 'pypy.interpreter.miscutils' or 'pypy.module.thread.threadlocals'. + # the result is assumed to be non-null: enter_thread() was called + # by space.startup(). + ec = self.threadlocals.get_ec() + assert ec is not None + return ec def _freeze_(self): return True @@ -963,6 +970,13 @@ """ return self.unpackiterable(w_iterable, expected_length) + def listview_no_unpack(self, w_iterable): + """ Same as listview() if cheap. If 'w_iterable' is something like + a generator, for example, then return None instead. + May return None anyway. + """ + return None + def listview_bytes(self, w_list): """ Return a list of unwrapped strings out of a list of strings. If the argument is not a list or does not contain only strings, return None. @@ -1487,9 +1501,7 @@ return buf.as_str() def str_or_None_w(self, w_obj): - if self.is_w(w_obj, self.w_None): - return None - return self.str_w(w_obj) + return None if self.is_none(w_obj) else self.str_w(w_obj) def str_w(self, w_obj): return w_obj.str_w(self) diff --git a/pypy/interpreter/executioncontext.py b/pypy/interpreter/executioncontext.py --- a/pypy/interpreter/executioncontext.py +++ b/pypy/interpreter/executioncontext.py @@ -496,6 +496,13 @@ """ +class UserDelCallback(object): + def __init__(self, w_obj, callback, descrname): + self.w_obj = w_obj + self.callback = callback + self.descrname = descrname + self.next = None + class UserDelAction(AsyncAction): """An action that invokes all pending app-level __del__() method. This is done as an action instead of immediately when the @@ -506,12 +513,18 @@ def __init__(self, space): AsyncAction.__init__(self, space) - self.dying_objects = [] + self.dying_objects = None + self.dying_objects_last = None self.finalizers_lock_count = 0 self.enabled_at_app_level = True def register_callback(self, w_obj, callback, descrname): - self.dying_objects.append((w_obj, callback, descrname)) + cb = UserDelCallback(w_obj, callback, descrname) + if self.dying_objects_last is None: + self.dying_objects = cb + else: + self.dying_objects_last.next = cb + self.dying_objects_last = cb self.fire() def perform(self, executioncontext, frame): @@ -525,13 +538,33 @@ # avoid too deep recursions of the kind of __del__ being called # while in the middle of another __del__ call. pending = self.dying_objects - self.dying_objects = [] + self.dying_objects = None + self.dying_objects_last = None space = self.space - for i in range(len(pending)): - w_obj, callback, descrname = pending[i] - pending[i] = (None, None, None) + while pending is not None: try: - callback(w_obj) + pending.callback(pending.w_obj) except OperationError, e: - e.write_unraisable(space, descrname, w_obj) + e.write_unraisable(space, pending.descrname, pending.w_obj) e.clear(space) # break up reference cycles + pending = pending.next + # + # Note: 'dying_objects' used to be just a regular list instead + # of a chained list. This was the cause of "leaks" if we have a + # program that constantly creates new objects with finalizers. + # Here is why: say 'dying_objects' is a long list, and there + # are n instances in it. Then we spend some time in this + # function, possibly triggering more GCs, but keeping the list + # of length n alive. Then the list is suddenly freed at the + # end, and we return to the user program. At this point the + # GC limit is still very high, because just before, there was + # a list of length n alive. Assume that the program continues + # to allocate a lot of instances with finalizers. The high GC + # limit means that it could allocate a lot of instances before + # reaching it --- possibly more than n. So the whole procedure + # repeats with higher and higher values of n. + # + # This does not occur in the current implementation because + # there is no list of length n: if n is large, then the GC + # will run several times while walking the list, but it will + # see lower and lower memory usage, with no lower bound of n. diff --git a/pypy/interpreter/gateway.py b/pypy/interpreter/gateway.py --- a/pypy/interpreter/gateway.py +++ b/pypy/interpreter/gateway.py @@ -895,7 +895,7 @@ "use unwrap_spec(...=WrappedDefault(default))" % ( self._code.identifier, name, defaultval)) defs_w.append(None) - else: + elif name != '__args__' and name != 'args_w': defs_w.append(space.wrap(defaultval)) if self._code._unwrap_spec: UNDEFINED = object() diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py --- a/pypy/interpreter/generator.py +++ b/pypy/interpreter/generator.py @@ -61,6 +61,13 @@ return self.send_ex(w_arg) def send_ex(self, w_arg, operr=None): + pycode = self.pycode + if jit.we_are_jitted() and should_not_inline(pycode): + generatorentry_driver.jit_merge_point(gen=self, w_arg=w_arg, + operr=operr, pycode=pycode) + return self._send_ex(w_arg, operr) + + def _send_ex(self, w_arg, operr): space = self.space if self.running: raise OperationError(space.w_ValueError, @@ -72,8 +79,7 @@ if operr is None: operr = OperationError(space.w_StopIteration, space.w_None) raise operr - # XXX it's not clear that last_instr should be promoted at all - # but as long as it is necessary for call_assembler, let's do it early + last_instr = jit.promote(frame.last_instr) if last_instr == -1: if w_arg and not space.is_w(w_arg, space.w_None): @@ -214,3 +220,38 @@ "interrupting generator of ") break block = block.previous + + + +def get_printable_location_genentry(bytecode): + return '%s ' % (bytecode.get_repr(),) +generatorentry_driver = jit.JitDriver(greens=['pycode'], + reds=['gen', 'w_arg', 'operr'], + get_printable_location = + get_printable_location_genentry, + name='generatorentry') + +from pypy.tool.stdlib_opcode import HAVE_ARGUMENT, opmap +YIELD_VALUE = opmap['YIELD_VALUE'] + + at jit.elidable_promote() +def should_not_inline(pycode): + # Should not inline generators with more than one "yield", + # as an approximative fix (see issue #1782). There are cases + # where it slows things down; for example calls to a simple + # generator that just produces a few simple values with a few + # consecutive "yield" statements. It fixes the near-infinite + # slow-down in issue #1782, though... + count_yields = 0 + code = pycode.co_code + n = len(code) + i = 0 + while i < n: + c = code[i] + op = ord(c) + if op == YIELD_VALUE: + count_yields += 1 + i += 1 + if op >= HAVE_ARGUMENT: + i += 2 + return count_yields >= 2 diff --git a/pypy/interpreter/miscutils.py b/pypy/interpreter/miscutils.py --- a/pypy/interpreter/miscutils.py +++ b/pypy/interpreter/miscutils.py @@ -11,11 +11,14 @@ """ _value = None - def getvalue(self): + def get_ec(self): return self._value - def setvalue(self, value): - self._value = value + def enter_thread(self, space): + self._value = space.createexecutioncontext() + + def try_enter_thread(self, space): + return False def signals_enabled(self): return True diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py --- a/pypy/interpreter/pyframe.py +++ b/pypy/interpreter/pyframe.py @@ -511,10 +511,10 @@ for i in range(min(len(varnames), self.getcode().co_nlocals)): name = varnames[i] w_value = self.locals_stack_w[i] - w_name = self.space.wrap(name) if w_value is not None: - self.space.setitem(self.w_locals, w_name, w_value) + self.space.setitem_str(self.w_locals, name, w_value) else: + w_name = self.space.wrap(name) try: self.space.delitem(self.w_locals, w_name) except OperationError as e: @@ -534,8 +534,7 @@ except ValueError: pass else: - w_name = self.space.wrap(name) - self.space.setitem(self.w_locals, w_name, w_value) + self.space.setitem_str(self.w_locals, name, w_value) @jit.unroll_safe From noreply at buildbot.pypy.org Sat Jul 26 14:52:48 2014 From: noreply at buildbot.pypy.org (Valentina Mukhamedzhanova) Date: Sat, 26 Jul 2014 14:52:48 +0200 (CEST) Subject: [pypy-commit] pypy py3.3: Fix zlib's test_decompress_eof. Message-ID: <20140726125248.4E4111C0695@cobra.cs.uni-duesseldorf.de> Author: Valentina Mukhamedzhanova Branch: py3.3 Changeset: r72504:c22750fbe7ad Date: 2014-07-26 14:46 +0200 http://bitbucket.org/pypy/pypy/changeset/c22750fbe7ad/ Log: Fix zlib's test_decompress_eof. diff --git a/pypy/module/zlib/interp_zlib.py b/pypy/module/zlib/interp_zlib.py --- a/pypy/module/zlib/interp_zlib.py +++ b/pypy/module/zlib/interp_zlib.py @@ -1,7 +1,7 @@ import sys from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.baseobjspace import W_Root -from pypy.interpreter.typedef import TypeDef, interp_attrproperty_bytes +from pypy.interpreter.typedef import TypeDef, interp_attrproperty_bytes, interp_attrproperty from pypy.interpreter.error import OperationError, oefmt from rpython.rlib.rarithmetic import intmask, r_uint from rpython.rlib.objectmodel import keepalive_until_here @@ -231,6 +231,7 @@ ZLibObject.__init__(self, space) self.unused_data = '' self.unconsumed_tail = '' + self.eof = False try: self.stream = rzlib.inflateInit(wbits) except rzlib.RZlibError, e: @@ -238,7 +239,7 @@ except ValueError: raise OperationError(space.w_ValueError, space.wrap("Invalid initialization option")) - + def __del__(self): """Automatically free the resources used by the stream.""" if self.stream: @@ -280,6 +281,7 @@ raise zlib_error(space, e.msg) string, finished, unused_len = result + self.eof = finished self._save_unconsumed_input(data, finished, unused_len) return space.wrapbytes(string) @@ -327,6 +329,7 @@ flush = interp2app(Decompress.flush), unused_data = interp_attrproperty_bytes('unused_data', Decompress), unconsumed_tail = interp_attrproperty_bytes('unconsumed_tail', Decompress), + eof = interp_attrproperty('eof', Decompress), __doc__ = """decompressobj([wbits]) -- Return a decompressor object. Optional arg wbits is the window buffer size. diff --git a/pypy/module/zlib/test/test_zlib.py b/pypy/module/zlib/test/test_zlib.py --- a/pypy/module/zlib/test/test_zlib.py +++ b/pypy/module/zlib/test/test_zlib.py @@ -166,6 +166,18 @@ dco = zlib.decompressobj() assert dco.flush() == b"" + def test_decompress_eof(self): + import zlib + x = b'x\x9cK\xcb\xcf\x07\x00\x02\x82\x01E' # 'foo' + dco = zlib.decompressobj() + assert dco.eof == False + dco.decompress(x[:-5]) + assert dco.eof == False + dco.decompress(x[-5:]) + assert dco.eof == True + dco.flush() + assert dco.eof == True + def test_decompress_incomplete_stream(self): import zlib # This is 'foo', deflated From noreply at buildbot.pypy.org Sat Jul 26 14:52:49 2014 From: noreply at buildbot.pypy.org (Valentina Mukhamedzhanova) Date: Sat, 26 Jul 2014 14:52:49 +0200 (CEST) Subject: [pypy-commit] pypy py3.3: Merge heads. Message-ID: <20140726125249.CD30A1C0695@cobra.cs.uni-duesseldorf.de> Author: Valentina Mukhamedzhanova Branch: py3.3 Changeset: r72505:4bba41190d41 Date: 2014-07-26 14:48 +0200 http://bitbucket.org/pypy/pypy/changeset/4bba41190d41/ Log: Merge heads. diff --git a/lib-python/3/json/__init__.py b/lib-python/3/json/__init__.py --- a/lib-python/3/json/__init__.py +++ b/lib-python/3/json/__init__.py @@ -104,6 +104,12 @@ __author__ = 'Bob Ippolito ' +try: + # PyPy speedup, the interface is different than CPython's _json + import _pypyjson +except ImportError: + _pypyjson = None + from .decoder import JSONDecoder from .encoder import JSONEncoder @@ -313,7 +319,7 @@ if (cls is None and object_hook is None and parse_int is None and parse_float is None and parse_constant is None and object_pairs_hook is None and not kw): - return _default_decoder.decode(s) + return _pypyjson.loads(s) if _pypyjson else _default_decoder.decode(s) if cls is None: cls = JSONDecoder if object_hook is not None: diff --git a/lib-python/3/test/test_csv.py b/lib-python/3/test/test_csv.py --- a/lib-python/3/test/test_csv.py +++ b/lib-python/3/test/test_csv.py @@ -766,8 +766,9 @@ mydialect.quotechar = "''" with self.assertRaises(csv.Error) as cm: mydialect() + # NOTE: Patched exception message since cpython uses bad grammar (cpython issue22076) self.assertEqual(str(cm.exception), - '"quotechar" must be an 1-character string') + '"quotechar" must be a 1-character string') mydialect.quotechar = 4 with self.assertRaises(csv.Error) as cm: @@ -789,14 +790,16 @@ mydialect.delimiter = ":::" with self.assertRaises(csv.Error) as cm: mydialect() + # NOTE: Patched exception message since cpython uses bad grammar (cpython issue22076) self.assertEqual(str(cm.exception), - '"delimiter" must be an 1-character string') + '"delimiter" must be a 1-character string') mydialect.delimiter = "" with self.assertRaises(csv.Error) as cm: mydialect() + # NOTE: Patched exception message since cpython uses bad grammar (cpython issue22076) self.assertEqual(str(cm.exception), - '"delimiter" must be an 1-character string') + '"delimiter" must be a 1-character string') mydialect.delimiter = b"," with self.assertRaises(csv.Error) as cm: diff --git a/lib-python/stdlib-version.txt b/lib-python/stdlib-version.txt --- a/lib-python/stdlib-version.txt +++ b/lib-python/stdlib-version.txt @@ -6,4 +6,4 @@ 2.7:: 3a1db0d2747e (2.7) v2.7.6 3:: - cef745775b65 (3.2) v3.2.5 + 62cf4e77f785 (3.3) v3.3.5 diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -204,7 +204,7 @@ elif opcode == opcodedesc.BREAK_LOOP.index: next_instr = self.BREAK_LOOP(oparg, next_instr) elif opcode == opcodedesc.CONTINUE_LOOP.index: - next_instr = self.CONTINUE_LOOP(oparg, next_instr) + return self.CONTINUE_LOOP(oparg, next_instr) elif opcode == opcodedesc.FOR_ITER.index: next_instr = self.FOR_ITER(oparg, next_instr) elif opcode == opcodedesc.JUMP_FORWARD.index: @@ -1022,7 +1022,6 @@ raise w_value = space.w_None self.pushvalue(w_value) - return next_instr else: # iter remains on stack, w_retval is value to be yielded. self.pushvalue(w_retval) diff --git a/pypy/interpreter/test/test_generator.py b/pypy/interpreter/test/test_generator.py --- a/pypy/interpreter/test/test_generator.py +++ b/pypy/interpreter/test/test_generator.py @@ -307,13 +307,13 @@ w_co = space.appexec([], '''(): def g(x): yield x + 5 - return g.func_code + return g.__code__ ''') assert should_not_inline(w_co) == False w_co = space.appexec([], '''(): def g(x): yield x + 5 yield x + 6 - return g.func_code + return g.__code__ ''') assert should_not_inline(w_co) == True diff --git a/pypy/module/__builtin__/app_inspect.py b/pypy/module/__builtin__/app_inspect.py --- a/pypy/module/__builtin__/app_inspect.py +++ b/pypy/module/__builtin__/app_inspect.py @@ -53,8 +53,7 @@ if dir_meth is not None: result = dir_meth() if not isinstance(result, list): - raise TypeError("__dir__() must return a list, not %r" % ( - type(result),)) + result = list(result) # Will throw TypeError if not iterable result.sort() return result elif isinstance(obj, types.ModuleType): diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py --- a/pypy/module/__builtin__/functional.py +++ b/pypy/module/__builtin__/functional.py @@ -482,7 +482,7 @@ def descr_hash(self, space): if space.eq_w(self.w_length, space.wrap(0)): w_tup = space.newtuple([self.w_length, space.w_None, space.w_None]) - elif space.eq_w(self.w_length, space.wrap(0)): + elif space.eq_w(self.w_length, space.wrap(1)): w_tup = space.newtuple([self.w_length, self.w_start, space.w_None]) else: w_tup = space.newtuple([self.w_length, self.w_start, self.w_step]) diff --git a/pypy/module/__builtin__/test/test_construct_singletons.py b/pypy/module/__builtin__/test/test_construct_singletons.py new file mode 100644 --- /dev/null +++ b/pypy/module/__builtin__/test/test_construct_singletons.py @@ -0,0 +1,7 @@ +class AppTestConstructSingletons: + + def test_construct_singletons(self): + none_type = type(None) + assert none_type() is None + raises(TypeError, none_type, 1, 2) + raises(TypeError, none_type, a=1, b=2) diff --git a/pypy/module/__builtin__/test/test_dir.py b/pypy/module/__builtin__/test/test_dir.py new file mode 100644 --- /dev/null +++ b/pypy/module/__builtin__/test/test_dir.py @@ -0,0 +1,26 @@ +class AppTestDir: + + def test_dir_obj__dir__tuple(self): + """If __dir__ method returns a tuple, cpython3 converts it to list.""" + class Foo(object): + def __dir__(self): + return ("b", "c", "a") + res = dir(Foo()) + assert isinstance(res, list) + assert res == ["a", "b", "c"] + + def test_dir_obj__dir__genexp(self): + """Generator expression is also converted to list by cpython3.""" + class Foo(object): + def __dir__(self): + return (i for i in ["b", "c", "a"]) + res = dir(Foo()) + assert isinstance(res, list) + assert res == ["a", "b", "c"] + + def test_dir_obj__dir__noniter(self): + """If result of __dir__ is not iterable, it's an error.""" + class Foo(object): + def __dir__(self): + return 42 + raises(TypeError, dir, Foo()) diff --git a/pypy/module/__builtin__/test/test_functional.py b/pypy/module/__builtin__/test/test_functional.py --- a/pypy/module/__builtin__/test/test_functional.py +++ b/pypy/module/__builtin__/test/test_functional.py @@ -484,7 +484,7 @@ for a in test_ranges: for b in test_ranges: if a == b: - assert (hash(a), hash(b)) + assert hash(a) == hash(b) # Ranges are unequal to other types (even sequence types) assert (range(0) == ()) is False diff --git a/pypy/module/_csv/interp_csv.py b/pypy/module/_csv/interp_csv.py --- a/pypy/module/_csv/interp_csv.py +++ b/pypy/module/_csv/interp_csv.py @@ -34,10 +34,15 @@ return default return space.int_w(w_src) -def _get_str(space, w_src, default): +def _get_str(space, w_src, default, attrname): if w_src is None: return default - return space.unicode_w(w_src) + try: + return space.unicode_w(w_src) + except OperationError as e: + if e.match(space, space.w_TypeError): + raise oefmt(space.w_TypeError, '"%s" must be a string', attrname) + raise def _get_char(space, w_src, default, name): if w_src is None: @@ -91,7 +96,7 @@ dialect.delimiter = _get_char(space, w_delimiter, u',', 'delimiter') dialect.doublequote = _get_bool(space, w_doublequote, True) dialect.escapechar = _get_char(space, w_escapechar, u'\0', 'escapechar') - dialect.lineterminator = _get_str(space, w_lineterminator, u'\r\n') + dialect.lineterminator = _get_str(space, w_lineterminator, u'\r\n', 'lineterminator') dialect.quotechar = _get_char(space, w_quotechar, u'"', 'quotechar') tmp_quoting = _get_int(space, w_quoting, QUOTE_MINIMAL) dialect.skipinitialspace = _get_bool(space, w_skipinitialspace, False) diff --git a/pypy/module/_csv/test/test_dialect.py b/pypy/module/_csv/test/test_dialect.py --- a/pypy/module/_csv/test/test_dialect.py +++ b/pypy/module/_csv/test/test_dialect.py @@ -67,6 +67,9 @@ kwargs = {name: value} raises(TypeError, _csv.register_dialect, 'foo1', **kwargs) + exc_info = raises(TypeError, _csv.register_dialect, 'foo1', lineterminator=4) + assert exc_info.value.args[0] == '"lineterminator" must be a string' + def test_bool_arg(self): # boolean arguments take *any* object and use its truth-value import _csv diff --git a/pypy/module/_io/interp_io.py b/pypy/module/_io/interp_io.py --- a/pypy/module/_io/interp_io.py +++ b/pypy/module/_io/interp_io.py @@ -4,7 +4,6 @@ from pypy.interpreter.gateway import interp2app, unwrap_spec from pypy.interpreter.typedef import ( TypeDef, interp_attrproperty, generic_new_descr) -from pypy.module.exceptions.interp_exceptions import W_IOError from pypy.module._io.interp_fileio import W_FileIO from pypy.module._io.interp_textio import W_TextIOWrapper from rpython.rtyper.module.ll_os_stat import STAT_FIELD_TYPES diff --git a/pypy/module/_pypyjson/interp_decoder.py b/pypy/module/_pypyjson/interp_decoder.py --- a/pypy/module/_pypyjson/interp_decoder.py +++ b/pypy/module/_pypyjson/interp_decoder.py @@ -56,6 +56,7 @@ self.end_ptr = lltype.malloc(rffi.CCHARPP.TO, 1, flavor='raw') self.pos = 0 self.last_type = TYPE_UNKNOWN + self.memo = {} def close(self): rffi.free_charp(self.ll_chars) @@ -261,6 +262,8 @@ w_name = self.decode_any(i) if self.last_type != TYPE_STRING: self._raise("Key name must be string for object starting at char %d", start) + w_name = self.memo.setdefault(self.space.unicode_w(w_name), w_name) + i = self.skip_whitespace(self.pos) ch = self.ll_chars[i] if ch != ':': diff --git a/pypy/module/_pypyjson/test/test__pypyjson.py b/pypy/module/_pypyjson/test/test__pypyjson.py --- a/pypy/module/_pypyjson/test/test__pypyjson.py +++ b/pypy/module/_pypyjson/test/test__pypyjson.py @@ -187,4 +187,12 @@ import _pypyjson # http://json.org/JSON_checker/test/fail25.json s = '["\ttab\tcharacter\tin\tstring\t"]' - raises(ValueError, "_pypyjson.loads(s)") \ No newline at end of file + raises(ValueError, "_pypyjson.loads(s)") + + def test_keys_reuse(self): + import _pypyjson + s = '[{"a_key": 1, "b_\xe9": 2}, {"a_key": 3, "b_\xe9": 4}]' + rval = _pypyjson.loads(s) + (a, b), (c, d) = sorted(rval[0]), sorted(rval[1]) + assert a is c + assert b is d diff --git a/pypy/module/array/interp_array.py b/pypy/module/array/interp_array.py --- a/pypy/module/array/interp_array.py +++ b/pypy/module/array/interp_array.py @@ -244,7 +244,7 @@ """ size = self.len if size == 0: - return space.wrap('') + return space.wrapbytes('') cbuf = self._charbuf_start() s = rffi.charpsize2str(cbuf, size * self.itemsize) self._charbuf_stop() diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -454,6 +454,10 @@ # PyExc_NameError, PyExc_MemoryError, PyExc_RuntimeError, # PyExc_UnicodeEncodeError, PyExc_UnicodeDecodeError, ... for exc_name in exceptions.Module.interpleveldefs.keys(): + if exc_name in ('EnvironmentError', 'IOError'): + # FIXME: aliases of OSError cause a clash of names via + # export_struct + continue GLOBALS['PyExc_' + exc_name] = ( 'PyTypeObject*', 'space.gettypeobject(interp_exceptions.W_%s.typedef)'% (exc_name, )) diff --git a/pypy/module/cpyext/include/patchlevel.h b/pypy/module/cpyext/include/patchlevel.h --- a/pypy/module/cpyext/include/patchlevel.h +++ b/pypy/module/cpyext/include/patchlevel.h @@ -20,13 +20,13 @@ /* Version parsed out into numeric values */ #define PY_MAJOR_VERSION 3 -#define PY_MINOR_VERSION 2 +#define PY_MINOR_VERSION 3 #define PY_MICRO_VERSION 5 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_FINAL #define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "3.2.5" +#define PY_VERSION "3.3.5" /* PyPy version as a string */ #define PYPY_VERSION "2.4.0-alpha0" diff --git a/pypy/module/cpyext/test/test_exception.py b/pypy/module/cpyext/test/test_exception.py --- a/pypy/module/cpyext/test/test_exception.py +++ b/pypy/module/cpyext/test/test_exception.py @@ -1,4 +1,5 @@ from pypy.module.cpyext.test.test_api import BaseApiTest +from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase from pypy.module.cpyext.pyobject import make_ref class TestExceptions(BaseApiTest): @@ -27,3 +28,16 @@ api.PyException_SetCause(w_exc, make_ref(space, w_cause)) assert space.is_w(api.PyException_GetCause(w_exc), w_cause) + +class AppTestExceptions(AppTestCpythonExtensionBase): + + def test_OSError_aliases(self): + module = self.import_extension('foo', [ + ("get_aliases", "METH_NOARGS", + """ + return PyTuple_Pack(2, + PyExc_EnvironmentError, + PyExc_IOError); + """), + ]) + assert module.get_aliases() == (OSError, OSError) diff --git a/pypy/module/exceptions/__init__.py b/pypy/module/exceptions/__init__.py --- a/pypy/module/exceptions/__init__.py +++ b/pypy/module/exceptions/__init__.py @@ -21,14 +21,14 @@ 'ConnectionResetError': 'interp_exceptions.W_ConnectionResetError', 'DeprecationWarning' : 'interp_exceptions.W_DeprecationWarning', 'EOFError' : 'interp_exceptions.W_EOFError', - 'EnvironmentError' : 'interp_exceptions.W_EnvironmentError', + 'EnvironmentError' : 'interp_exceptions.W_OSError', 'Exception' : 'interp_exceptions.W_Exception', 'FileExistsError': 'interp_exceptions.W_FileExistsError', 'FileNotFoundError': 'interp_exceptions.W_FileNotFoundError', 'FloatingPointError' : 'interp_exceptions.W_FloatingPointError', 'FutureWarning' : 'interp_exceptions.W_FutureWarning', 'GeneratorExit' : 'interp_exceptions.W_GeneratorExit', - 'IOError' : 'interp_exceptions.W_IOError', + 'IOError' : 'interp_exceptions.W_OSError', 'ImportError' : 'interp_exceptions.W_ImportError', 'ImportWarning' : 'interp_exceptions.W_ImportWarning', 'IndentationError' : 'interp_exceptions.W_IndentationError', diff --git a/pypy/module/exceptions/interp_exceptions.py b/pypy/module/exceptions/interp_exceptions.py --- a/pypy/module/exceptions/interp_exceptions.py +++ b/pypy/module/exceptions/interp_exceptions.py @@ -497,9 +497,6 @@ filename = readwrite_attrproperty_w('w_filename', W_OSError), ) -W_EnvironmentError = W_OSError -W_IOError = W_OSError - class W_WindowsError(W_OSError): """MS-Windows OS system call failed.""" diff --git a/pypy/module/math/__init__.py b/pypy/module/math/__init__.py --- a/pypy/module/math/__init__.py +++ b/pypy/module/math/__init__.py @@ -23,6 +23,7 @@ 'frexp' : 'interp_math.frexp', 'degrees' : 'interp_math.degrees', 'log' : 'interp_math.log', + 'log2' : 'interp_math.log2', 'log10' : 'interp_math.log10', 'fmod' : 'interp_math.fmod', 'atan' : 'interp_math.atan', diff --git a/pypy/module/math/interp_math.py b/pypy/module/math/interp_math.py --- a/pypy/module/math/interp_math.py +++ b/pypy/module/math/interp_math.py @@ -228,6 +228,11 @@ return math1(space, math.log, w_base) return _log_any(space, w_x, base) +def log2(space, w_x): + """log2(x) -> the base 2 logarithm of x. + """ + return _log_any(space, w_x, 2.0) + def log10(space, w_x): """log10(x) -> the base 10 logarithm of x. """ diff --git a/pypy/module/math/test/test_math.py b/pypy/module/math/test/test_math.py --- a/pypy/module/math/test/test_math.py +++ b/pypy/module/math/test/test_math.py @@ -148,6 +148,19 @@ raises(ValueError, math.log1p, -1) raises(ValueError, math.log1p, -100) + def test_log2(self): + import math + self.ftest(math.log2(0.125), -3) + self.ftest(math.log2(0.5), -1) + self.ftest(math.log2(4), 2) + + def test_log10(self): + import math + self.ftest(math.log10(0.1), -1) + self.ftest(math.log10(10), 1) + self.ftest(math.log10(100), 2) + self.ftest(math.log10(0.01), -2) + def test_acosh(self): import math self.ftest(math.acosh(1), 0) diff --git a/pypy/module/posix/__init__.py b/pypy/module/posix/__init__.py --- a/pypy/module/posix/__init__.py +++ b/pypy/module/posix/__init__.py @@ -1,5 +1,6 @@ from pypy.interpreter.mixedmodule import MixedModule from rpython.rtyper.module.ll_os import RegisterOs +from rpython.rlib import rdynload import os exec 'import %s as posix' % os.name @@ -173,6 +174,12 @@ if hasattr(os, name): interpleveldefs[name] = 'interp_posix.' + name + for _name in ["RTLD_LAZY", "RTLD_NOW", "RTLD_GLOBAL", "RTLD_LOCAL", + "RTLD_NODELETE", "RTLD_NOLOAD", "RTLD_DEEPBIND"]: + if getattr(rdynload.cConfig, _name) is not None: + interpleveldefs[_name] = 'space.wrap(%d)' % ( + getattr(rdynload.cConfig, _name),) + # os.py uses this list to build os.supports_dir_fd() and os.supports_fd(). # Fill with e.g. HAVE_FCHDIR, when os.chdir() supports file descriptors. interpleveldefs['_have_functions'] = 'space.newlist([])' diff --git a/pypy/module/posix/interp_posix.py b/pypy/module/posix/interp_posix.py --- a/pypy/module/posix/interp_posix.py +++ b/pypy/module/posix/interp_posix.py @@ -488,8 +488,7 @@ cur = os.getlogin() except OSError, e: raise wrap_oserror(space, e) - else: - return space.wrap(cur) + return space.fsdecode(space.wrapbytes(cur)) # ____________________________________________________________ @@ -702,14 +701,21 @@ except OSError, e: raise wrap_oserror(space, e) - at unwrap_spec(path='fsencode') -def readlink(space, path): +def readlink(space, w_path): "Return a string representing the path to which the symbolic link points." + is_unicode = space.isinstance_w(w_path, space.w_unicode) + if is_unicode: + path = space.fsencode_w(w_path) + else: + path = space.bytes0_w(w_path) try: result = os.readlink(path) except OSError, e: - raise wrap_oserror(space, e, path) - return space.wrap(result) + raise wrap_oserror2(space, e, w_path) + w_result = space.wrapbytes(result) + if is_unicode: + return space.fsdecode(w_result) + return w_result before_fork_hooks = [] after_fork_child_hooks = [] @@ -899,7 +905,8 @@ r = os.uname() except OSError, e: raise wrap_oserror(space, e) - l_w = [space.wrap(i) for i in [r[0], r[1], r[2], r[3], r[4]]] + l_w = [space.fsdecode(space.wrapbytes(i)) + for i in [r[0], r[1], r[2], r[3], r[4]]] w_tuple = space.newtuple(l_w) w_uname_result = space.getattr(space.getbuiltinmodule(os.name), space.wrap('uname_result')) @@ -1229,7 +1236,7 @@ @unwrap_spec(fd=c_int) def ttyname(space, fd): try: - return space.wrap(os.ttyname(fd)) + return space.fsdecode(space.wrapbytes(os.ttyname(fd))) except OSError, e: raise wrap_oserror(space, e) @@ -1364,7 +1371,7 @@ Return the name of the controlling terminal for this process. """ - return space.wrap(os.ctermid()) + return space.fsdecode(space.wrapbytes(os.ctermid())) @unwrap_spec(fd=c_int) def device_encoding(space, fd): diff --git a/pypy/module/posix/test/test_posix2.py b/pypy/module/posix/test/test_posix2.py --- a/pypy/module/posix/test/test_posix2.py +++ b/pypy/module/posix/test/test_posix2.py @@ -17,7 +17,7 @@ usemodules = ['binascii', 'posix', 'signal', 'struct', 'rctime'] # py3k os.open uses subprocess, requiring the following per platform if os.name != 'nt': - usemodules += ['fcntl', 'select'] + usemodules += ['fcntl', 'select', '_posixsubprocess'] else: usemodules += ['_rawffi', 'thread'] mod.space = gettestobjspace(usemodules=usemodules) @@ -1042,6 +1042,12 @@ # just ensure it returns something reasonable assert encoding is None or type(encoding) is str + def test_rtld_constants(self): + # check presence of major RTLD_* constants + self.posix.RTLD_LAZY + self.posix.RTLD_NOW + self.posix.RTLD_GLOBAL + self.posix.RTLD_LOCAL class AppTestEnvironment(object): def setup_class(cls): diff --git a/pypy/module/sys/version.py b/pypy/module/sys/version.py --- a/pypy/module/sys/version.py +++ b/pypy/module/sys/version.py @@ -6,7 +6,7 @@ from pypy.interpreter import gateway #XXX # the release serial 42 is not in range(16) -CPYTHON_VERSION = (3, 2, 5, "final", 0) +CPYTHON_VERSION = (3, 3, 5, "final", 0) #XXX # sync CPYTHON_VERSION with patchlevel.h, package.py CPYTHON_API_VERSION = 1013 #XXX # sync with include/modsupport.h diff --git a/pypy/objspace/std/nonetype.py b/pypy/objspace/std/nonetype.py --- a/pypy/objspace/std/nonetype.py +++ b/pypy/objspace/std/nonetype.py @@ -1,8 +1,15 @@ from pypy.objspace.std.stdtypedef import StdTypeDef +from pypy.interpreter import gateway +def descr__new__(space, w_type): + return space.w_None + # ____________________________________________________________ none_typedef = StdTypeDef("NoneType", + __new__ = gateway.interp2app(descr__new__) ) none_typedef.acceptable_as_base_class = False + + From noreply at buildbot.pypy.org Sat Jul 26 15:08:00 2014 From: noreply at buildbot.pypy.org (numerodix) Date: Sat, 26 Jul 2014 15:08:00 +0200 (CEST) Subject: [pypy-commit] pypy py3.3: Removing sys.flags.division_warning (removed in 3.3) Message-ID: <20140726130800.C84121C0695@cobra.cs.uni-duesseldorf.de> Author: Martin Matusiak Branch: py3.3 Changeset: r72506:d4642496139c Date: 2014-07-26 15:02 +0200 http://bitbucket.org/pypy/pypy/changeset/d4642496139c/ Log: Removing sys.flags.division_warning (removed in 3.3) diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py --- a/pypy/interpreter/app_main.py +++ b/pypy/interpreter/app_main.py @@ -317,7 +317,6 @@ # Order is significant! sys_flags = ( "debug", - "division_warning", "inspect", "interactive", "optimize", diff --git a/pypy/module/sys/app.py b/pypy/module/sys/app.py --- a/pypy/module/sys/app.py +++ b/pypy/module/sys/app.py @@ -94,20 +94,19 @@ name = "sys.flags" debug = structseqfield(0) - division_warning = structseqfield(1) - inspect = structseqfield(2) - interactive = structseqfield(3) - optimize = structseqfield(4) - dont_write_bytecode = structseqfield(5) - no_user_site = structseqfield(6) - no_site = structseqfield(7) - ignore_environment = structseqfield(8) - verbose = structseqfield(9) - bytes_warning = structseqfield(10) - quiet = structseqfield(11) - hash_randomization = structseqfield(12) + inspect = structseqfield(1) + interactive = structseqfield(2) + optimize = structseqfield(3) + dont_write_bytecode = structseqfield(4) + no_user_site = structseqfield(5) + no_site = structseqfield(6) + ignore_environment = structseqfield(7) + verbose = structseqfield(8) + bytes_warning = structseqfield(9) + quiet = structseqfield(10) + hash_randomization = structseqfield(11) -null_sysflags = sysflags((0,)*13) +null_sysflags = sysflags((0,)*12) null__xoptions = {} From noreply at buildbot.pypy.org Sat Jul 26 15:08:02 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 26 Jul 2014 15:08:02 +0200 (CEST) Subject: [pypy-commit] pypy py3.3: Merged in numerodix/pypy/py3.3 (pull request #249) Message-ID: <20140726130802.1ABBE1C0695@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: py3.3 Changeset: r72507:a85920372113 Date: 2014-07-26 15:07 +0200 http://bitbucket.org/pypy/pypy/changeset/a85920372113/ Log: Merged in numerodix/pypy/py3.3 (pull request #249) Removing sys.flags.division_warning (removed in 3.3) diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py --- a/pypy/interpreter/app_main.py +++ b/pypy/interpreter/app_main.py @@ -317,7 +317,6 @@ # Order is significant! sys_flags = ( "debug", - "division_warning", "inspect", "interactive", "optimize", diff --git a/pypy/module/sys/app.py b/pypy/module/sys/app.py --- a/pypy/module/sys/app.py +++ b/pypy/module/sys/app.py @@ -94,20 +94,19 @@ name = "sys.flags" debug = structseqfield(0) - division_warning = structseqfield(1) - inspect = structseqfield(2) - interactive = structseqfield(3) - optimize = structseqfield(4) - dont_write_bytecode = structseqfield(5) - no_user_site = structseqfield(6) - no_site = structseqfield(7) - ignore_environment = structseqfield(8) - verbose = structseqfield(9) - bytes_warning = structseqfield(10) - quiet = structseqfield(11) - hash_randomization = structseqfield(12) + inspect = structseqfield(1) + interactive = structseqfield(2) + optimize = structseqfield(3) + dont_write_bytecode = structseqfield(4) + no_user_site = structseqfield(5) + no_site = structseqfield(6) + ignore_environment = structseqfield(7) + verbose = structseqfield(8) + bytes_warning = structseqfield(9) + quiet = structseqfield(10) + hash_randomization = structseqfield(11) -null_sysflags = sysflags((0,)*13) +null_sysflags = sysflags((0,)*12) null__xoptions = {} From noreply at buildbot.pypy.org Sat Jul 26 15:18:13 2014 From: noreply at buildbot.pypy.org (Manuel Jacob) Date: Sat, 26 Jul 2014 15:18:13 +0200 (CEST) Subject: [pypy-commit] pypy improve-docs: RPython docs: Use read the docs theme locally if available (copied from PyPy docs configuration). Message-ID: <20140726131813.129071C046A@cobra.cs.uni-duesseldorf.de> Author: Manuel Jacob Branch: improve-docs Changeset: r72508:8e884ef6d1d1 Date: 2014-07-26 15:17 +0200 http://bitbucket.org/pypy/pypy/changeset/8e884ef6d1d1/ Log: RPython docs: Use read the docs theme locally if available (copied from PyPy docs configuration). diff --git a/rpython/doc/conf.py b/rpython/doc/conf.py --- a/rpython/doc/conf.py +++ b/rpython/doc/conf.py @@ -18,6 +18,24 @@ # documentation root, use os.path.abspath to make it absolute, like shown here. #sys.path.insert(0, os.path.abspath('.')) + +# -- Read The Docs theme config ------------------------------------------------ + +# on_rtd is whether we are on readthedocs.org, this line of code grabbed from docs.readthedocs.org +on_rtd = os.environ.get('READTHEDOCS', None) == 'True' + +if not on_rtd: # only import and set the theme if we're building docs locally + try: + import sphinx_rtd_theme + html_theme = 'sphinx_rtd_theme' + html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] + except ImportError: + print('sphinx_rtd_theme is not installed') + html_theme = 'default' + +# otherwise, readthedocs.org uses their theme by default, so no need to specify it + + # -- General configuration ----------------------------------------------------- # If your documentation needs a minimal Sphinx version, state it here. @@ -91,7 +109,7 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'default' +#html_theme = 'default' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the From noreply at buildbot.pypy.org Sat Jul 26 15:28:11 2014 From: noreply at buildbot.pypy.org (Valentina Mukhamedzhanova) Date: Sat, 26 Jul 2014 15:28:11 +0200 (CEST) Subject: [pypy-commit] pypy py3.3: Add test_decompress_eof_incomplete_stream to test_zlib.py. Message-ID: <20140726132811.1631E1C0695@cobra.cs.uni-duesseldorf.de> Author: Valentina Mukhamedzhanova Branch: py3.3 Changeset: r72509:2b2441e85ee2 Date: 2014-07-26 15:14 +0200 http://bitbucket.org/pypy/pypy/changeset/2b2441e85ee2/ Log: Add test_decompress_eof_incomplete_stream to test_zlib.py. diff --git a/pypy/module/zlib/test/test_zlib.py b/pypy/module/zlib/test/test_zlib.py --- a/pypy/module/zlib/test/test_zlib.py +++ b/pypy/module/zlib/test/test_zlib.py @@ -177,6 +177,16 @@ assert dco.eof == True dco.flush() assert dco.eof == True + + def test_decompress_eof_incomplete_stream(self): + import zlib + x = b'x\x9cK\xcb\xcf\x07\x00\x02\x82\x01E' # 'foo' + dco = zlib.decompressobj() + assert dco.eof == False + dco.decompress(x[:-5]) + assert dco.eof == False + dco.flush() + assert dco.eof == False def test_decompress_incomplete_stream(self): import zlib From noreply at buildbot.pypy.org Sat Jul 26 15:28:12 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 26 Jul 2014 15:28:12 +0200 (CEST) Subject: [pypy-commit] pypy py3.3: merge heads Message-ID: <20140726132812.441251C0695@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: py3.3 Changeset: r72510:4e00d712484e Date: 2014-07-26 15:27 +0200 http://bitbucket.org/pypy/pypy/changeset/4e00d712484e/ Log: merge heads diff --git a/pypy/module/zlib/test/test_zlib.py b/pypy/module/zlib/test/test_zlib.py --- a/pypy/module/zlib/test/test_zlib.py +++ b/pypy/module/zlib/test/test_zlib.py @@ -177,6 +177,16 @@ assert dco.eof == True dco.flush() assert dco.eof == True + + def test_decompress_eof_incomplete_stream(self): + import zlib + x = b'x\x9cK\xcb\xcf\x07\x00\x02\x82\x01E' # 'foo' + dco = zlib.decompressobj() + assert dco.eof == False + dco.decompress(x[:-5]) + assert dco.eof == False + dco.flush() + assert dco.eof == False def test_decompress_incomplete_stream(self): import zlib From noreply at buildbot.pypy.org Sat Jul 26 15:34:20 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 26 Jul 2014 15:34:20 +0200 (CEST) Subject: [pypy-commit] pypy default: Remove these old comments Message-ID: <20140726133420.9CE021C06AE@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: Changeset: r72511:e65dfce91ce5 Date: 2014-07-26 15:34 +0200 http://bitbucket.org/pypy/pypy/changeset/e65dfce91ce5/ Log: Remove these old comments diff --git a/pypy/objspace/std/iterobject.py b/pypy/objspace/std/iterobject.py --- a/pypy/objspace/std/iterobject.py +++ b/pypy/objspace/std/iterobject.py @@ -30,10 +30,6 @@ raise NotImplementedError def descr_reduce(self, space): - """ - XXX to do: remove this __reduce__ method and do - a registration with copy_reg, instead. - """ from pypy.interpreter.mixedmodule import MixedModule w_mod = space.getbuiltinmodule('_pickle_support') mod = space.interp_w(MixedModule, w_mod) @@ -125,10 +121,6 @@ self.index = space.int_w(self.w_len) + index def descr_reduce(self, space): - """ - XXX to do: remove this __reduce__ method and do - a registration with copy_reg, instead. - """ from pypy.interpreter.mixedmodule import MixedModule w_mod = space.getbuiltinmodule('_pickle_support') mod = space.interp_w(MixedModule, w_mod) From noreply at buildbot.pypy.org Sat Jul 26 15:36:13 2014 From: noreply at buildbot.pypy.org (kvas) Date: Sat, 26 Jul 2014 15:36:13 +0200 (CEST) Subject: [pypy-commit] pypy py3.3: Fixed pickling of iterators produced by filter(func, iter). Message-ID: <20140726133613.C70961C0A1B@cobra.cs.uni-duesseldorf.de> Author: Vasily Kuznetsov Branch: py3.3 Changeset: r72512:6aecbffc27c8 Date: 2014-07-26 15:32 +0200 http://bitbucket.org/pypy/pypy/changeset/6aecbffc27c8/ Log: Fixed pickling of iterators produced by filter(func, iter). diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py --- a/pypy/module/__builtin__/functional.py +++ b/pypy/module/__builtin__/functional.py @@ -711,6 +711,13 @@ if pred ^ self.reverse: return w_obj + def descr_reduce(self, space): + w_filter = space.getattr(space.getbuiltinmodule('builtins'), + space.wrap('filter')) + args = [space.w_None if self.no_predicate else self.w_predicate, + self.iterable] + return space.newtuple([w_filter, space.newtuple(args)]) + def W_Filter___new__(space, w_subtype, w_predicate, w_iterable): r = space.allocate_instance(W_Filter, w_subtype) @@ -722,6 +729,7 @@ __new__ = interp2app(W_Filter___new__), __iter__ = interp2app(W_Filter.iter_w), __next__ = interp2app(W_Filter.next_w), + __reduce__ = interp2app(W_Filter.descr_reduce), __doc__ = """\ Return an iterator yielding those items of iterable for which function(item) is true. If function is None, return the items that are true.""") diff --git a/pypy/module/__builtin__/test/test_filter_pickle.py b/pypy/module/__builtin__/test/test_filter_pickle.py new file mode 100644 --- /dev/null +++ b/pypy/module/__builtin__/test/test_filter_pickle.py @@ -0,0 +1,72 @@ +class AppTestFilterPickle: + + def test_filter_unpickle(self): + """Test just the unpickling.""" + import pickle + + # This is filter(None, 'abc') pickled with cpython + dump = b'\x80\x03cbuiltins\nfilter\nq\x00Ncbuiltins\niter\nq\x01X\x03'\ + b'\x00\x00\x00abcq\x02\x85q\x03Rq\x04K\x00b\x86q\x05Rq\x06.' + t = pickle.loads(dump) + assert list(t) == ['a', 'b', 'c'] + + def test_iterator_pickle(self): + """Pickle and unpickle just a simple iterator.""" + import pickle + + i0 = iter("abc") + i1 = iter("abc") + + d = pickle.dumps(i1) + i1 = pickle.loads(d) + + assert list(i0) == list(i1) + + def test_reduce_ex(self): + """""" + f0 = filter(None, "abc") + f1 = filter(None, "abc") + + print(f0) + r = f1.__reduce_ex__(3) + # __reduce_ex__ doesn't return any arguments to the filter, so the next + # line will fail with TypeError. + f1 = r[0](*r[1]) + + assert list(f0) == list(f1) + + def test_nonfilter_pickle(self): + """Pickle and unpickle a filter with no filtering.""" + import pickle + + f0 = filter(None, "abc") + d = pickle.dumps(f0) + f1 = pickle.loads(d) + + def test_filter_pickle(self): + """Clone of the original test.""" + import pickle + + def check_iter_pickle(it, seq): + itorg = it + d = pickle.dumps(it) + it = pickle.loads(d) + assert type(itorg) == type(it) + assert list(it) == seq + + #test the iterator after dropping one from it + it = pickle.loads(d) + try: + next(it) + except StopIteration: + return + d = pickle.dumps(it) + it = pickle.loads(d) + assert list(it) == seq[1:] + + # We use ord instead of filter_char because the filter function has to + # be defined in the global scope for the picking to work and we can't + # do it from this test. + f1 = filter(ord, "abcdeabcde") + f2 = filter(ord, "abcdeabcde") + check_iter_pickle(f1, list(f2)) From noreply at buildbot.pypy.org Sat Jul 26 15:36:14 2014 From: noreply at buildbot.pypy.org (arigo) Date: Sat, 26 Jul 2014 15:36:14 +0200 (CEST) Subject: [pypy-commit] pypy py3.3: merge heads Message-ID: <20140726133614.E9EF31C0A1B@cobra.cs.uni-duesseldorf.de> Author: Armin Rigo Branch: py3.3 Changeset: r72513:b14679e8321d Date: 2014-07-26 15:35 +0200 http://bitbucket.org/pypy/pypy/changeset/b14679e8321d/ Log: merge heads diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py --- a/pypy/module/__builtin__/functional.py +++ b/pypy/module/__builtin__/functional.py @@ -711,6 +711,13 @@ if pred ^ self.reverse: return w_obj + def descr_reduce(self, space): + w_filter = space.getattr(space.getbuiltinmodule('builtins'), + space.wrap('filter')) + args = [space.w_None if self.no_predicate else self.w_predicate, + self.iterable] + return space.newtuple([w_filter, space.newtuple(args)]) + def W_Filter___new__(space, w_subtype, w_predicate, w_iterable): r = space.allocate_instance(W_Filter, w_subtype) @@ -722,6 +729,7 @@ __new__ = interp2app(W_Filter___new__), __iter__ = interp2app(W_Filter.iter_w), __next__ = interp2app(W_Filter.next_w), + __reduce__ = interp2app(W_Filter.descr_reduce), __doc__ = """\ Return an iterator yielding those items of iterable for which function(item) is true. If function is None, return the items that are true.""") diff --git a/pypy/module/__builtin__/test/test_filter_pickle.py b/pypy/module/__builtin__/test/test_filter_pickle.py new file mode 100644 --- /dev/null +++ b/pypy/module/__builtin__/test/test_filter_pickle.py @@ -0,0 +1,72 @@ +class AppTestFilterPickle: + + def test_filter_unpickle(self): + """Test just the unpickling.""" + import pickle + + # This is filter(None, 'abc') pickled with cpython + dump = b'\x80\x03cbuiltins\nfilter\nq\x00Ncbuiltins\niter\nq\x01X\x03'\ + b'\x00\x00\x00abcq\x02\x85q\x03Rq\x04K\x00b\x86q\x05Rq\x06.' + t = pickle.loads(dump) + assert list(t) == ['a', 'b', 'c'] + + def test_iterator_pickle(self): + """Pickle and unpickle just a simple iterator.""" + import pickle + + i0 = iter("abc") + i1 = iter("abc") + + d = pickle.dumps(i1) + i1 = pickle.loads(d) + + assert list(i0) == list(i1) + + def test_reduce_ex(self): + """""" + f0 = filter(None, "abc") + f1 = filter(None, "abc") + + print(f0) + r = f1.__reduce_ex__(3) + # __reduce_ex__ doesn't return any arguments to the filter, so the next + # line will fail with TypeError. + f1 = r[0](*r[1]) + + assert list(f0) == list(f1) + + def test_nonfilter_pickle(self): + """Pickle and unpickle a filter with no filtering.""" + import pickle + + f0 = filter(None, "abc") + d = pickle.dumps(f0) + f1 = pickle.loads(d) + + def test_filter_pickle(self): + """Clone of the original test.""" + import pickle + + def check_iter_pickle(it, seq): + itorg = it + d = pickle.dumps(it) + it = pickle.loads(d) + assert type(itorg) == type(it) + assert list(it) == seq + + #test the iterator after dropping one from it + it = pickle.loads(d) + try: + next(it) + except StopIteration: + return + d = pickle.dumps(it) + it = pickle.loads(d) + assert list(it) == seq[1:] + + # We use ord instead of filter_char because the filter function has to + # be defined in the global scope for the picking to work and we can't + # do it from this test. + f1 = filter(ord, "abcdeabcde") + f2 = filter(ord, "abcdeabcde") + check_iter_pickle(f1, list(f2)) From noreply at buildbot.pypy.org Sat Jul 26 15:54:09 2014 From: noreply at buildbot.pypy.org (Arjun Naik) Date: Sat, 26 Jul 2014 15:54:09 +0200 (CEST) Subject: [pypy-commit] pypy py3.3: Changed the _get_inttime() to raise OverflowError. Message-ID: <20140726135409.0D7F91C0588@cobra.cs.uni-duesseldorf.de> Author: Arjun Naik Branch: py3.3 Changeset: r72514:8ccff189b82f Date: 2014-07-26 15:11 +0200 http://bitbucket.org/pypy/pypy/changeset/8ccff189b82f/ Log: Changed the _get_inttime() to raise OverflowError. diff --git a/pypy/module/rctime/interp_time.py b/pypy/module/rctime/interp_time.py --- a/pypy/module/rctime/interp_time.py +++ b/pypy/module/rctime/interp_time.py @@ -364,7 +364,7 @@ # input doesn't fit in a time_t; call it an error. diff = seconds - rffi.cast(lltype.Float, t) if diff <= -1.0 or diff >= 1.0: - raise OperationError(space.w_ValueError, + raise OperationError(space.w_OverflowError, space.wrap("timestamp out of range for platform time_t")) return t diff --git a/pypy/module/rctime/test/test_rctime.py b/pypy/module/rctime/test/test_rctime.py --- a/pypy/module/rctime/test/test_rctime.py +++ b/pypy/module/rctime/test/test_rctime.py @@ -42,7 +42,7 @@ res = rctime.ctime(0) assert isinstance(res, str) rctime.ctime(rctime.time()) - raises(ValueError, rctime.ctime, 1E200) + raises(OverflowError, rctime.ctime, 1E200) raises(OverflowError, rctime.ctime, 10**900) for year in [-100, 100, 1000, 2000, 10000]: try: @@ -68,8 +68,8 @@ assert 0 <= (t1 - t0) < 1.2 t = rctime.time() assert rctime.gmtime(t) == rctime.gmtime(t) - raises(ValueError, rctime.gmtime, 2**64) - raises(ValueError, rctime.gmtime, -2**64) + raises(OverflowError, rctime.gmtime, 2**64) + raises(OverflowError, rctime.gmtime, -2**64) def test_localtime(self): import time as rctime From noreply at buildbot.pypy.org Sat Jul 26 15:54:10 2014 From: noreply at buildbot.pypy.org (Arjun Naik) Date: Sat, 26 Jul 2014 15:54:10 +0200 (CEST) Subject: [pypy-commit] pypy py3.3: Merge Message-ID: <20140726135410.4F99D1C0588@cobra.cs.uni-duesseldorf.de> Author: Arjun Naik Branch: py3.3 Changeset: r72515:2c99b4f912ad Date: 2014-07-26 15:52 +0200 http://bitbucket.org/pypy/pypy/changeset/2c99b4f912ad/ Log: Merge diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py --- a/pypy/interpreter/app_main.py +++ b/pypy/interpreter/app_main.py @@ -317,7 +317,6 @@ # Order is significant! sys_flags = ( "debug", - "division_warning", "inspect", "interactive", "optimize", diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py --- a/pypy/module/__builtin__/functional.py +++ b/pypy/module/__builtin__/functional.py @@ -711,6 +711,13 @@ if pred ^ self.reverse: return w_obj + def descr_reduce(self, space): + w_filter = space.getattr(space.getbuiltinmodule('builtins'), + space.wrap('filter')) + args = [space.w_None if self.no_predicate else self.w_predicate, + self.iterable] + return space.newtuple([w_filter, space.newtuple(args)]) + def W_Filter___new__(space, w_subtype, w_predicate, w_iterable): r = space.allocate_instance(W_Filter, w_subtype) @@ -722,6 +729,7 @@ __new__ = interp2app(W_Filter___new__), __iter__ = interp2app(W_Filter.iter_w), __next__ = interp2app(W_Filter.next_w), + __reduce__ = interp2app(W_Filter.descr_reduce), __doc__ = """\ Return an iterator yielding those items of iterable for which function(item) is true. If function is None, return the items that are true.""") diff --git a/pypy/module/__builtin__/test/test_filter_pickle.py b/pypy/module/__builtin__/test/test_filter_pickle.py new file mode 100644 --- /dev/null +++ b/pypy/module/__builtin__/test/test_filter_pickle.py @@ -0,0 +1,72 @@ +class AppTestFilterPickle: + + def test_filter_unpickle(self): + """Test just the unpickling.""" + import pickle + + # This is filter(None, 'abc') pickled with cpython + dump = b'\x80\x03cbuiltins\nfilter\nq\x00Ncbuiltins\niter\nq\x01X\x03'\ + b'\x00\x00\x00abcq\x02\x85q\x03Rq\x04K\x00b\x86q\x05Rq\x06.' + t = pickle.loads(dump) + assert list(t) == ['a', 'b', 'c'] + + def test_iterator_pickle(self): + """Pickle and unpickle just a simple iterator.""" + import pickle + + i0 = iter("abc") + i1 = iter("abc") + + d = pickle.dumps(i1) + i1 = pickle.loads(d) + + assert list(i0) == list(i1) + + def test_reduce_ex(self): + """""" + f0 = filter(None, "abc") + f1 = filter(None, "abc") + + print(f0) + r = f1.__reduce_ex__(3) + # __reduce_ex__ doesn't return any arguments to the filter, so the next + # line will fail with TypeError. + f1 = r[0](*r[1]) + + assert list(f0) == list(f1) + + def test_nonfilter_pickle(self): + """Pickle and unpickle a filter with no filtering.""" + import pickle + + f0 = filter(None, "abc") + d = pickle.dumps(f0) + f1 = pickle.loads(d) + + def test_filter_pickle(self): + """Clone of the original test.""" + import pickle + + def check_iter_pickle(it, seq): + itorg = it + d = pickle.dumps(it) + it = pickle.loads(d) + assert type(itorg) == type(it) + assert list(it) == seq + + #test the iterator after dropping one from it + it = pickle.loads(d) + try: + next(it) + except StopIteration: + return + d = pickle.dumps(it) + it = pickle.loads(d) + assert list(it) == seq[1:] + + # We use ord instead of filter_char because the filter function has to + # be defined in the global scope for the picking to work and we can't + # do it from this test. + f1 = filter(ord, "abcdeabcde") + f2 = filter(ord, "abcdeabcde") + check_iter_pickle(f1, list(f2)) diff --git a/pypy/module/sys/app.py b/pypy/module/sys/app.py --- a/pypy/module/sys/app.py +++ b/pypy/module/sys/app.py @@ -94,20 +94,19 @@ name = "sys.flags" debug = structseqfield(0) - division_warning = structseqfield(1) - inspect = structseqfield(2) - interactive = structseqfield(3) - optimize = structseqfield(4) - dont_write_bytecode = structseqfield(5) - no_user_site = structseqfield(6) - no_site = structseqfield(7) - ignore_environment = structseqfield(8) - verbose = structseqfield(9) - bytes_warning = structseqfield(10) - quiet = structseqfield(11) - hash_randomization = structseqfield(12) + inspect = structseqfield(1) + interactive = structseqfield(2) + optimize = structseqfield(3) + dont_write_bytecode = structseqfield(4) + no_user_site = structseqfield(5) + no_site = structseqfield(6) + ignore_environment = structseqfield(7) + verbose = structseqfield(8) + bytes_warning = structseqfield(9) + quiet = structseqfield(10) + hash_randomization = structseqfield(11) -null_sysflags = sysflags((0,)*13) +null_sysflags = sysflags((0,)*12) null__xoptions = {} diff --git a/pypy/module/zlib/test/test_zlib.py b/pypy/module/zlib/test/test_zlib.py --- a/pypy/module/zlib/test/test_zlib.py +++ b/pypy/module/zlib/test/test_zlib.py @@ -177,6 +177,16 @@ assert dco.eof == True dco.flush() assert dco.eof == True + + def test_decompress_eof_incomplete_stream(self): + import zlib + x = b'x\x9cK\xcb\xcf\x07\x00\x02\x82\x01E' # 'foo' + dco = zlib.decompressobj() + assert dco.eof == False + dco.decompress(x[:-5]) + assert dco.eof == False + dco.flush() + assert dco.eof == False def test_decompress_incomplete_stream(self): import zlib From noreply at buildbot.pypy.org Sat Jul 26 16:00:49 2014 From: noreply at buildbot.pypy.org (kvas) Date: Sat, 26 Jul 2014 16:00:49 +0200 (CEST) Subject: [pypy-commit] pypy py3.3: Fixed pickling of iterators produced by map(func, iter, ...). Message-ID: <20140726140049.309011C1106@cobra.cs.uni-duesseldorf.de> Author: Vasily Kuznetsov Branch: py3.3 Changeset: r72516:b30c4330c8b3 Date: 2014-07-26 15:57 +0200 http://bitbucket.org/pypy/pypy/changeset/b30c4330c8b3/ Log: Fixed pickling of iterators produced by map(func, iter, ...). diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py --- a/pypy/module/__builtin__/functional.py +++ b/pypy/module/__builtin__/functional.py @@ -667,6 +667,12 @@ # the loop is out of the way of the JIT return [self.space.next(w_elem) for w_elem in self.iterators_w] + def descr_reduce(self, space): + w_map = space.getattr(space.getbuiltinmodule('builtins'), + space.wrap('map')) + args = [self.w_fun] + self.iterators_w + return space.newtuple([w_map, space.newtuple(args)]) + def W_Map___new__(space, w_subtype, w_fun, args_w): if len(args_w) == 0: @@ -681,6 +687,7 @@ __new__ = interp2app(W_Map___new__), __iter__ = interp2app(W_Map.iter_w), __next__ = interp2app(W_Map.next_w), + __reduce__ = interp2app(W_Map.descr_reduce), __doc__ = """\ Make an iterator that computes the function using arguments from each of the iterables. Stops when the shortest iterable is exhausted.""") diff --git a/pypy/module/__builtin__/test/test_map_pickle.py b/pypy/module/__builtin__/test/test_map_pickle.py new file mode 100644 --- /dev/null +++ b/pypy/module/__builtin__/test/test_map_pickle.py @@ -0,0 +1,28 @@ +class AppTestMapPickle: + + def test_map_pickle(self): + """Pickle a map with one sequence.""" + import pickle + + def pickle_unpickle(obj): + d = pickle.dumps(obj) + return pickle.loads(d) + + m1 = map(ord, "Is this the real life?") + m1_ = pickle_unpickle(m1) + + assert list(m1) == list(m1_) + + def test_map2_pickle(self): + """Pickle a map with multiple sequences.""" + import pickle + + def pickle_unpickle(obj): + d = pickle.dumps(obj) + return pickle.loads(d) + + m1 = map(max, "abc", "def") + m1_ = pickle_unpickle(m1) + + assert list(m1) == list(m1_) + From noreply at buildbot.pypy.org Sat Jul 26 16:06:27 2014 From: noreply at buildbot.pypy.org (kvas) Date: Sat, 26 Jul 2014 16:06:27 +0200 (CEST) Subject: [pypy-commit] pypy py3.3: Added a test for the situation when one of the iterators given to map has been already advanced (suggested by Armin Rigo). Message-ID: <20140726140627.B35601C1106@cobra.cs.uni-duesseldorf.de> Author: Vasily Kuznetsov Branch: py3.3 Changeset: r72517:88e3c7e587b4 Date: 2014-07-26 16:05 +0200 http://bitbucket.org/pypy/pypy/changeset/88e3c7e587b4/ Log: Added a test for the situation when one of the iterators given to map has been already advanced (suggested by Armin Rigo). diff --git a/pypy/module/__builtin__/test/test_map_pickle.py b/pypy/module/__builtin__/test/test_map_pickle.py --- a/pypy/module/__builtin__/test/test_map_pickle.py +++ b/pypy/module/__builtin__/test/test_map_pickle.py @@ -26,3 +26,19 @@ assert list(m1) == list(m1_) + def test_map2_adv_pickle(self): + """If some iterator was advanced, the pickling preserves it.""" + import pickle + + def pickle_unpickle(obj): + d = pickle.dumps(obj) + return pickle.loads(d) + + s1 = iter("abc") + s2 = iter("defg") + next(s2) + + m1 = map(max, s1, s2) + m1_ = pickle_unpickle(m1) + + assert list(m1) == list(m1_) From noreply at buildbot.pypy.org Sat Jul 26 16:10:40 2014 From: noreply at buildbot.pypy.org (Manuel Jacob) Date: Sat, 26 Jul 2014 16:10:40 +0200 (CEST) Subject: [pypy-commit] pypy improve-docs: RPython docs: Delete contents.rst and move toctree to index.rst. Message-ID: <20140726141040.9C8001C046A@cobra.cs.uni-duesseldorf.de> Author: Manuel Jacob Branch: improve-docs Changeset: r72518:e8bf18c22cdb Date: 2014-07-26 15:23 +0200 http://bitbucket.org/pypy/pypy/changeset/e8bf18c22cdb/ Log: RPython docs: Delete contents.rst and move toctree to index.rst. diff --git a/rpython/doc/contents.rst b/rpython/doc/contents.rst deleted file mode 100644 --- a/rpython/doc/contents.rst +++ /dev/null @@ -1,19 +0,0 @@ -Table of Contents -=================================== - -Contents: - -.. toctree:: - :maxdepth: 2 - - getting-started - faq - rpython - rlib - rffi - translation - rtyper - garbage_collection - cli-backend - windows - diff --git a/rpython/doc/index.rst b/rpython/doc/index.rst --- a/rpython/doc/index.rst +++ b/rpython/doc/index.rst @@ -1,7 +1,23 @@ Welcome to RPython's documentation! =================================== -* :doc:`contents` +Table of Contents +----------------- + +.. toctree:: + :maxdepth: 1 + + getting-started + faq + rpython + rlib + rffi + translation + rtyper + garbage_collection + cli-backend + windows + Indices and tables ================== @@ -9,4 +25,3 @@ * :ref:`genindex` * :ref:`modindex` * :ref:`search` - From noreply at buildbot.pypy.org Sat Jul 26 16:10:41 2014 From: noreply at buildbot.pypy.org (Manuel Jacob) Date: Sat, 26 Jul 2014 16:10:41 +0200 (CEST) Subject: [pypy-commit] pypy improve-docs: Rename title of rpython.rst from "RPython" to "RPython Language". Message-ID: <20140726141041.CA3C51C046A@cobra.cs.uni-duesseldorf.de> Author: Manuel Jacob Branch: improve-docs Changeset: r72519:79f23ae997e3 Date: 2014-07-26 16:10 +0200 http://bitbucket.org/pypy/pypy/changeset/79f23ae997e3/ Log: Rename title of rpython.rst from "RPython" to "RPython Language". diff --git a/rpython/doc/rpython.rst b/rpython/doc/rpython.rst --- a/rpython/doc/rpython.rst +++ b/rpython/doc/rpython.rst @@ -1,8 +1,8 @@ -RPython -======= +RPython Language +================ -RPython Definition ------------------- +Definition +---------- RPython is a restricted subset of Python that is amenable to static analysis. Although there are additions to the language and some things might surprisingly From noreply at buildbot.pypy.org Sat Jul 26 16:47:45 2014 From: noreply at buildbot.pypy.org (groggi) Date: Sat, 26 Jul 2014 16:47:45 +0200 (CEST) Subject: [pypy-commit] pypy gc-incminimark-pinning: extend and enable test cases for still missing feature Message-ID: <20140726144745.68C551C050D@cobra.cs.uni-duesseldorf.de> Author: Gregor Wegberg


Branch: gc-incminimark-pinning
Changeset: r72520:e2b83c3965c0
Date: 2014-07-02 14:15 +0200
http://bitbucket.org/pypy/pypy/changeset/e2b83c3965c0/

Log:	extend and enable test cases for still missing feature

diff --git a/rpython/memory/gc/test/test_object_pinning.py b/rpython/memory/gc/test/test_object_pinning.py
--- a/rpython/memory/gc/test/test_object_pinning.py
+++ b/rpython/memory/gc/test/test_object_pinning.py
@@ -76,25 +76,69 @@
         ptr_old = self.stackroots[0]
         assert ptr_old.someInt == 100
 
-    @py.test.mark.xfail(reason="Not implemented yet", run=False)
-    def test_pin_referenced_from_stackroot(self):
-        # XXX most likely somehow connected with `old_objects_pointing_to_young`
-        # (groggi)
+    def test_pin_referenced_from_stackroot_young(self):
+        #
+        # create both objects and reference the pinned one
+        # from the one that will be moved out of the
+        # nursery.
         root_ptr = self.malloc(S)
         next_ptr = self.malloc(S)
         self.write(root_ptr, 'next', next_ptr)
         self.stackroots.append(root_ptr)
+        #
         next_ptr.someInt = 100
-
+        root_ptr.someInt = 999
+        #
         next_adr = llmemory.cast_ptr_to_adr(next_ptr)
         assert self.gc.pin(next_adr)
+        #
+        # in this step the 'root_ptr' object will be
+        # outside the nursery, pointing to the still
+        # young (because it's pinned) 'next_ptr'.
+        self.gc.collect()
+        #
+        root_ptr = self.stackroots[0]
+        assert not self.gc.is_in_nursery(llmemory.cast_ptr_to_adr(root_ptr))
+        assert self.gc.is_in_nursery(next_adr)
+        assert next_ptr.someInt == 100
+        assert root_ptr.next == next_ptr
+        #
+        # now we remove the reference to the pinned object and do a collect
+        # to check if the pinned object was removed from nursery.
+        self.write(root_ptr, 'next', lltype.nullptr(S))
+        self.gc.collect()
+        try:
+            # should fail as this was the pinned object that is now collected
+            next_ptr.someInt = 0
+        except RuntimeError as ex:
+            assert "freed" in str(ex)
 
+    def test_old_points_to_pinned(self):
+        # Test if we handle the case that an old object can point
+        # to a pinned object and keeps the pinned object alive by
+        # that.
+        #
+        # create the old object that will point to a pinned object
+        old_ptr = self.malloc(S)
+        old_ptr.someInt = 999
+        self.stackroots.append(old_ptr)
         self.gc.collect()
-
-        assert self.gc.is_in_nursery(adr)
-        assert next_ptr.someInt == 100
-        root_ptr = self.stackroots[0]
-        assert root_ptr.next == next_ptr
+        assert not self.gc.is_in_nursery(llmemory.cast_ptr_to_adr(self.stackroots[0]))
+        #
+        # create the young pinned object and attach it to the old object
+        pinned_ptr = self.malloc(S)
+        pinned_ptr.someInt = 6
+        assert self.gc.pin(llmemory.cast_ptr_to_adr(pinned_ptr))
+        self.write(self.stackroots[0], 'next', pinned_ptr)
+        #
+        # let's check if everything stays in place before/after a collection
+        assert self.gc.is_in_nursery(llmemory.cast_ptr_to_adr(pinned_ptr))
+        self.gc.collect()
+        assert self.gc.is_in_nursery(llmemory.cast_ptr_to_adr(pinned_ptr))
+        #
+        self.stackroots[0].next.someInt = 100
+        self.gc.collect()
+        assert self.stackroots[0].next.someInt == 100
 
     def test_pin_old(self):
         ptr = self.malloc(S)
@@ -109,22 +153,24 @@
         # ^^^ should not be possible, struct is already old and won't
         # move.
 
-    def test_old_points_to_pinned(self):
+    def test_groggi(self):
         # Test if we handle the case that an old object can point
         # to a pinned object and keeps the pinned object alive by
         # that.
         #
+        # create the young pinned object and attach it to the old object
+        pinned_ptr = self.malloc(S)
+        pinned_ptr.someInt = 6
+        assert self.gc.pin(llmemory.cast_ptr_to_adr(pinned_ptr))
+        #
         # create the old object that will point to a pinned object
         old_ptr = self.malloc(S)
+        old_ptr.someInt = 999
         self.stackroots.append(old_ptr)
+        self.write(self.stackroots[0], 'next', pinned_ptr)
         self.gc.collect()
         assert not self.gc.is_in_nursery(llmemory.cast_ptr_to_adr(self.stackroots[0]))
         #
-        # create the young pinned object and attach it to the old object
-        pinned_ptr = self.malloc(S)
-        assert self.gc.pin(llmemory.cast_ptr_to_adr(pinned_ptr))
-        self.write(self.stackroots[0], 'next', pinned_ptr)
-        #
         # let's check if everything stays in place before/after a collection
         assert self.gc.is_in_nursery(llmemory.cast_ptr_to_adr(pinned_ptr))
         self.gc.collect()

From noreply at buildbot.pypy.org  Sat Jul 26 16:47:46 2014
From: noreply at buildbot.pypy.org (groggi)
Date: Sat, 26 Jul 2014 16:47:46 +0200 (CEST)
Subject: [pypy-commit] pypy gc-incminimark-pinning: ups. remove test used to
	play around
Message-ID: <20140726144746.9D5001C050D@cobra.cs.uni-duesseldorf.de>

Author: Gregor Wegberg 
Branch: gc-incminimark-pinning
Changeset: r72521:3fca2532c67d
Date: 2014-07-02 14:43 +0200
http://bitbucket.org/pypy/pypy/changeset/3fca2532c67d/

Log:	ups. remove test used to play around

diff --git a/rpython/memory/gc/test/test_object_pinning.py b/rpython/memory/gc/test/test_object_pinning.py
--- a/rpython/memory/gc/test/test_object_pinning.py
+++ b/rpython/memory/gc/test/test_object_pinning.py
@@ -153,33 +153,6 @@
         # ^^^ should not be possible, struct is already old and won't
         # move.
 
-    def test_groggi(self):
-        # Test if we handle the case that an old object can point
-        # to a pinned object and keeps the pinned object alive by
-        # that.
-        #
-        # create the young pinned object and attach it to the old object
-        pinned_ptr = self.malloc(S)
-        pinned_ptr.someInt = 6
-        assert self.gc.pin(llmemory.cast_ptr_to_adr(pinned_ptr))
-        #
-        # create the old object that will point to a pinned object
-        old_ptr = self.malloc(S)
-        old_ptr.someInt = 999
-        self.stackroots.append(old_ptr)
-        self.write(self.stackroots[0], 'next', pinned_ptr)
-        self.gc.collect()
-        assert not self.gc.is_in_nursery(llmemory.cast_ptr_to_adr(self.stackroots[0]))
-        #
-        # let's check if everything stays in place before/after a collection
-        assert self.gc.is_in_nursery(llmemory.cast_ptr_to_adr(pinned_ptr))
-        self.gc.collect()
-        assert self.gc.is_in_nursery(llmemory.cast_ptr_to_adr(pinned_ptr))
-        #
-        self.stackroots[0].next.someInt = 100
-        self.gc.collect()
-        assert self.stackroots[0].next.someInt == 100
-
     def test_pin_malloc_pin(self):
         first_ptr = self.malloc(S)
         first_ptr.someInt = 101

From noreply at buildbot.pypy.org  Sat Jul 26 16:47:47 2014
From: noreply at buildbot.pypy.org (groggi)
Date: Sat, 26 Jul 2014 16:47:47 +0200 (CEST)
Subject: [pypy-commit] pypy gc-incminimark-pinning: wip: implement use case
 "old object points to pinned object".
Message-ID: <20140726144747.BA4B11C050D@cobra.cs.uni-duesseldorf.de>

Author: Gregor Wegberg 
Branch: gc-incminimark-pinning
Changeset: r72522:754f9578ee1a
Date: 2014-07-02 15:15 +0200
http://bitbucket.org/pypy/pypy/changeset/754f9578ee1a/

Log:	wip: implement use case "old object points to pinned object".

	This is very much a work in progress commit. Still a lot to check,
	refactor and document.

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -380,6 +380,8 @@
         # Counter tracking how many pinned objects currently reside inside
         # the nursery.
         self.pinned_objects_in_nursery = 0
+        # TTT XXX
+        self.pinned_objects_keep_alive = self.AddressStack()
         #
         # Allocate a nursery.  In case of auto_nursery_size, start by
         # allocating a very small nursery, enough to do things like look
@@ -1601,6 +1603,15 @@
         # them or make them old.
         if self.young_rawmalloced_objects:
             self.free_young_rawmalloced_objects()
+
+        #
+        # In case we have to keep some pinned objects alive, add them
+        # to 'surviving_pinned_objects'. Such a case comes up if an old
+        # object references a pinned young one (pinned object inside
+        # the nursery). See '_trace_drag_out' for more details.
+        if self.pinned_objects_keep_alive.non_empty():
+            self.pinned_objects_keep_alive.foreach(
+                self._populate_to_surviving_pinned_objects, None)
         #
         # All live nursery objects are out of the nursery or pinned inside
         # the nursery.  Create nursery barriers to protect the pinned object,
@@ -1620,6 +1631,7 @@
             # clear the arena between the last pinned object (or arena start)
             # and the pinned object
             pinned_obj_size = llarena.getfakearenaaddress(cur) - prev
+            debug_print("before A")
             llarena.arena_reset(prev, pinned_obj_size, 2)
             #
             # clean up object's flags
@@ -1634,6 +1646,7 @@
                 (size_gc_header + self.get_size(obj))
         #
         # reset everything after the last pinned object till the end of the arena
+        debug_print("before B")
         llarena.arena_reset(prev, self.nursery_real_top - prev, 0)
         #
         # We assume that there are only a few pinned objects. Therefore, if there
@@ -1644,6 +1657,7 @@
         if prev - self.nursery >= self.nursery_cleanup:
             nursery_barriers.append(prev)
         else:
+            debug_print("before C")
             llarena.arena_reset(prev, self.nursery_cleanup, 2)
             nursery_barriers.append(prev + self.nursery_cleanup)
         #
@@ -1665,6 +1679,12 @@
         #
         debug_stop("gc-minor")
 
+    def _populate_to_surviving_pinned_objects(self, obj, ignored):
+        self.surviving_pinned_objects.append(obj)
+        # we have to update the counter each time, because it was set to 0
+        # at the start of the *minor* collection. The 'obj' survives
+        # *major* collections and therefore also multiple minor collections.
+        self.pinned_objects_in_nursery += 1
 
     def collect_roots_in_nursery(self):
         # we don't need to trace prebuilt GcStructs during a minor collect:
@@ -1676,6 +1696,7 @@
             callback = IncrementalMiniMarkGC._trace_drag_out1_marking_phase
         else:
             callback = IncrementalMiniMarkGC._trace_drag_out1
+        #
         self.root_walker.walk_roots(
             callback,     # stack roots
             callback,     # static in prebuilt non-gc
@@ -1772,7 +1793,7 @@
         """obj must not be in the nursery.  This copies all the
         young objects it references out of the nursery.
         """
-        self.trace(obj, self._trace_drag_out, None)
+        self.trace(obj, self._trace_drag_out, obj)
 
     def trace_and_drag_out_of_nursery_partial(self, obj, start, stop):
         """Like trace_and_drag_out_of_nursery(), but limited to the array
@@ -1801,13 +1822,18 @@
         if not self.header(obj).tid & GCFLAG_VISITED:
             self.more_objects_to_trace.append(obj)
 
-    def _trace_drag_out(self, root, ignored):
+    def _trace_drag_out(self, root, parent):
+        # 'parent' is only set if we visit a pinned objects that is referenced
+        # by an other object. This is used to handle pinned object specially in
+        # such a case.
         obj = root.address[0]
         #print '_trace_drag_out(%x: %r)' % (hash(obj.ptr._obj), obj)
+        debug_print('_trace_drag_out(%x: %r)' % (hash(obj.ptr._obj), obj))
         #
         # If 'obj' is not in the nursery, nothing to change -- expect
         # that we must set GCFLAG_VISITED_RMY on young raw-malloced objects.
         if not self.is_in_nursery(obj):
+            debug_print("\tnot in nursery")
             # cache usage trade-off: I think that it is a better idea to
             # check if 'obj' is in young_rawmalloced_objects with an access
             # to this (small) dictionary, rather than risk a lot of cache
@@ -1818,6 +1844,7 @@
                 self._visit_young_rawmalloced_object(obj)
             return
         #
+        debug_print("\tin nursery")
         size_gc_header = self.gcheaderbuilder.size_gc_header
         if self.header(obj).tid & (GCFLAG_HAS_SHADOW | GCFLAG_PINNED) == 0:
             #
@@ -1841,14 +1868,29 @@
             if hdr.tid & GCFLAG_VISITED:
                 # already visited and keeping track of the object
                 return
+            #
+            if parent:
+                # pinned object is referenced by an other object.
+                # We must keep the pinned object alive between
+                # major collections for the case that an old
+                # object references the young pinned object as
+                # the old one is only visited every major collection.
+                debug_print("-> added to keep alive:", obj)
+                debug_print("-> ... because of parent:", parent)
+                self.pinned_objects_keep_alive.append(obj - size_gc_header)
+                self.write_barrier(parent)
+            else:
+                debug_print("-> usual pinned object")
+                self.surviving_pinned_objects.append(
+                    llarena.getfakearenaaddress(obj - size_gc_header))
+                self.pinned_objects_in_nursery += 1
+            #
             hdr.tid |= GCFLAG_VISITED
+            debug_print("\tstate:", self.gc_state)
             # XXX add additional checks for unsupported pinned objects (groggi)
             # XXX implement unsupported object types with pinning
             ll_assert(not self.header(obj).tid & GCFLAG_HAS_CARDS,
                 "pinned object with GCFLAG_HAS_CARDS not supported")
-            self.surviving_pinned_objects.append(
-                llarena.getfakearenaaddress(obj - size_gc_header))
-            self.pinned_objects_in_nursery += 1
             return
         else:
             # First visit to an object that has already a shadow.
@@ -2017,6 +2059,8 @@
             self.more_objects_to_trace = self.AddressStack()
             #END SCANNING
         elif self.gc_state == STATE_MARKING:
+            self.pinned_objects_keep_alive.delete()
+            self.pinned_objects_keep_alive = self.AddressStack()
             debug_print("number of objects to mark",
                         self.objects_to_trace.length(),
                         "plus",
@@ -2144,6 +2188,8 @@
         else:
             pass #XXX which exception to raise here. Should be unreachable.
 
+        debug_print("BLUB:", self.pinned_objects_keep_alive.length())
+
         debug_print("stopping, now in gc state: ", GC_STATES[self.gc_state])
         debug_stop("gc-collect-step")
 
@@ -2210,7 +2256,7 @@
         #
         # Add the roots from the other sources.
         self.root_walker.walk_roots(
-            IncrementalMiniMarkGC._collect_ref_stk, # stack roots
+            IncrementalMiniMarkGC._collect_ref_stk2, # stack roots
             IncrementalMiniMarkGC._collect_ref_stk, # static in prebuilt non-gc structures
             None)   # we don't need the static in all prebuilt gc objects
         #
@@ -2228,6 +2274,17 @@
     def _collect_obj(obj, objects_to_trace):
         objects_to_trace.append(obj)
 
+    def _collect_ref_stk2(self, root):
+        obj = root.address[0]
+        llop.debug_nonnull_pointer(lltype.Void, obj)
+        hdr = self.header(obj)
+        if hdr.tid & GCFLAG_PINNED:
+            # XXX think really hard about this (groggi):
+            # is a pinned object found on stack already
+            # being traced?
+            return
+        self.objects_to_trace.append(obj)
+
     def _collect_ref_stk(self, root):
         obj = root.address[0]
         llop.debug_nonnull_pointer(lltype.Void, obj)
@@ -2251,6 +2308,7 @@
         return size_to_track
 
     def visit(self, obj):
+        debug_print("visit:", obj)
         #
         # 'obj' is a live object.  Check GCFLAG_VISITED to know if we
         # have already seen it before.
@@ -2261,9 +2319,15 @@
         # flag set, then the object should be in 'prebuilt_root_objects',
         # and the GCFLAG_VISITED will be reset at the end of the
         # collection.
+        #
+        # XXX pinned object case doc (groggi)
         hdr = self.header(obj)
-        if hdr.tid & (GCFLAG_VISITED | GCFLAG_NO_HEAP_PTRS | GCFLAG_PINNED):
-            # XXX ^^^ update doc in any way because of GCFLAG_PINNED addition? (groggi)
+        if hdr.tid & GCFLAG_PINNED:
+            if self.gc_state == STATE_MARKING:
+                debug_print("STATE MARKING AND PINNED, ADD")
+                self.pinned_objects_keep_alive.append(obj - self.gcheaderbuilder.size_gc_header)
+            return 0
+        elif hdr.tid & (GCFLAG_VISITED | GCFLAG_NO_HEAP_PTRS):
             return 0
         #
         # It's the first time.  We set the flag VISITED.  The trick is

From noreply at buildbot.pypy.org  Sat Jul 26 16:47:48 2014
From: noreply at buildbot.pypy.org (groggi)
Date: Sat, 26 Jul 2014 16:47:48 +0200 (CEST)
Subject: [pypy-commit] pypy gc-incminimark-pinning: modify tests for new
 pinning implementation
Message-ID: <20140726144748.DD31F1C050D@cobra.cs.uni-duesseldorf.de>

Author: Gregor Wegberg 
Branch: gc-incminimark-pinning
Changeset: r72523:0bbc22db2b13
Date: 2014-07-13 12:42 +0200
http://bitbucket.org/pypy/pypy/changeset/0bbc22db2b13/

Log:	modify tests for new pinning implementation

	pinned objects will stay inside the nursery as long as a major
	collection

diff --git a/rpython/memory/gc/test/test_object_pinning.py b/rpython/memory/gc/test/test_object_pinning.py
--- a/rpython/memory/gc/test/test_object_pinning.py
+++ b/rpython/memory/gc/test/test_object_pinning.py
@@ -46,7 +46,8 @@
 class TestIncminimark(PinningGCTest):
     from rpython.memory.gc.incminimark import IncrementalMiniMarkGC as GCClass
 
-    def test_simple_pin(self):
+    def test_simple_pin_stack(self):
+        # create object, pin it and point from stackroots to it
         ptr = self.malloc(S)
         ptr.someInt = 100
         self.stackroots.append(ptr)
@@ -59,26 +60,33 @@
         assert self.gc.is_in_nursery(adr)
         assert ptr.someInt == 100
 
-    def test_simple_pin_unpin(self):
+    def test_simple_pin_unpin_stack(self):
         ptr = self.malloc(S)
         ptr.someInt = 100
+        
         self.stackroots.append(ptr)
+        
         adr = llmemory.cast_ptr_to_adr(ptr)
-        # check if pin worked
         assert self.gc.pin(adr)
+        
         self.gc.collect()
+
         assert self.gc.is_in_nursery(adr)
         assert ptr.someInt == 100
+        
         # unpin and check if object is gone from nursery
         self.gc.unpin(adr)
         self.gc.collect()
         py.test.raises(RuntimeError, 'ptr.someInt')
+        
+        # check if we object is still accessible
         ptr_old = self.stackroots[0]
+        assert not self.gc.is_in_nursery(llmemory.cast_ptr_to_adr(ptr_old))
         assert ptr_old.someInt == 100
 
     def test_pin_referenced_from_stackroot_young(self):
         #
-        # create both objects and reference the pinned one
+        # create two objects and reference the pinned one
         # from the one that will be moved out of the
         # nursery.
         root_ptr = self.malloc(S)
@@ -114,7 +122,7 @@
             assert "freed" in str(ex)
 
     def test_old_points_to_pinned(self):
-        # Test if we handle the case that an old object can point
+        # Test if we handle the case that an already old object can point
         # to a pinned object and keeps the pinned object alive by
         # that.
         #
@@ -123,7 +131,8 @@
         old_ptr.someInt = 999
         self.stackroots.append(old_ptr)
         self.gc.collect()
-        assert not self.gc.is_in_nursery(llmemory.cast_ptr_to_adr(self.stackroots[0]))
+        assert not self.gc.is_in_nursery(
+                llmemory.cast_ptr_to_adr(self.stackroots[0]))
         #
         # create the young pinned object and attach it to the old object
         pinned_ptr = self.malloc(S)
@@ -178,11 +187,11 @@
         ptr.someInt = 100
         assert self.gc.pin(adr)
         self.gc.id(ptr) # allocate shadow
-        self.gc.minor_collection()
+        self.gc.collect()
         assert self.gc.is_in_nursery(adr)
         assert ptr.someInt == 100
         self.gc.unpin(adr)
-        self.gc.minor_collection() # move to shadow
+        self.gc.collect() # move to shadow
         adr = llmemory.cast_ptr_to_adr(self.stackroots[0])
         assert not self.gc.is_in_nursery(adr)
 
@@ -193,11 +202,11 @@
         ptr.someInt = 100
         assert self.gc.pin(adr)
         self.gc.identityhash(ptr) # allocate shadow
-        self.gc.minor_collection()
+        self.gc.collect()
         assert self.gc.is_in_nursery(adr)
         assert ptr.someInt == 100
         self.gc.unpin(adr)
-        self.gc.minor_collection() # move to shadow
+        self.gc.collect() # move to shadow
         adr = llmemory.cast_ptr_to_adr(self.stackroots[0])
         assert not self.gc.is_in_nursery(adr)
 
@@ -270,7 +279,7 @@
         # +- nursery_free
         # +- nursery_top
         #
-        self.gc.minor_collection()
+        self.gc.collect()
 
         assert self.gc.nursery_free == self.gc.nursery_top
         assert self.gc.nursery_top == self.gc.nursery
@@ -311,7 +320,7 @@
         # +- nursery_free
         #
         self.gc.unpin(adr1)
-        self.gc.minor_collection()
+        self.gc.collect()
 
         assert self.gc.nursery_free == self.gc.nursery
         assert self.gc.nursery_top > self.gc.nursery_free
@@ -353,7 +362,7 @@
         #
         self.gc.unpin(adr1)
         self.gc.unpin(adr2)
-        self.gc.minor_collection()
+        self.gc.collect()
 
         assert self.gc.nursery_free == self.gc.nursery
         assert self.gc.nursery_free < self.gc.nursery_top
@@ -413,7 +422,7 @@
         self.gc.unpin(adr1)
         self.gc.unpin(adr2)
         self.gc.unpin(adr3)
-        self.gc.minor_collection()
+        self.gc.collect()
 
         assert self.gc.nursery_free == self.gc.nursery
         # the following assert is important: make sure that
@@ -438,7 +447,7 @@
         assert self.gc.pin(llmemory.cast_ptr_to_adr(ptr_stackroot_2))
         assert self.gc.pinned_objects_in_nursery == 3
 
-        self.gc.minor_collection()
+        self.gc.collect()
         # now the one not on the stack should be gone.
         assert self.gc.pinned_objects_in_nursery == 2
         assert ptr_stackroot_1.someInt == 100
@@ -466,7 +475,7 @@
         # there were some bugs regarding the 'arena_reset()' calls at
         # the end of the minor collection.  This test brought them to light.
         self.fill_nursery_with_pinned_objects()
-        self.gc.minor_collection()
+        self.gc.collect()
 
     def test_pinning_limit(self):
         for instance_nr in xrange(self.gc.max_number_of_pinned_objects):

From noreply at buildbot.pypy.org  Sat Jul 26 16:47:50 2014
From: noreply at buildbot.pypy.org (groggi)
Date: Sat, 26 Jul 2014 16:47:50 +0200 (CEST)
Subject: [pypy-commit] pypy gc-incminimark-pinning: additional test to cover
 more cases of referencing a pinned object
Message-ID: <20140726144750.163361C050D@cobra.cs.uni-duesseldorf.de>

Author: Gregor Wegberg 
Branch: gc-incminimark-pinning
Changeset: r72524:513744aa7a90
Date: 2014-07-13 13:05 +0200
http://bitbucket.org/pypy/pypy/changeset/513744aa7a90/

Log:	additional test to cover more cases of referencing a pinned object

diff --git a/rpython/memory/gc/test/test_object_pinning.py b/rpython/memory/gc/test/test_object_pinning.py
--- a/rpython/memory/gc/test/test_object_pinning.py
+++ b/rpython/memory/gc/test/test_object_pinning.py
@@ -149,6 +149,29 @@
         self.gc.collect()
         assert self.stackroots[0].next.someInt == 100
 
+    def test_old_and_stackroots_point_to_pinned(self):
+        # In this test case we point to a pinned object from an old object
+        # *and* from the stackroots
+        old_ptr = self.malloc(S)
+        old_ptr.someInt = 999
+        self.stackroots.append(old_ptr)
+        self.gc.collect() # old_ptr is now old
+        old_ptr = self.stackroots[0]
+        assert not self.gc.is_in_nursery(llmemory.cast_ptr_to_adr(old_ptr))
+
+        pinned_ptr = self.malloc(S)
+        pinned_ptr.someInt = 111
+        assert self.gc.pin(llmemory.cast_ptr_to_adr(pinned_ptr))
+
+        self.stackroots.append(pinned_ptr)
+        self.write(old_ptr, 'next', pinned_ptr)
+        
+        self.gc.collect()
+        # done with preparation. do some basic checks
+        assert self.gc.is_in_nursery(llmemory.cast_ptr_to_adr(pinned_ptr))
+        assert pinned_ptr.someInt == 111
+        assert self.stackroots[0].next == pinned_ptr
+
     def test_pin_old(self):
         ptr = self.malloc(S)
         ptr.someInt = 100

From noreply at buildbot.pypy.org  Sat Jul 26 16:47:51 2014
From: noreply at buildbot.pypy.org (groggi)
Date: Sat, 26 Jul 2014 16:47:51 +0200 (CEST)
Subject: [pypy-commit] pypy gc-incminimark-pinning: generalize previously
 added test and use it to get two tests
Message-ID: <20140726144751.280BB1C050D@cobra.cs.uni-duesseldorf.de>

Author: Gregor Wegberg 
Branch: gc-incminimark-pinning
Changeset: r72525:4aa9c7ae56dd
Date: 2014-07-13 13:13 +0200
http://bitbucket.org/pypy/pypy/changeset/4aa9c7ae56dd/

Log:	generalize previously added test and use it to get two tests

	previouse test added in 513744aa7a905300b7389cd1c3724c0150762ee4

diff --git a/rpython/memory/gc/test/test_object_pinning.py b/rpython/memory/gc/test/test_object_pinning.py
--- a/rpython/memory/gc/test/test_object_pinning.py
+++ b/rpython/memory/gc/test/test_object_pinning.py
@@ -149,22 +149,25 @@
         self.gc.collect()
         assert self.stackroots[0].next.someInt == 100
 
-    def test_old_and_stackroots_point_to_pinned(self):
+    def not_pinned_and_stackroots_point_to_pinned(self, make_old):
         # In this test case we point to a pinned object from an old object
         # *and* from the stackroots
-        old_ptr = self.malloc(S)
-        old_ptr.someInt = 999
-        self.stackroots.append(old_ptr)
-        self.gc.collect() # old_ptr is now old
-        old_ptr = self.stackroots[0]
-        assert not self.gc.is_in_nursery(llmemory.cast_ptr_to_adr(old_ptr))
+        obj_ptr = self.malloc(S)
+        obj_ptr.someInt = 999
+        self.stackroots.append(obj_ptr)
+        if make_old:
+            self.gc.collect() # old_ptr is now old
+            obj_ptr = self.stackroots[0]
+            assert not self.gc.is_in_nursery(llmemory.cast_ptr_to_adr(obj_ptr))
+        else:
+            assert self.gc.is_in_nursery(llmemory.cast_ptr_to_adr(obj_ptr))
 
         pinned_ptr = self.malloc(S)
         pinned_ptr.someInt = 111
         assert self.gc.pin(llmemory.cast_ptr_to_adr(pinned_ptr))
 
         self.stackroots.append(pinned_ptr)
-        self.write(old_ptr, 'next', pinned_ptr)
+        self.write(obj_ptr, 'next', pinned_ptr)
         
         self.gc.collect()
         # done with preparation. do some basic checks
@@ -172,6 +175,12 @@
         assert pinned_ptr.someInt == 111
         assert self.stackroots[0].next == pinned_ptr
 
+    def test_old_and_stackroots_point_to_pinned(self):
+        self.not_pinned_and_stackroots_point_to_pinned(make_old=True)
+
+    def test_young_and_stackroots_point_to_pinned(self):
+        self.not_pinned_and_stackroots_point_to_pinned(make_old=False)
+
     def test_pin_old(self):
         ptr = self.malloc(S)
         ptr.someInt = 100

From noreply at buildbot.pypy.org  Sat Jul 26 16:47:52 2014
From: noreply at buildbot.pypy.org (groggi)
Date: Sat, 26 Jul 2014 16:47:52 +0200 (CEST)
Subject: [pypy-commit] pypy gc-incminimark-pinning: unnecessary comment
	removed
Message-ID: <20140726144752.9A61A1C050D@cobra.cs.uni-duesseldorf.de>

Author: Gregor Wegberg 
Branch: gc-incminimark-pinning
Changeset: r72526:5868377b4b39
Date: 2014-07-13 13:48 +0200
http://bitbucket.org/pypy/pypy/changeset/5868377b4b39/

Log:	unnecessary comment removed

diff --git a/rpython/memory/gc/test/test_object_pinning.py b/rpython/memory/gc/test/test_object_pinning.py
--- a/rpython/memory/gc/test/test_object_pinning.py
+++ b/rpython/memory/gc/test/test_object_pinning.py
@@ -156,7 +156,7 @@
         obj_ptr.someInt = 999
         self.stackroots.append(obj_ptr)
         if make_old:
-            self.gc.collect() # old_ptr is now old
+            self.gc.collect()
             obj_ptr = self.stackroots[0]
             assert not self.gc.is_in_nursery(llmemory.cast_ptr_to_adr(obj_ptr))
         else:

From noreply at buildbot.pypy.org  Sat Jul 26 16:47:53 2014
From: noreply at buildbot.pypy.org (groggi)
Date: Sat, 26 Jul 2014 16:47:53 +0200 (CEST)
Subject: [pypy-commit] pypy gc-incminimark-pinning: Backout changeset
 754f9578ee1aee71b29f8a4ea6b23a60f89be3c3
Message-ID: <20140726144753.B45231C050D@cobra.cs.uni-duesseldorf.de>

Author: Gregor Wegberg 
Branch: gc-incminimark-pinning
Changeset: r72527:a4e98d2f48ff
Date: 2014-07-13 14:23 +0200
http://bitbucket.org/pypy/pypy/changeset/a4e98d2f48ff/

Log:	Backout changeset 754f9578ee1aee71b29f8a4ea6b23a60f89be3c3

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -380,8 +380,6 @@
         # Counter tracking how many pinned objects currently reside inside
         # the nursery.
         self.pinned_objects_in_nursery = 0
-        # TTT XXX
-        self.pinned_objects_keep_alive = self.AddressStack()
         #
         # Allocate a nursery.  In case of auto_nursery_size, start by
         # allocating a very small nursery, enough to do things like look
@@ -1603,15 +1601,6 @@
         # them or make them old.
         if self.young_rawmalloced_objects:
             self.free_young_rawmalloced_objects()
-
-        #
-        # In case we have to keep some pinned objects alive, add them
-        # to 'surviving_pinned_objects'. Such a case comes up if an old
-        # object references a pinned young one (pinned object inside
-        # the nursery). See '_trace_drag_out' for more details.
-        if self.pinned_objects_keep_alive.non_empty():
-            self.pinned_objects_keep_alive.foreach(
-                self._populate_to_surviving_pinned_objects, None)
         #
         # All live nursery objects are out of the nursery or pinned inside
         # the nursery.  Create nursery barriers to protect the pinned object,
@@ -1631,7 +1620,6 @@
             # clear the arena between the last pinned object (or arena start)
             # and the pinned object
             pinned_obj_size = llarena.getfakearenaaddress(cur) - prev
-            debug_print("before A")
             llarena.arena_reset(prev, pinned_obj_size, 2)
             #
             # clean up object's flags
@@ -1646,7 +1634,6 @@
                 (size_gc_header + self.get_size(obj))
         #
         # reset everything after the last pinned object till the end of the arena
-        debug_print("before B")
         llarena.arena_reset(prev, self.nursery_real_top - prev, 0)
         #
         # We assume that there are only a few pinned objects. Therefore, if there
@@ -1657,7 +1644,6 @@
         if prev - self.nursery >= self.nursery_cleanup:
             nursery_barriers.append(prev)
         else:
-            debug_print("before C")
             llarena.arena_reset(prev, self.nursery_cleanup, 2)
             nursery_barriers.append(prev + self.nursery_cleanup)
         #
@@ -1679,12 +1665,6 @@
         #
         debug_stop("gc-minor")
 
-    def _populate_to_surviving_pinned_objects(self, obj, ignored):
-        self.surviving_pinned_objects.append(obj)
-        # we have to update the counter each time, because it was set to 0
-        # at the start of the *minor* collection. The 'obj' survives
-        # *major* collections and therefore also multiple minor collections.
-        self.pinned_objects_in_nursery += 1
 
     def collect_roots_in_nursery(self):
         # we don't need to trace prebuilt GcStructs during a minor collect:
@@ -1696,7 +1676,6 @@
             callback = IncrementalMiniMarkGC._trace_drag_out1_marking_phase
         else:
             callback = IncrementalMiniMarkGC._trace_drag_out1
-        #
         self.root_walker.walk_roots(
             callback,     # stack roots
             callback,     # static in prebuilt non-gc
@@ -1793,7 +1772,7 @@
         """obj must not be in the nursery.  This copies all the
         young objects it references out of the nursery.
         """
-        self.trace(obj, self._trace_drag_out, obj)
+        self.trace(obj, self._trace_drag_out, None)
 
     def trace_and_drag_out_of_nursery_partial(self, obj, start, stop):
         """Like trace_and_drag_out_of_nursery(), but limited to the array
@@ -1822,18 +1801,13 @@
         if not self.header(obj).tid & GCFLAG_VISITED:
             self.more_objects_to_trace.append(obj)
 
-    def _trace_drag_out(self, root, parent):
-        # 'parent' is only set if we visit a pinned objects that is referenced
-        # by an other object. This is used to handle pinned object specially in
-        # such a case.
+    def _trace_drag_out(self, root, ignored):
         obj = root.address[0]
         #print '_trace_drag_out(%x: %r)' % (hash(obj.ptr._obj), obj)
-        debug_print('_trace_drag_out(%x: %r)' % (hash(obj.ptr._obj), obj))
         #
         # If 'obj' is not in the nursery, nothing to change -- expect
         # that we must set GCFLAG_VISITED_RMY on young raw-malloced objects.
         if not self.is_in_nursery(obj):
-            debug_print("\tnot in nursery")
             # cache usage trade-off: I think that it is a better idea to
             # check if 'obj' is in young_rawmalloced_objects with an access
             # to this (small) dictionary, rather than risk a lot of cache
@@ -1844,7 +1818,6 @@
                 self._visit_young_rawmalloced_object(obj)
             return
         #
-        debug_print("\tin nursery")
         size_gc_header = self.gcheaderbuilder.size_gc_header
         if self.header(obj).tid & (GCFLAG_HAS_SHADOW | GCFLAG_PINNED) == 0:
             #
@@ -1868,29 +1841,14 @@
             if hdr.tid & GCFLAG_VISITED:
                 # already visited and keeping track of the object
                 return
-            #
-            if parent:
-                # pinned object is referenced by an other object.
-                # We must keep the pinned object alive between
-                # major collections for the case that an old
-                # object references the young pinned object as
-                # the old one is only visited every major collection.
-                debug_print("-> added to keep alive:", obj)
-                debug_print("-> ... because of parent:", parent)
-                self.pinned_objects_keep_alive.append(obj - size_gc_header)
-                self.write_barrier(parent)
-            else:
-                debug_print("-> usual pinned object")
-                self.surviving_pinned_objects.append(
-                    llarena.getfakearenaaddress(obj - size_gc_header))
-                self.pinned_objects_in_nursery += 1
-            #
             hdr.tid |= GCFLAG_VISITED
-            debug_print("\tstate:", self.gc_state)
             # XXX add additional checks for unsupported pinned objects (groggi)
             # XXX implement unsupported object types with pinning
             ll_assert(not self.header(obj).tid & GCFLAG_HAS_CARDS,
                 "pinned object with GCFLAG_HAS_CARDS not supported")
+            self.surviving_pinned_objects.append(
+                llarena.getfakearenaaddress(obj - size_gc_header))
+            self.pinned_objects_in_nursery += 1
             return
         else:
             # First visit to an object that has already a shadow.
@@ -2059,8 +2017,6 @@
             self.more_objects_to_trace = self.AddressStack()
             #END SCANNING
         elif self.gc_state == STATE_MARKING:
-            self.pinned_objects_keep_alive.delete()
-            self.pinned_objects_keep_alive = self.AddressStack()
             debug_print("number of objects to mark",
                         self.objects_to_trace.length(),
                         "plus",
@@ -2188,8 +2144,6 @@
         else:
             pass #XXX which exception to raise here. Should be unreachable.
 
-        debug_print("BLUB:", self.pinned_objects_keep_alive.length())
-
         debug_print("stopping, now in gc state: ", GC_STATES[self.gc_state])
         debug_stop("gc-collect-step")
 
@@ -2256,7 +2210,7 @@
         #
         # Add the roots from the other sources.
         self.root_walker.walk_roots(
-            IncrementalMiniMarkGC._collect_ref_stk2, # stack roots
+            IncrementalMiniMarkGC._collect_ref_stk, # stack roots
             IncrementalMiniMarkGC._collect_ref_stk, # static in prebuilt non-gc structures
             None)   # we don't need the static in all prebuilt gc objects
         #
@@ -2274,17 +2228,6 @@
     def _collect_obj(obj, objects_to_trace):
         objects_to_trace.append(obj)
 
-    def _collect_ref_stk2(self, root):
-        obj = root.address[0]
-        llop.debug_nonnull_pointer(lltype.Void, obj)
-        hdr = self.header(obj)
-        if hdr.tid & GCFLAG_PINNED:
-            # XXX think really hard about this (groggi):
-            # is a pinned object found on stack already
-            # being traced?
-            return
-        self.objects_to_trace.append(obj)
-
     def _collect_ref_stk(self, root):
         obj = root.address[0]
         llop.debug_nonnull_pointer(lltype.Void, obj)
@@ -2308,7 +2251,6 @@
         return size_to_track
 
     def visit(self, obj):
-        debug_print("visit:", obj)
         #
         # 'obj' is a live object.  Check GCFLAG_VISITED to know if we
         # have already seen it before.
@@ -2319,15 +2261,9 @@
         # flag set, then the object should be in 'prebuilt_root_objects',
         # and the GCFLAG_VISITED will be reset at the end of the
         # collection.
-        #
-        # XXX pinned object case doc (groggi)
         hdr = self.header(obj)
-        if hdr.tid & GCFLAG_PINNED:
-            if self.gc_state == STATE_MARKING:
-                debug_print("STATE MARKING AND PINNED, ADD")
-                self.pinned_objects_keep_alive.append(obj - self.gcheaderbuilder.size_gc_header)
-            return 0
-        elif hdr.tid & (GCFLAG_VISITED | GCFLAG_NO_HEAP_PTRS):
+        if hdr.tid & (GCFLAG_VISITED | GCFLAG_NO_HEAP_PTRS | GCFLAG_PINNED):
+            # XXX ^^^ update doc in any way because of GCFLAG_PINNED addition? (groggi)
             return 0
         #
         # It's the first time.  We set the flag VISITED.  The trick is

From noreply at buildbot.pypy.org  Sat Jul 26 16:47:54 2014
From: noreply at buildbot.pypy.org (groggi)
Date: Sat, 26 Jul 2014 16:47:54 +0200 (CEST)
Subject: [pypy-commit] pypy gc-incminimark-pinning: revert
 754f9578ee1aee71b29f8a4ea6b23a60f89be3c3
Message-ID: <20140726144754.CAD501C050D@cobra.cs.uni-duesseldorf.de>

Author: Gregor Wegberg 
Branch: gc-incminimark-pinning
Changeset: r72528:cd9ec650486f
Date: 2014-07-13 14:25 +0200
http://bitbucket.org/pypy/pypy/changeset/cd9ec650486f/

Log:	revert 754f9578ee1aee71b29f8a4ea6b23a60f89be3c3

	time to think first and implement afterwards

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -380,8 +380,6 @@
         # Counter tracking how many pinned objects currently reside inside
         # the nursery.
         self.pinned_objects_in_nursery = 0
-        # TTT XXX
-        self.pinned_objects_keep_alive = self.AddressStack()
         #
         # Allocate a nursery.  In case of auto_nursery_size, start by
         # allocating a very small nursery, enough to do things like look
@@ -1603,15 +1601,6 @@
         # them or make them old.
         if self.young_rawmalloced_objects:
             self.free_young_rawmalloced_objects()
-
-        #
-        # In case we have to keep some pinned objects alive, add them
-        # to 'surviving_pinned_objects'. Such a case comes up if an old
-        # object references a pinned young one (pinned object inside
-        # the nursery). See '_trace_drag_out' for more details.
-        if self.pinned_objects_keep_alive.non_empty():
-            self.pinned_objects_keep_alive.foreach(
-                self._populate_to_surviving_pinned_objects, None)
         #
         # All live nursery objects are out of the nursery or pinned inside
         # the nursery.  Create nursery barriers to protect the pinned object,
@@ -1631,7 +1620,6 @@
             # clear the arena between the last pinned object (or arena start)
             # and the pinned object
             pinned_obj_size = llarena.getfakearenaaddress(cur) - prev
-            debug_print("before A")
             llarena.arena_reset(prev, pinned_obj_size, 2)
             #
             # clean up object's flags
@@ -1646,7 +1634,6 @@
                 (size_gc_header + self.get_size(obj))
         #
         # reset everything after the last pinned object till the end of the arena
-        debug_print("before B")
         llarena.arena_reset(prev, self.nursery_real_top - prev, 0)
         #
         # We assume that there are only a few pinned objects. Therefore, if there
@@ -1657,7 +1644,6 @@
         if prev - self.nursery >= self.nursery_cleanup:
             nursery_barriers.append(prev)
         else:
-            debug_print("before C")
             llarena.arena_reset(prev, self.nursery_cleanup, 2)
             nursery_barriers.append(prev + self.nursery_cleanup)
         #
@@ -1679,12 +1665,6 @@
         #
         debug_stop("gc-minor")
 
-    def _populate_to_surviving_pinned_objects(self, obj, ignored):
-        self.surviving_pinned_objects.append(obj)
-        # we have to update the counter each time, because it was set to 0
-        # at the start of the *minor* collection. The 'obj' survives
-        # *major* collections and therefore also multiple minor collections.
-        self.pinned_objects_in_nursery += 1
 
     def collect_roots_in_nursery(self):
         # we don't need to trace prebuilt GcStructs during a minor collect:
@@ -1696,7 +1676,6 @@
             callback = IncrementalMiniMarkGC._trace_drag_out1_marking_phase
         else:
             callback = IncrementalMiniMarkGC._trace_drag_out1
-        #
         self.root_walker.walk_roots(
             callback,     # stack roots
             callback,     # static in prebuilt non-gc
@@ -1793,7 +1772,7 @@
         """obj must not be in the nursery.  This copies all the
         young objects it references out of the nursery.
         """
-        self.trace(obj, self._trace_drag_out, obj)
+        self.trace(obj, self._trace_drag_out, None)
 
     def trace_and_drag_out_of_nursery_partial(self, obj, start, stop):
         """Like trace_and_drag_out_of_nursery(), but limited to the array
@@ -1822,18 +1801,13 @@
         if not self.header(obj).tid & GCFLAG_VISITED:
             self.more_objects_to_trace.append(obj)
 
-    def _trace_drag_out(self, root, parent):
-        # 'parent' is only set if we visit a pinned objects that is referenced
-        # by an other object. This is used to handle pinned object specially in
-        # such a case.
+    def _trace_drag_out(self, root, ignored):
         obj = root.address[0]
         #print '_trace_drag_out(%x: %r)' % (hash(obj.ptr._obj), obj)
-        debug_print('_trace_drag_out(%x: %r)' % (hash(obj.ptr._obj), obj))
         #
         # If 'obj' is not in the nursery, nothing to change -- expect
         # that we must set GCFLAG_VISITED_RMY on young raw-malloced objects.
         if not self.is_in_nursery(obj):
-            debug_print("\tnot in nursery")
             # cache usage trade-off: I think that it is a better idea to
             # check if 'obj' is in young_rawmalloced_objects with an access
             # to this (small) dictionary, rather than risk a lot of cache
@@ -1844,7 +1818,6 @@
                 self._visit_young_rawmalloced_object(obj)
             return
         #
-        debug_print("\tin nursery")
         size_gc_header = self.gcheaderbuilder.size_gc_header
         if self.header(obj).tid & (GCFLAG_HAS_SHADOW | GCFLAG_PINNED) == 0:
             #
@@ -1868,29 +1841,14 @@
             if hdr.tid & GCFLAG_VISITED:
                 # already visited and keeping track of the object
                 return
-            #
-            if parent:
-                # pinned object is referenced by an other object.
-                # We must keep the pinned object alive between
-                # major collections for the case that an old
-                # object references the young pinned object as
-                # the old one is only visited every major collection.
-                debug_print("-> added to keep alive:", obj)
-                debug_print("-> ... because of parent:", parent)
-                self.pinned_objects_keep_alive.append(obj - size_gc_header)
-                self.write_barrier(parent)
-            else:
-                debug_print("-> usual pinned object")
-                self.surviving_pinned_objects.append(
-                    llarena.getfakearenaaddress(obj - size_gc_header))
-                self.pinned_objects_in_nursery += 1
-            #
             hdr.tid |= GCFLAG_VISITED
-            debug_print("\tstate:", self.gc_state)
             # XXX add additional checks for unsupported pinned objects (groggi)
             # XXX implement unsupported object types with pinning
             ll_assert(not self.header(obj).tid & GCFLAG_HAS_CARDS,
                 "pinned object with GCFLAG_HAS_CARDS not supported")
+            self.surviving_pinned_objects.append(
+                llarena.getfakearenaaddress(obj - size_gc_header))
+            self.pinned_objects_in_nursery += 1
             return
         else:
             # First visit to an object that has already a shadow.
@@ -2059,8 +2017,6 @@
             self.more_objects_to_trace = self.AddressStack()
             #END SCANNING
         elif self.gc_state == STATE_MARKING:
-            self.pinned_objects_keep_alive.delete()
-            self.pinned_objects_keep_alive = self.AddressStack()
             debug_print("number of objects to mark",
                         self.objects_to_trace.length(),
                         "plus",
@@ -2188,8 +2144,6 @@
         else:
             pass #XXX which exception to raise here. Should be unreachable.
 
-        debug_print("BLUB:", self.pinned_objects_keep_alive.length())
-
         debug_print("stopping, now in gc state: ", GC_STATES[self.gc_state])
         debug_stop("gc-collect-step")
 
@@ -2256,7 +2210,7 @@
         #
         # Add the roots from the other sources.
         self.root_walker.walk_roots(
-            IncrementalMiniMarkGC._collect_ref_stk2, # stack roots
+            IncrementalMiniMarkGC._collect_ref_stk, # stack roots
             IncrementalMiniMarkGC._collect_ref_stk, # static in prebuilt non-gc structures
             None)   # we don't need the static in all prebuilt gc objects
         #
@@ -2274,17 +2228,6 @@
     def _collect_obj(obj, objects_to_trace):
         objects_to_trace.append(obj)
 
-    def _collect_ref_stk2(self, root):
-        obj = root.address[0]
-        llop.debug_nonnull_pointer(lltype.Void, obj)
-        hdr = self.header(obj)
-        if hdr.tid & GCFLAG_PINNED:
-            # XXX think really hard about this (groggi):
-            # is a pinned object found on stack already
-            # being traced?
-            return
-        self.objects_to_trace.append(obj)
-
     def _collect_ref_stk(self, root):
         obj = root.address[0]
         llop.debug_nonnull_pointer(lltype.Void, obj)
@@ -2308,7 +2251,6 @@
         return size_to_track
 
     def visit(self, obj):
-        debug_print("visit:", obj)
         #
         # 'obj' is a live object.  Check GCFLAG_VISITED to know if we
         # have already seen it before.
@@ -2319,15 +2261,9 @@
         # flag set, then the object should be in 'prebuilt_root_objects',
         # and the GCFLAG_VISITED will be reset at the end of the
         # collection.
-        #
-        # XXX pinned object case doc (groggi)
         hdr = self.header(obj)
-        if hdr.tid & GCFLAG_PINNED:
-            if self.gc_state == STATE_MARKING:
-                debug_print("STATE MARKING AND PINNED, ADD")
-                self.pinned_objects_keep_alive.append(obj - self.gcheaderbuilder.size_gc_header)
-            return 0
-        elif hdr.tid & (GCFLAG_VISITED | GCFLAG_NO_HEAP_PTRS):
+        if hdr.tid & (GCFLAG_VISITED | GCFLAG_NO_HEAP_PTRS | GCFLAG_PINNED):
+            # XXX ^^^ update doc in any way because of GCFLAG_PINNED addition? (groggi)
             return 0
         #
         # It's the first time.  We set the flag VISITED.  The trick is

From noreply at buildbot.pypy.org  Sat Jul 26 16:47:55 2014
From: noreply at buildbot.pypy.org (groggi)
Date: Sat, 26 Jul 2014 16:47:55 +0200 (CEST)
Subject: [pypy-commit] pypy gc-incminimark-pinning: generalize tests. tests
 pinning for minor collections and major ones now
Message-ID: <20140726144755.DD2201C050D@cobra.cs.uni-duesseldorf.de>

Author: Gregor Wegberg 
Branch: gc-incminimark-pinning
Changeset: r72529:daf002afa713
Date: 2014-07-13 14:52 +0200
http://bitbucket.org/pypy/pypy/changeset/daf002afa713/

Log:	generalize tests. tests pinning for minor collections and major ones
	now

diff --git a/rpython/memory/gc/test/test_object_pinning.py b/rpython/memory/gc/test/test_object_pinning.py
--- a/rpython/memory/gc/test/test_object_pinning.py
+++ b/rpython/memory/gc/test/test_object_pinning.py
@@ -46,7 +46,7 @@
 class TestIncminimark(PinningGCTest):
     from rpython.memory.gc.incminimark import IncrementalMiniMarkGC as GCClass
 
-    def test_simple_pin_stack(self):
+    def simple_pin_stack(self, collect_func):
         # create object, pin it and point from stackroots to it
         ptr = self.malloc(S)
         ptr.someInt = 100
@@ -55,12 +55,18 @@
         adr = llmemory.cast_ptr_to_adr(ptr)
         assert self.gc.pin(adr)
 
-        self.gc.collect()
+        collect_func()
         
         assert self.gc.is_in_nursery(adr)
         assert ptr.someInt == 100
 
-    def test_simple_pin_unpin_stack(self):
+    def test_simple_pin_stack_full_collect(self):
+        self.simple_pin_stack(self.gc.collect)
+
+    def test_simple_pin_stack_minor_collect(self):
+        self.simple_pin_stack(self.gc.minor_collection)
+
+    def simple_pin_unpin_stack(self, collect_func):
         ptr = self.malloc(S)
         ptr.someInt = 100
         
@@ -69,14 +75,14 @@
         adr = llmemory.cast_ptr_to_adr(ptr)
         assert self.gc.pin(adr)
         
-        self.gc.collect()
+        collect_func()
 
         assert self.gc.is_in_nursery(adr)
         assert ptr.someInt == 100
         
         # unpin and check if object is gone from nursery
         self.gc.unpin(adr)
-        self.gc.collect()
+        collect_func()
         py.test.raises(RuntimeError, 'ptr.someInt')
         
         # check if we object is still accessible
@@ -84,6 +90,12 @@
         assert not self.gc.is_in_nursery(llmemory.cast_ptr_to_adr(ptr_old))
         assert ptr_old.someInt == 100
 
+    def test_simple_pin_unpin_stack_full_collect(self):
+        self.simple_pin_unpin_stack(self.gc.collect)
+
+    def test_simple_pin_unpin_stack_minor_collect(self):
+        self.simple_pin_unpin_stack(self.gc.minor_collection)
+
     def test_pin_referenced_from_stackroot_young(self):
         #
         # create two objects and reference the pinned one
@@ -150,7 +162,7 @@
         assert self.stackroots[0].next.someInt == 100
 
     def not_pinned_and_stackroots_point_to_pinned(self, make_old):
-        # In this test case we point to a pinned object from an old object
+        # In this test case we point to a pinned object from an (old) object
         # *and* from the stackroots
         obj_ptr = self.malloc(S)
         obj_ptr.someInt = 999

From noreply at buildbot.pypy.org  Sat Jul 26 16:47:57 2014
From: noreply at buildbot.pypy.org (groggi)
Date: Sat, 26 Jul 2014 16:47:57 +0200 (CEST)
Subject: [pypy-commit] pypy gc-incminimark-pinning: shadow tests now tested
 with minor and full major collections.
Message-ID: <20140726144757.0AFF11C050D@cobra.cs.uni-duesseldorf.de>

Author: Gregor Wegberg 
Branch: gc-incminimark-pinning
Changeset: r72530:2b62d7537a85
Date: 2014-07-26 15:46 +0200
http://bitbucket.org/pypy/pypy/changeset/2b62d7537a85/

Log:	shadow tests now tested with minor and full major collections.

diff --git a/rpython/memory/gc/test/test_object_pinning.py b/rpython/memory/gc/test/test_object_pinning.py
--- a/rpython/memory/gc/test/test_object_pinning.py
+++ b/rpython/memory/gc/test/test_object_pinning.py
@@ -224,36 +224,48 @@
         assert first_ptr.someInt == 101
         assert second_ptr.someInt == 102
 
-    def test_pin_shadow_1(self):
+    def pin_shadow_1(self, collect_func):
         ptr = self.malloc(S)
         adr = llmemory.cast_ptr_to_adr(ptr)
         self.stackroots.append(ptr)
         ptr.someInt = 100
         assert self.gc.pin(adr)
         self.gc.id(ptr) # allocate shadow
-        self.gc.collect()
+        collect_func()
         assert self.gc.is_in_nursery(adr)
         assert ptr.someInt == 100
         self.gc.unpin(adr)
-        self.gc.collect() # move to shadow
+        collect_func() # move to shadow
         adr = llmemory.cast_ptr_to_adr(self.stackroots[0])
         assert not self.gc.is_in_nursery(adr)
 
-    def test_pin_shadow_2(self):
+    def test_pin_shadow_1_minor(self):
+        self.pin_shadow_1(self.gc.minor_collection)
+
+    def test_pin_shadow_1_full(self):
+        self.pin_shadow_1(self.gc.collect)
+
+    def pin_shadow_2(self, collect_func):
         ptr = self.malloc(S)
         adr = llmemory.cast_ptr_to_adr(ptr)
         self.stackroots.append(ptr)
         ptr.someInt = 100
         assert self.gc.pin(adr)
         self.gc.identityhash(ptr) # allocate shadow
-        self.gc.collect()
+        collect_func()
         assert self.gc.is_in_nursery(adr)
         assert ptr.someInt == 100
         self.gc.unpin(adr)
-        self.gc.collect() # move to shadow
+        collect_func() # move to shadow
         adr = llmemory.cast_ptr_to_adr(self.stackroots[0])
         assert not self.gc.is_in_nursery(adr)
 
+    def test_pin_shadow_2_minor(self):
+        self.pin_shadow_2(self.gc.minor_collection)
+
+    def test_pin_shadow_2_full(self):
+        self.pin_shadow_2(self.gc.collect)
+
     def test_pin_nursery_top_scenario1(self):
         ptr1 = self.malloc(S)
         adr1 = llmemory.cast_ptr_to_adr(ptr1)

From noreply at buildbot.pypy.org  Sat Jul 26 16:47:58 2014
From: noreply at buildbot.pypy.org (groggi)
Date: Sat, 26 Jul 2014 16:47:58 +0200 (CEST)
Subject: [pypy-commit] pypy gc-incminimark-pinning: add test to make sure
 not referenced pinned objects are collected
Message-ID: <20140726144758.1E4FB1C050D@cobra.cs.uni-duesseldorf.de>

Author: Gregor Wegberg 
Branch: gc-incminimark-pinning
Changeset: r72531:1f3c0f2801cb
Date: 2014-07-26 15:48 +0200
http://bitbucket.org/pypy/pypy/changeset/1f3c0f2801cb/

Log:	add test to make sure not referenced pinned objects are collected

	Basically we have an old object pointing to a young pinned one. The
	moment the old object is not used anymore (not referenced from
	stack) the GC should not only collect the old object but also the
	still pinned one.

diff --git a/rpython/memory/gc/test/test_object_pinning.py b/rpython/memory/gc/test/test_object_pinning.py
--- a/rpython/memory/gc/test/test_object_pinning.py
+++ b/rpython/memory/gc/test/test_object_pinning.py
@@ -96,6 +96,29 @@
     def test_simple_pin_unpin_stack_minor_collect(self):
         self.simple_pin_unpin_stack(self.gc.minor_collection)
 
+    def test_pinned_obj_collected_after_old_object_collected(self):
+        root_ptr = self.malloc(S)
+        root_ptr.someInt = 999
+        self.stackroots.append(root_ptr)
+        self.gc.collect()
+
+        root_ptr = self.stackroots[0]
+        next_ptr = self.malloc(S)
+        next_ptr.someInt = 111
+        assert self.gc.pin(llmemory.cast_ptr_to_adr(next_ptr))
+        self.write(root_ptr, 'next', next_ptr)
+        self.gc.collect()
+        # check still alive
+        assert self.gc.is_in_nursery(llmemory.cast_ptr_to_adr(root_ptr.next))
+        self.stackroots.remove(root_ptr)
+        self.gc.collect()
+        # root_ptr was collected and therefore also the pinned object should
+        # be gone
+        try:
+            next_ptr.someInt = 101
+        except RuntimeError as ex:
+            assert "freed" in str(ex)
+
     def test_pin_referenced_from_stackroot_young(self):
         #
         # create two objects and reference the pinned one

From noreply at buildbot.pypy.org  Sat Jul 26 16:47:59 2014
From: noreply at buildbot.pypy.org (groggi)
Date: Sat, 26 Jul 2014 16:47:59 +0200 (CEST)
Subject: [pypy-commit] pypy gc-incminimark-pinning: add XXX todo item
 reminding to add more object pinning related tests
Message-ID: <20140726144759.2C6521C050D@cobra.cs.uni-duesseldorf.de>

Author: Gregor Wegberg 
Branch: gc-incminimark-pinning
Changeset: r72532:e02894d635ae
Date: 2014-07-26 15:49 +0200
http://bitbucket.org/pypy/pypy/changeset/e02894d635ae/

Log:	add XXX todo item reminding to add more object pinning related tests

diff --git a/rpython/memory/gc/test/test_object_pinning.py b/rpython/memory/gc/test/test_object_pinning.py
--- a/rpython/memory/gc/test/test_object_pinning.py
+++ b/rpython/memory/gc/test/test_object_pinning.py
@@ -119,6 +119,10 @@
         except RuntimeError as ex:
             assert "freed" in str(ex)
 
+    # XXX more tests like the one above. Make list of all possible cases and
+    # write tests for each one. Also: minor/full major collection tests maybe
+    # needed
+
     def test_pin_referenced_from_stackroot_young(self):
         #
         # create two objects and reference the pinned one

From noreply at buildbot.pypy.org  Sat Jul 26 16:48:00 2014
From: noreply at buildbot.pypy.org (groggi)
Date: Sat, 26 Jul 2014 16:48:00 +0200 (CEST)
Subject: [pypy-commit] pypy gc-incminimark-pinning: (wip) add handling for
 pinned objects referenced by old ones.
Message-ID: <20140726144800.445B41C050D@cobra.cs.uni-duesseldorf.de>

Author: Gregor Wegberg 
Branch: gc-incminimark-pinning
Changeset: r72533:cb701ab5f828
Date: 2014-07-26 16:46 +0200
http://bitbucket.org/pypy/pypy/changeset/cb701ab5f828/

Log:	(wip) add handling for pinned objects referenced by old ones.

	The added AddressStack keeps track of old objects which point to
	young pinned objects. This way if a pinned object is only referenced
	from an old object, it isn't collected as before.

	This is a work in progress. For now test_object_pinning.py tests
	pass. However, there is still a lot to be tested and double checked
	(see XXX). In addition some refactoring & additional comments are in
	order after we make sure it works as expected.

	For now this changes stop pypy from being translated.

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -381,6 +381,14 @@
         # the nursery.
         self.pinned_objects_in_nursery = 0
         #
+        # Keeps track of objects pointing to pinned objects. These objects
+        # must be revisited every minor collection. Without this list
+        # any old object inside this list would only be visited in case a
+        # write barrier was triggered, which would result in not visiting
+        # the young pinned object and would therefore result in removing
+        # the pinned object.
+        self.old_objects_pointing_to_pinned = self.AddressStack()
+        #
         # Allocate a nursery.  In case of auto_nursery_size, start by
         # allocating a very small nursery, enough to do things like look
         # up the env var, which requires the GC; and then really
@@ -1502,6 +1510,10 @@
         obj = obj + self.gcheaderbuilder.size_gc_header
         shadow = self.nursery_objects_shadows.get(obj)
         if shadow != NULL:
+            # visit shadow to keep it alive
+            # XXX seems like it is save to set GCFLAG_VISITED, however
+            # should be double checked
+            self.header(shadow).tid |= GCFLAG_VISITED
             new_shadow_object_dict.setitem(obj, shadow)
 
     # ----------
@@ -1554,6 +1566,13 @@
         self.nursery_surviving_size = 0
         self.collect_roots_in_nursery()
         #
+        # visit all objects that are known for pointing to pinned
+        # objects. This way we populate 'surviving_pinned_objects'
+        # with pinned object that are (only) visible from an old
+        # object.
+        self.old_objects_pointing_to_pinned.foreach(
+                self._visit_old_objects_pointing_to_pinned, None)
+        #
         while True:
             # If we are using card marking, do a partial trace of the arrays
             # that are flagged with GCFLAG_CARDS_SET.
@@ -1665,6 +1684,8 @@
         #
         debug_stop("gc-minor")
 
+    def _visit_old_objects_pointing_to_pinned(self, obj, ignore):
+        self.trace(obj, self._trace_drag_out, None)
 
     def collect_roots_in_nursery(self):
         # we don't need to trace prebuilt GcStructs during a minor collect:
@@ -1772,7 +1793,7 @@
         """obj must not be in the nursery.  This copies all the
         young objects it references out of the nursery.
         """
-        self.trace(obj, self._trace_drag_out, None)
+        self.trace(obj, self._trace_drag_out, obj)
 
     def trace_and_drag_out_of_nursery_partial(self, obj, start, stop):
         """Like trace_and_drag_out_of_nursery(), but limited to the array
@@ -1801,7 +1822,7 @@
         if not self.header(obj).tid & GCFLAG_VISITED:
             self.more_objects_to_trace.append(obj)
 
-    def _trace_drag_out(self, root, ignored):
+    def _trace_drag_out(self, root, parent):
         obj = root.address[0]
         #print '_trace_drag_out(%x: %r)' % (hash(obj.ptr._obj), obj)
         #
@@ -1842,8 +1863,11 @@
                 # already visited and keeping track of the object
                 return
             hdr.tid |= GCFLAG_VISITED
+            #
+            if parent:
+                self.old_objects_pointing_to_pinned.append(parent)
+            #
             # XXX add additional checks for unsupported pinned objects (groggi)
-            # XXX implement unsupported object types with pinning
             ll_assert(not self.header(obj).tid & GCFLAG_HAS_CARDS,
                 "pinned object with GCFLAG_HAS_CARDS not supported")
             self.surviving_pinned_objects.append(
@@ -2073,6 +2097,15 @@
             #END MARKING
         elif self.gc_state == STATE_SWEEPING:
             #
+            # get rid of objects pointing to pinned objects that were not
+            # visited
+            new_old_objects_pointing_to_pinned = self.AddressStack()
+            self.old_objects_pointing_to_pinned.foreach(
+                    self._sweep_old_objects_pointing_to_pinned,
+                    new_old_objects_pointing_to_pinned)
+            self.old_objects_pointing_to_pinned.delete()
+            self.old_objects_pointing_to_pinned = new_old_objects_pointing_to_pinned
+            #
             if self.raw_malloc_might_sweep.non_empty():
                 # Walk all rawmalloced objects and free the ones that don't
                 # have the GCFLAG_VISITED flag.  Visit at most 'limit' objects.
@@ -2147,6 +2180,10 @@
         debug_print("stopping, now in gc state: ", GC_STATES[self.gc_state])
         debug_stop("gc-collect-step")
 
+    def _sweep_old_objects_pointing_to_pinned(self, obj, new_list):
+        if self.header(obj).tid & GCFLAG_VISITED:
+            new_list.append(obj)
+
     def _free_if_unvisited(self, hdr):
         size_gc_header = self.gcheaderbuilder.size_gc_header
         obj = hdr + size_gc_header

From noreply at buildbot.pypy.org  Sat Jul 26 17:39:58 2014
From: noreply at buildbot.pypy.org (kvas)
Date: Sat, 26 Jul 2014 17:39:58 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: Fixed test_construct_singletons for
 Ellipsis and NotImplemented as well,
 so now ellipsis and NotImplementedType can be constructed.
Message-ID: <20140726153958.633CF1C03AC@cobra.cs.uni-duesseldorf.de>

Author: Vasily Kuznetsov 
Branch: py3.3
Changeset: r72534:1c542d770ddb
Date: 2014-07-26 16:25 +0200
http://bitbucket.org/pypy/pypy/changeset/1c542d770ddb/

Log:	Fixed test_construct_singletons for Ellipsis and NotImplemented as
	well, so now ellipsis and NotImplementedType can be constructed.

diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -950,11 +950,13 @@
 Cell.typedef.acceptable_as_base_class = False
 
 Ellipsis.typedef = TypeDef("Ellipsis",
+    __new__ = interp2app(lambda space, w_type: space.w_Ellipsis),
     __repr__ = interp2app(Ellipsis.descr__repr__),
 )
 Ellipsis.typedef.acceptable_as_base_class = False
 
 NotImplemented.typedef = TypeDef("NotImplemented",
+    __new__ = interp2app(lambda space, w_type: space.w_NotImplemented),
     __repr__ = interp2app(NotImplemented.descr__repr__),
 )
 NotImplemented.typedef.acceptable_as_base_class = False
diff --git a/pypy/module/__builtin__/test/test_construct_singletons.py b/pypy/module/__builtin__/test/test_construct_singletons.py
--- a/pypy/module/__builtin__/test/test_construct_singletons.py
+++ b/pypy/module/__builtin__/test/test_construct_singletons.py
@@ -1,7 +1,8 @@
 class AppTestConstructSingletons:
 
     def test_construct_singletons(self):
-        none_type = type(None)
-        assert none_type() is None
-        raises(TypeError, none_type, 1, 2)
-        raises(TypeError, none_type, a=1, b=2)
+        for const in None, Ellipsis, NotImplemented:
+            const_type = type(const)
+            assert const_type() is const
+            raises(TypeError, const_type, 1, 2)
+            raises(TypeError, const_type, a=1, b=2)

From noreply at buildbot.pypy.org  Sat Jul 26 17:39:59 2014
From: noreply at buildbot.pypy.org (kvas)
Date: Sat, 26 Jul 2014 17:39:59 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: Pickling of iterators produced by zip(seq,
	...).
Message-ID: <20140726153959.8CAE71C03AC@cobra.cs.uni-duesseldorf.de>

Author: Vasily Kuznetsov 
Branch: py3.3
Changeset: r72535:1f1711d14798
Date: 2014-07-26 16:38 +0200
http://bitbucket.org/pypy/pypy/changeset/1f1711d14798/

Log:	Pickling of iterators produced by zip(seq, ...).

diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py
--- a/pypy/module/__builtin__/functional.py
+++ b/pypy/module/__builtin__/functional.py
@@ -753,6 +753,12 @@
             raise OperationError(self.space.w_StopIteration, self.space.w_None)
         return W_Map.next_w(self)
 
+    def descr_reduce(self, space):
+        w_zip = space.getattr(space.getbuiltinmodule('builtins'),
+                space.wrap('zip'))
+        return space.newtuple([w_zip, space.newtuple(self.iterators_w)])
+
+
 def W_Zip___new__(space, w_subtype, args_w):
     r = space.allocate_instance(W_Zip, w_subtype)
     r.__init__(space, None, args_w)
@@ -763,6 +769,7 @@
         __new__  = interp2app(W_Zip___new__),
         __iter__ = interp2app(W_Zip.iter_w),
         __next__ = interp2app(W_Zip.next_w),
+        __reduce__ = interp2app(W_Zip.descr_reduce),
         __doc__  = """\
 Return a zip object whose .__next__() method returns a tuple where
 the i-th element comes from the i-th iterable argument.  The .__next__()
diff --git a/pypy/module/__builtin__/test/test_zip_pickle.py b/pypy/module/__builtin__/test/test_zip_pickle.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/__builtin__/test/test_zip_pickle.py
@@ -0,0 +1,14 @@
+class AppTestZipPickle:
+
+    def test_zip_pickle(self):
+        import pickle
+
+        def pickle_unpickle(obj):
+            d = pickle.dumps(obj)
+            return pickle.loads(d)
+
+        z1 = zip([1, 2, 3], [4, 5, 6])
+        z1_ = pickle_unpickle(z1)
+        l1, l1_ = list(z1), list(z1_)
+
+        assert l1 == l1_

From noreply at buildbot.pypy.org  Sat Jul 26 17:40:00 2014
From: noreply at buildbot.pypy.org (kvas)
Date: Sat, 26 Jul 2014 17:40:00 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: Additional checks related to pickling of
	zip iterators.
Message-ID: <20140726154000.B14781C03AC@cobra.cs.uni-duesseldorf.de>

Author: Vasily Kuznetsov 
Branch: py3.3
Changeset: r72536:d26183fe9a2b
Date: 2014-07-26 16:43 +0200
http://bitbucket.org/pypy/pypy/changeset/d26183fe9a2b/

Log:	Additional checks related to pickling of zip iterators.

diff --git a/pypy/module/__builtin__/test/test_zip_pickle.py b/pypy/module/__builtin__/test/test_zip_pickle.py
--- a/pypy/module/__builtin__/test/test_zip_pickle.py
+++ b/pypy/module/__builtin__/test/test_zip_pickle.py
@@ -3,12 +3,16 @@
     def test_zip_pickle(self):
         import pickle
 
-        def pickle_unpickle(obj):
-            d = pickle.dumps(obj)
-            return pickle.loads(d)
+        def check_pickle_unpickle(itr):
+            d = pickle.dumps(itr)
+            itr_ = pickle.loads(d)
+            lst, lst_ = list(itr), list(itr_)
+            assert lst == lst_
 
-        z1 = zip([1, 2, 3], [4, 5, 6])
-        z1_ = pickle_unpickle(z1)
-        l1, l1_ = list(z1), list(z1_)
+        check_pickle_unpickle(zip([1, 2, 3], [4, 5, 6]))
+        check_pickle_unpickle(zip())
 
-        assert l1 == l1_
+        a = iter("abc")
+        b = iter("cdef")
+        next(b)
+        check_pickle_unpickle(zip(a, b))

From noreply at buildbot.pypy.org  Sat Jul 26 18:21:08 2014
From: noreply at buildbot.pypy.org (kvas)
Date: Sat, 26 Jul 2014 18:21:08 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: Refuse to sum bytes or bytearrays like
	cpython3 sum.
Message-ID: <20140726162108.D6FA51C06AE@cobra.cs.uni-duesseldorf.de>

Author: Vasily Kuznetsov 
Branch: py3.3
Changeset: r72537:c4be82e0fa27
Date: 2014-07-26 18:08 +0200
http://bitbucket.org/pypy/pypy/changeset/c4be82e0fa27/

Log:	Refuse to sum bytes or bytearrays like cpython3 sum.

diff --git a/pypy/module/__builtin__/app_functional.py b/pypy/module/__builtin__/app_functional.py
--- a/pypy/module/__builtin__/app_functional.py
+++ b/pypy/module/__builtin__/app_functional.py
@@ -37,6 +37,10 @@
 empty, returns start."""
     if isinstance(start, str):
         raise TypeError("sum() can't sum strings [use ''.join(seq) instead]")
+    if isinstance(start, bytes):
+        raise TypeError("sum() can't sum bytes [use b''.join(seq) instead]")
+    if isinstance(start, bytearray):
+        raise TypeError("sum() can't sum bytearray [use b''.join(seq) instead]")
     last = start
     for x in sequence:
         # Very intentionally *not* +=, that would have different semantics if
diff --git a/pypy/module/__builtin__/test/test_sum.py b/pypy/module/__builtin__/test/test_sum.py
new file mode 100644
--- /dev/null
+++ b/pypy/module/__builtin__/test/test_sum.py
@@ -0,0 +1,9 @@
+class AppTestSum:
+
+    def test_sum(self):
+        raises(TypeError, sum, [b'a', b'c'], b'')
+        raises(TypeError, sum, [bytearray(b'a'), bytearray(b'b')],
+                bytearray(b''))
+        raises(TypeError, sum, [[1], [2], [3]])
+        raises(TypeError, sum, [{2:3}])
+        raises(TypeError, sum, [{2:3}]*2, {2:3})

From noreply at buildbot.pypy.org  Sat Jul 26 21:43:38 2014
From: noreply at buildbot.pypy.org (pjenvey)
Date: Sat, 26 Jul 2014 21:43:38 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: skip hash randomization test
Message-ID: <20140726194338.B096F1C050D@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey 
Branch: py3.3
Changeset: r72538:a2f29529053a
Date: 2014-07-26 12:27 -0700
http://bitbucket.org/pypy/pypy/changeset/a2f29529053a/

Log:	skip hash randomization test

diff --git a/lib-python/3/test/test_cmd_line.py b/lib-python/3/test/test_cmd_line.py
--- a/lib-python/3/test/test_cmd_line.py
+++ b/lib-python/3/test/test_cmd_line.py
@@ -341,6 +341,9 @@
     def test_hash_randomization(self):
         # Verify that -R enables hash randomization:
         self.verify_valid_flag('-R')
+        if test.support.check_impl_detail(cpython=False):
+            # PyPy doesn't support hash randomization
+            return
         hashes = []
         for i in range(2):
             code = 'print(hash("spam"))'

From noreply at buildbot.pypy.org  Sat Jul 26 21:43:39 2014
From: noreply at buildbot.pypy.org (pjenvey)
Date: Sat, 26 Jul 2014 21:43:39 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: missing import
Message-ID: <20140726194339.E051A1C050D@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey 
Branch: py3.3
Changeset: r72539:6a34ba3e09b7
Date: 2014-07-26 12:27 -0700
http://bitbucket.org/pypy/pypy/changeset/6a34ba3e09b7/

Log:	missing import

diff --git a/lib-python/3/test/pickletester.py b/lib-python/3/test/pickletester.py
--- a/lib-python/3/test/pickletester.py
+++ b/lib-python/3/test/pickletester.py
@@ -9,7 +9,7 @@
 
 from test.support import (
     TestFailed, TESTFN, run_with_locale, no_tracing,
-    _2G, _4G, bigmemtest,
+    _2G, _4G, bigmemtest, check_impl_detail
     )
 
 from pickle import bytes_types

From noreply at buildbot.pypy.org  Sat Jul 26 21:43:41 2014
From: noreply at buildbot.pypy.org (pjenvey)
Date: Sat, 26 Jul 2014 21:43:41 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: testing of __del__ needs a gc_collect
Message-ID: <20140726194341.1B9711C050D@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey 
Branch: py3.3
Changeset: r72540:48d61dbba1ff
Date: 2014-07-26 12:30 -0700
http://bitbucket.org/pypy/pypy/changeset/48d61dbba1ff/

Log:	testing of __del__ needs a gc_collect

diff --git a/lib-python/3/test/test_scope.py b/lib-python/3/test/test_scope.py
--- a/lib-python/3/test/test_scope.py
+++ b/lib-python/3/test/test_scope.py
@@ -1,7 +1,8 @@
 import unittest
 import weakref
 
-from test.support import check_syntax_error, cpython_only, run_unittest
+from test.support import (
+    check_syntax_error, cpython_only, run_unittest, gc_collect)
 
 
 class ScopeTests(unittest.TestCase):
@@ -422,6 +423,7 @@
         for i in range(100):
             f1()
 
+        gc_collect()
         self.assertEqual(Foo.count, 0)
 
     def testClassAndGlobal(self):

From noreply at buildbot.pypy.org  Sat Jul 26 21:43:42 2014
From: noreply at buildbot.pypy.org (pjenvey)
Date: Sat, 26 Jul 2014 21:43:42 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: skip new GIL API test
Message-ID: <20140726194342.49F151C050D@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey 
Branch: py3.3
Changeset: r72541:3224b56f5c8e
Date: 2014-07-26 12:36 -0700
http://bitbucket.org/pypy/pypy/changeset/3224b56f5c8e/

Log:	skip new GIL API test

diff --git a/lib-python/3/test/test_sys.py b/lib-python/3/test/test_sys.py
--- a/lib-python/3/test/test_sys.py
+++ b/lib-python/3/test/test_sys.py
@@ -169,7 +169,8 @@
                 sys.setcheckinterval(n)
                 self.assertEqual(sys.getcheckinterval(), n)
 
-    @unittest.skipUnless(threading, 'Threading required for this test.')
+    @unittest.skipUnless(hasattr(sys, 'getswitchinterval') and threading,
+                         'New GIL & threading required for this test.')
     def test_switchinterval(self):
         self.assertRaises(TypeError, sys.setswitchinterval)
         self.assertRaises(TypeError, sys.setswitchinterval, "a")

From noreply at buildbot.pypy.org  Sat Jul 26 21:43:43 2014
From: noreply at buildbot.pypy.org (pjenvey)
Date: Sat, 26 Jul 2014 21:43:43 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: __basic/itemsize__ are impl details
Message-ID: <20140726194343.8BDF41C050D@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey 
Branch: py3.3
Changeset: r72542:4f1d9525b86f
Date: 2014-07-26 12:40 -0700
http://bitbucket.org/pypy/pypy/changeset/4f1d9525b86f/

Log:	__basic/itemsize__ are impl details

diff --git a/lib-python/3/test/test_types.py b/lib-python/3/test/test_types.py
--- a/lib-python/3/test/test_types.py
+++ b/lib-python/3/test/test_types.py
@@ -1,6 +1,6 @@
 # Python test set -- part 6, built-in types
 
-from test.support import run_unittest, run_with_locale
+from test.support import run_unittest, run_with_locale, impl_detail
 import collections
 import locale
 import sys
@@ -566,6 +566,7 @@
         for code in 'xXobns':
             self.assertRaises(ValueError, format, 0, ',' + code)
 
+    @impl_detail
     def test_internal_sizes(self):
         self.assertGreater(object.__basicsize__, 0)
         self.assertGreater(tuple.__itemsize__, 0)

From noreply at buildbot.pypy.org  Sat Jul 26 21:43:44 2014
From: noreply at buildbot.pypy.org (pjenvey)
Date: Sat, 26 Jul 2014 21:43:44 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: clarify
Message-ID: <20140726194344.B17511C050D@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey 
Branch: py3.3
Changeset: r72543:d8277de0ae4e
Date: 2014-07-26 12:40 -0700
http://bitbucket.org/pypy/pypy/changeset/d8277de0ae4e/

Log:	clarify

diff --git a/lib-python/3/test/test_cmd_line.py b/lib-python/3/test/test_cmd_line.py
--- a/lib-python/3/test/test_cmd_line.py
+++ b/lib-python/3/test/test_cmd_line.py
@@ -341,7 +341,7 @@
     def test_hash_randomization(self):
         # Verify that -R enables hash randomization:
         self.verify_valid_flag('-R')
-        if test.support.check_impl_detail(cpython=False):
+        if test.support.check_impl_detail(pypy=True):
             # PyPy doesn't support hash randomization
             return
         hashes = []

From noreply at buildbot.pypy.org  Sat Jul 26 22:05:29 2014
From: noreply at buildbot.pypy.org (groggi)
Date: Sat, 26 Jul 2014 22:05:29 +0200 (CEST)
Subject: [pypy-commit] pypy gc-incminimark-pinning: incminimark can be
	translated now
Message-ID: <20140726200529.468E91C0945@cobra.cs.uni-duesseldorf.de>

Author: Gregor Wegberg 
Branch: gc-incminimark-pinning
Changeset: r72544:f03b71e80cc6
Date: 2014-07-26 20:11 +0200
http://bitbucket.org/pypy/pypy/changeset/f03b71e80cc6/

Log:	incminimark can be translated now

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -1685,7 +1685,7 @@
         debug_stop("gc-minor")
 
     def _visit_old_objects_pointing_to_pinned(self, obj, ignore):
-        self.trace(obj, self._trace_drag_out, None)
+        self.trace(obj, self._trace_drag_out, NULL)
 
     def collect_roots_in_nursery(self):
         # we don't need to trace prebuilt GcStructs during a minor collect:
@@ -1802,14 +1802,14 @@
         ll_assert(start < stop, "empty or negative range "
                                 "in trace_and_drag_out_of_nursery_partial()")
         #print 'trace_partial:', start, stop, '\t', obj
-        self.trace_partial(obj, start, stop, self._trace_drag_out, None)
+        self.trace_partial(obj, start, stop, self._trace_drag_out, NULL)
 
 
     def _trace_drag_out1(self, root):
-        self._trace_drag_out(root, None)
+        self._trace_drag_out(root, NULL)
 
     def _trace_drag_out1_marking_phase(self, root):
-        self._trace_drag_out(root, None)
+        self._trace_drag_out(root, NULL)
         #
         # We are in the MARKING state: we must also record this object
         # if it was young.  Don't bother with old objects in general,
@@ -1864,7 +1864,7 @@
                 return
             hdr.tid |= GCFLAG_VISITED
             #
-            if parent:
+            if parent != NULL:
                 self.old_objects_pointing_to_pinned.append(parent)
             #
             # XXX add additional checks for unsupported pinned objects (groggi)

From noreply at buildbot.pypy.org  Sat Jul 26 22:05:30 2014
From: noreply at buildbot.pypy.org (groggi)
Date: Sat, 26 Jul 2014 22:05:30 +0200 (CEST)
Subject: [pypy-commit] pypy gc-incminimark-pinning: 'NULL' ->
 'llmemory.NULL'. Make it consistent. Use explicit version
Message-ID: <20140726200530.6C2F21C0945@cobra.cs.uni-duesseldorf.de>

Author: Gregor Wegberg 
Branch: gc-incminimark-pinning
Changeset: r72545:4a4fffb737e5
Date: 2014-07-26 20:44 +0200
http://bitbucket.org/pypy/pypy/changeset/4a4fffb737e5/

Log:	'NULL' -> 'llmemory.NULL'. Make it consistent. Use explicit version

diff --git a/rpython/memory/gc/incminimark.py b/rpython/memory/gc/incminimark.py
--- a/rpython/memory/gc/incminimark.py
+++ b/rpython/memory/gc/incminimark.py
@@ -82,7 +82,6 @@
 # XXX update doc string to contain object pinning (groggi)
 
 WORD = LONG_BIT // 8
-NULL = llmemory.NULL
 
 first_gcflag = 1 << (LONG_BIT//2)
 
@@ -302,10 +301,10 @@
         # it gives a lower bound on the allowed size of the nursery.
         self.nonlarge_max = large_object - 1
         #
-        self.nursery      = NULL
-        self.nursery_free = NULL
-        self.nursery_top  = NULL
-        self.nursery_real_top = NULL
+        self.nursery      = llmemory.NULL
+        self.nursery_free = llmemory.NULL
+        self.nursery_top  = llmemory.NULL
+        self.nursery_real_top = llmemory.NULL
         self.debug_tiny_nursery = -1
         self.debug_rotating_nurseries = lltype.nullptr(NURSARRAY)
         self.extra_threshold = 0
@@ -1509,7 +1508,7 @@
         # dict of shadows.
         obj = obj + self.gcheaderbuilder.size_gc_header
         shadow = self.nursery_objects_shadows.get(obj)
-        if shadow != NULL:
+        if shadow != llmemory.NULL:
             # visit shadow to keep it alive
             # XXX seems like it is save to set GCFLAG_VISITED, however
             # should be double checked
@@ -1685,7 +1684,7 @@
         debug_stop("gc-minor")
 
     def _visit_old_objects_pointing_to_pinned(self, obj, ignore):
-        self.trace(obj, self._trace_drag_out, NULL)
+        self.trace(obj, self._trace_drag_out, llmemory.NULL)
 
     def collect_roots_in_nursery(self):
         # we don't need to trace prebuilt GcStructs during a minor collect:
@@ -1802,14 +1801,14 @@
         ll_assert(start < stop, "empty or negative range "
                                 "in trace_and_drag_out_of_nursery_partial()")
         #print 'trace_partial:', start, stop, '\t', obj
-        self.trace_partial(obj, start, stop, self._trace_drag_out, NULL)
+        self.trace_partial(obj, start, stop, self._trace_drag_out, llmemory.NULL)
 
 
     def _trace_drag_out1(self, root):
-        self._trace_drag_out(root, NULL)
+        self._trace_drag_out(root, llmemory.NULL)
 
     def _trace_drag_out1_marking_phase(self, root):
-        self._trace_drag_out(root, NULL)
+        self._trace_drag_out(root, llmemory.NULL)
         #
         # We are in the MARKING state: we must also record this object
         # if it was young.  Don't bother with old objects in general,
@@ -1864,7 +1863,7 @@
                 return
             hdr.tid |= GCFLAG_VISITED
             #
-            if parent != NULL:
+            if parent != llmemory.NULL:
                 self.old_objects_pointing_to_pinned.append(parent)
             #
             # XXX add additional checks for unsupported pinned objects (groggi)
@@ -1877,7 +1876,7 @@
         else:
             # First visit to an object that has already a shadow.
             newobj = self.nursery_objects_shadows.get(obj)
-            ll_assert(newobj != NULL, "GCFLAG_HAS_SHADOW but no shadow found")
+            ll_assert(newobj != llmemory.NULL, "GCFLAG_HAS_SHADOW but no shadow found")
             newhdr = newobj - size_gc_header
             #
             # Remove the flag GCFLAG_HAS_SHADOW, so that it doesn't get
@@ -2352,7 +2351,7 @@
         # collection
         if self.header(obj).tid & GCFLAG_HAS_SHADOW:
             shadow = self.nursery_objects_shadows.get(obj)
-            ll_assert(shadow != NULL,
+            ll_assert(shadow != llmemory.NULL,
                       "GCFLAG_HAS_SHADOW but no shadow found")
         else:
             shadow = self._allocate_shadow(obj)

From noreply at buildbot.pypy.org  Sat Jul 26 23:11:16 2014
From: noreply at buildbot.pypy.org (pjenvey)
Date: Sat, 26 Jul 2014 23:11:16 +0200 (CEST)
Subject: [pypy-commit] pypy py3.3: reapply workarounds from py3k branch
Message-ID: <20140726211116.4A7B21C046A@cobra.cs.uni-duesseldorf.de>

Author: Philip Jenvey 
Branch: py3.3
Changeset: r72546:9b1c5ecf653b
Date: 2014-07-26 13:20 -0700
http://bitbucket.org/pypy/pypy/changeset/9b1c5ecf653b/

Log:	reapply workarounds from py3k branch

diff --git a/lib-python/3/test/test_pydoc.py b/lib-python/3/test/test_pydoc.py
--- a/lib-python/3/test/test_pydoc.py
+++ b/lib-python/3/test/test_pydoc.py
@@ -18,7 +18,7 @@
 from collections import namedtuple
 from test.script_helper import assert_python_ok
 from test.support import (
-    TESTFN, rmtree,
+    TESTFN, rmtree, check_impl_detail,
     reap_children, reap_threads, captured_output, captured_stdout,
     captured_stderr, unlink
 )
@@ -105,7 +105,23 @@
 expected_text_data_docstrings = tuple('\n     |      ' + s if s else ''
                                       for s in expected_data_docstrings)
 
-expected_html_pattern = """
+if check_impl_detail(pypy=True):
+    # pydoc_mod.__builtins__ is always a module on PyPy (but a dict on
+    # CPython), hence an extra 'Modules' section
+    module_section = """
+

@@ -201,7 +217,7 @@ \x20\x20\x20\x20
		Nobody

PyPy status talk (a.k.a.: no no, PyPy is not dead)

Abstract

Description

Using All These Cores: Transactional Memory in PyPy

Abstract

Description

numpy compatability test results, generated automatically by running pypy/module/micronumpy/tool/numready/main.py <path-to-latest-pypy>

Overall: {{ msg }}

Warning: a positive result does not mean the function is actually working! It only means that the function/module/constant is present. It may be missing other things.

Using All These Cores: Transactional Memory in PyPy

Using All These Cores: Transactional Memory in PyPy

Part 1 - Intro and Current Status

Why is there a GIL?

Consequences (+)

Consequences (-)

Removing the GIL

Fine-grained locking

Shared-nothing

Transactional Memory

Big Point

Demo 1

PyPy-STM

Demo 2

Summary

Part 2 - Under The Hood

PyPy-STM status

Summary

Part 2 - Under The Hood

Big Point

Demo 1

Part 1 - Intro and Current Status

Introduction

Why is there a GIL?

Demo 1

Transactional Memory

Benefits

Issues

Summary

Big Point

PyPy-STM

How does it work?

Demo

Long Transactions

Big Point

Long Transactions

Demo 1

Demo

PyPy-STM

PyPy-STM Programming Model

Demo 2

Summary: Benefits

Benefits

Issues

Summary: Issues

Summary

Summary: PyPy-STM

Part 2 - Under The Hood

Overview

Trick #1

Trick #2

Barriers

...

Thank You

PyPy-STM Programming Model

PyPy-STM status

...

numpy compatability test results, generated automatically by running
`pypy/module/micronumpy/tool/numready/main.py <path-to-latest-pypy>`