From jython-checkins at python.org  Sun Mar  6 21:45:37 2016
From: jython-checkins at python.org (darjus.loktevic)
Date: Mon, 07 Mar 2016 02:45:37 +0000
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Make_sure_that_from_=3Cjava?=
 =?utf-8?q?package=3E_import_=3Cclass=3E_is_not_reloaded=2E_Fixes_=232480?=
Message-ID: <20160307024536.89274.48190@psf.io>

https://hg.python.org/jython/rev/1a14d360dc8d
changeset:   7915:1a14d360dc8d
user:        Darjus Loktevic <darjus at gmail.com>
date:        Mon Mar 07 13:45:39 2016 +1100
summary:
  Make sure that from <javapackage> import <class> is not reloaded. Fixes #2480

files:
  src/org/python/core/imp.java |  4 ----
  1 files changed, 0 insertions(+), 4 deletions(-)


diff --git a/src/org/python/core/imp.java b/src/org/python/core/imp.java
--- a/src/org/python/core/imp.java
+++ b/src/org/python/core/imp.java
@@ -968,10 +968,6 @@
 
     private static void ensureFromList(PyObject mod, PyObject fromlist, String name,
             boolean recursive) {
-        if (mod.__findattr__("__path__") == null) {
-            return;
-        }
-
         // This can happen with imports like "from . import foo"
         if (name.length() == 0) {
             name = mod.__findattr__("__name__").toString();

-- 
Repository URL: https://hg.python.org/jython

From jython-checkins at python.org  Thu Mar 17 20:44:16 2016
From: jython-checkins at python.org (darjus.loktevic)
Date: Fri, 18 Mar 2016 00:44:16 +0000
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Fix_deadlocks_in_PyType=2Ef?=
 =?utf-8?q?romClass_by_using_IOD_and_selective_locking=2E_Fixes?=
Message-ID: <20160318004404.98667.59929.8B8A7435@psf.io>

https://hg.python.org/jython/rev/cffe5b824a21
changeset:   7916:cffe5b824a21
user:        Darjus Loktevic <darjus at gmail.com>
date:        Fri Mar 18 11:43:56 2016 +1100
summary:
  Fix deadlocks in PyType.fromClass by using IOD and selective locking. Fixes #2487

files:
  Lib/test/test_jy_internals.py         |    4 +-
  src/org/python/core/PyType.java       |  137 ++++++++-----
  tests/java/javatests/TestSupport.java |   13 +
  3 files changed, 96 insertions(+), 58 deletions(-)


diff --git a/Lib/test/test_jy_internals.py b/Lib/test/test_jy_internals.py
--- a/Lib/test/test_jy_internals.py
+++ b/Lib/test/test_jy_internals.py
@@ -10,7 +10,7 @@
 
 from org.python.core import Py
 from org.python.util import PythonInterpreter
-from javatests.TestSupport import getField
+from javatests.TestSupport import invokePyTypeMethod
 from java.sql import Date, Time, Timestamp
 import datetime
 
@@ -24,7 +24,7 @@
         # by using this helper function that reflects on PyType (and
         # demonstrates here that it's the same as the builtin function
         # `type`!)
-        class_to_type_map = getField(type, 'class_to_type').get(None)
+        class_to_type_map = invokePyTypeMethod(type, 'getClassToType')
 
         def create_proxies():
             pi = PythonInterpreter()
diff --git a/src/org/python/core/PyType.java b/src/org/python/core/PyType.java
--- a/src/org/python/core/PyType.java
+++ b/src/org/python/core/PyType.java
@@ -7,8 +7,8 @@
 import java.lang.ref.WeakReference;
 import java.util.Iterator;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.atomic.AtomicReferenceArray;
 
 import org.python.expose.ExposeAsSuperclass;
@@ -99,13 +99,6 @@
     /** Global mro cache. */
     private static final MethodCache methodCache = new MethodCache();
 
-    /** Mapping of Java classes to their PyTypes. */
-    private static Map<Class<?>, PyType> class_to_type;
-    private static Set<PyType> exposedTypes;
-
-    /** Mapping of Java classes to their TypeBuilders. */
-    private static Map<Class<?>, TypeBuilder> classToBuilder;
-
     protected PyType(PyType subtype) {
         super(subtype);
     }
@@ -513,7 +506,7 @@
         } else {
             Class<?> baseClass;
             if (!BootstrapTypesSingleton.getInstance().contains(underlying_class)) {
-                baseClass = classToBuilder.get(underlying_class).getBase();
+                baseClass = getClassToBuilder().get(underlying_class).getBase();
             } else {
                 baseClass = PyObject.class;
             }
@@ -527,7 +520,7 @@
             // BOOTSTRAP_TYPES
             return;
         }
-        TypeBuilder builder = classToBuilder.get(underlying_class);
+        TypeBuilder builder = getClassToBuilder().get(underlying_class);
         name = builder.getName();
         dict = builder.getDict(this);
         String doc = builder.getDoc();
@@ -1324,42 +1317,34 @@
         return null;
     }
 
-    public synchronized static void addBuilder(Class<?> forClass, TypeBuilder builder) {
-        if (classToBuilder == null) {
-            classToBuilder = Generic.map();
-        }
-        classToBuilder.put(forClass, builder);
+    public static void addBuilder(Class<?> forClass, TypeBuilder builder) {
+        getClassToBuilder().put(forClass, builder);
 
-        if (class_to_type.containsKey(forClass)) {
+        if (getClassToType().containsKey(forClass)) {
             if (!BootstrapTypesSingleton.getInstance().remove(forClass)) {
-                Py.writeWarning("init", "Bootstrapping class not in BootstrapTypesSingleton.getInstance()[class="
-                                + forClass + "]");
+                Py.writeWarning("init", "Bootstrapping class not in BootstrapTypesSingleton.getClassToType()[class="
+                        + forClass + "]");
             }
-            // The types in BootstrapTypesSingleton.getInstance() are initialized before their builders are assigned,
+            // The types in BootstrapTypesSingleton.getClassToType() are initialized before their builders are assigned,
             // so do the work of addFromClass & fillFromClass after the fact
             fromClass(builder.getTypeClass()).init(builder.getTypeClass(), null);
         }
     }
 
-    private synchronized static PyType addFromClass(Class<?> c, Set<PyJavaType> needsInners) {
+    private static PyType addFromClass(Class<?> c, Set<PyJavaType> needsInners) {
         if (ExposeAsSuperclass.class.isAssignableFrom(c)) {
             PyType exposedAs = fromClass(c.getSuperclass(), false);
-            class_to_type.put(c, exposedAs);
+            PyType origExposedAs = getClassToType().putIfAbsent(c, exposedAs);
             return exposedAs;
         }
         return createType(c, needsInners);
     }
 
     static boolean hasBuilder(Class<?> c) {
-        return classToBuilder != null && classToBuilder.containsKey(c);
+        return getClassToBuilder().containsKey(c);
     }
     
     private static TypeBuilder getBuilder(Class<?> c) {
-        if (classToBuilder == null) {
-            // PyType itself has yet to be initialized.  This should be a bootstrap type, so it'll
-            // go through the builder process in a second
-            return null;
-        }
         if (c.isPrimitive() || !PyObject.class.isAssignableFrom(c)) {
             // If this isn't a PyObject, don't bother forcing it to be initialized to load its
             // builder
@@ -1380,7 +1365,7 @@
         } catch (SecurityException e) {
             exc = e;
         }
-        TypeBuilder builder = classToBuilder.get(c);
+        TypeBuilder builder = getClassToBuilder().get(c);
         if (builder == null && exc != null) {
             Py.writeComment("type",
                     "Unable to initialize " + c.getName() + ", a PyObject subclass, due to a " +
@@ -1391,7 +1376,7 @@
         return builder;
     }
 
-    private synchronized static PyType createType(Class<?> c, Set<PyJavaType> needsInners) {
+    private static PyType createType(Class<?> c, Set<PyJavaType> needsInners) {
 //        System.out.println("createType c=" + c + ", needsInners=" + needsInners + ", BootstrapTypesSingleton.getInstance()=" + BootstrapTypesSingleton.getInstance());
         PyType newtype;
         if (c == PyType.class) {
@@ -1402,37 +1387,40 @@
             newtype = new PyJavaType();
         }
 
+        // try to put the type into our cache but if one exists already ignore our new creation and instead return
+        // existing instance. This saves us from locking the whole createType and instead depend on the
+        // atomicity of putIfAbsent and finicky ordering :(
+        PyType type = getClassToType().putIfAbsent(c, newtype);
+        // synchronize on the c here to make sure the compiler does not re-order
+        synchronized (c) {
+            if (type != null) {
+                return type;
+            }
 
-        // If filling in the type above filled the type under creation, use that one
-        PyType type = class_to_type.get(c);
-        if (type != null) {
-            return type;
+            newtype.builtin = true;
+            newtype.init(c, needsInners);
+            newtype.invalidateMethodCache();
+            return newtype;
         }
-
-        class_to_type.put(c, newtype);
-        newtype.builtin = true;
-        newtype.init(c, needsInners);
-        newtype.invalidateMethodCache();
-        return newtype;
     }
 
-    // re the synchronization used here: this result is cached in each type obj,
-    // so we just need to prevent data races. all public methods that access class_to_type
-    // are themselves synchronized. However, if we use Google Collections/Guava,
-    // MapMaker only uses ConcurrentMap anyway
+    /*
+    Instead of using synchronization on the whole method we depend on IOD and putIfAbsent behaviour,
+    essentially letting the first type creation win and throwing away any concurrent ones.
+    This let's us have much more fine-grained locking.
+     */
 
-    public static synchronized PyType fromClass(Class<?> c) {
+    public static PyType fromClass(Class<?> c) {
         return fromClass(c, true);
     }
 
-    public static synchronized PyType fromClass(Class<?> c, boolean hardRef) {
-        if (class_to_type == null) {
-            class_to_type = new MapMaker().weakKeys().weakValues().makeMap();
-            addFromClass(PyType.class, null);
-        }
-        PyType type = class_to_type.get(c);
+    public static PyType fromClass(Class<?> c, boolean hardRef) {
+        PyType type = getClassToType().get(c);
         if (type != null) {
-            return type;
+            // synchronize on the c here to make sure the compiler does not re-order
+            synchronized (c) {
+                return type;
+            }
         }
         // We haven't seen this class before, so its type needs to be created. If it's being
         // exposed as a Java class, defer processing its inner types until it's completely
@@ -1461,17 +1449,14 @@
             }
         }
         if (hardRef && result != null) {
-            if (exposedTypes == null) {
-                exposedTypes = Generic.set();
-            }
-            exposedTypes.add(result) ;
+            getExposedTypes().add(result) ;
         }
 
         return result;
     }
 
     static PyType fromClassSkippingInners(Class<?> c, Set<PyJavaType> needsInners) {
-        PyType type = class_to_type.get(c);
+        PyType type = getClassToType().get(c);
         if (type != null) {
             return type;
         }
@@ -2129,6 +2114,46 @@
         }
     }
 
+    /**
+     *
+     * Using the IOD http://www.cs.umd.edu/~pugh/java/memoryModel/jsr-133-faq.html#dcl
+     * to reduce the risk for race conditions in a few of our fields
+     *
+     */
+
+    /**
+     * Mapping of Java classes to their PyTypes.
+     */
+    private static class LazyClassToTypeHolder {
+        private static ConcurrentMap<Class<?>, PyType> classToType = new MapMaker().weakKeys().weakValues().makeMap();
+
+    }
+
+    private static ConcurrentMap<Class<?>, PyType> getClassToType() {
+        return LazyClassToTypeHolder.classToType;
+    }
+
+    /**
+     * Mapping of Java classes to their TypeBuilders.
+     */
+    private static class LazyClassToBuilderHolder {
+        private static ConcurrentMap<Class<?>, TypeBuilder> classToBuilder = Generic.concurrentMap();
+    }
+
+    private static ConcurrentMap<Class<?>, TypeBuilder> getClassToBuilder() {
+        return LazyClassToBuilderHolder.classToBuilder;
+    }
+
+    /**
+     * Set of exposed types
+     */
+    private static class LazyExposedTypes {
+        private static Set<PyType> exposedTypes = Generic.set();
+    }
+
+    private static Set<PyType> getExposedTypes() {
+        return LazyExposedTypes.exposedTypes;
+    }
 
     /* Traverseproc implementation */
     @Override
diff --git a/tests/java/javatests/TestSupport.java b/tests/java/javatests/TestSupport.java
--- a/tests/java/javatests/TestSupport.java
+++ b/tests/java/javatests/TestSupport.java
@@ -1,8 +1,11 @@
 //Copyright (c) Corporation for National Research Initiatives
 package javatests;
 
+import org.python.core.PyType;
+
 import java.lang.reflect.Field;
 import java.lang.NoSuchFieldException;
+import java.lang.reflect.Method;
 
 /**
  * @author updikca1
@@ -60,5 +63,15 @@
         }
     }
 
+    public static Object invokePyTypeMethod(Object type, String name, Class<?>... param_types) {
+        try {
+            Method method = PyType.class.getDeclaredMethod(name, param_types);
+            method.setAccessible(true);
+            return method.invoke(type);
+        } catch (Throwable ex) {
+            throw new RuntimeException(ex);
+        }
+    }
+
 }
 

-- 
Repository URL: https://hg.python.org/jython

From jython-checkins at python.org  Fri Mar 18 19:49:25 2016
From: jython-checkins at python.org (darjus.loktevic)
Date: Fri, 18 Mar 2016 23:49:25 +0000
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Switch_to_using_test=5Fsupp?=
 =?utf-8?q?ort=2Egc=5Fcollect_for_test=5Fweakset_test=2C_which_should?=
Message-ID: <20160318234925.13988.58466.0798F9E5@psf.io>

https://hg.python.org/jython/rev/36edf877d361
changeset:   7917:36edf877d361
user:        Darjus Loktevic <darjus at gmail.com>
date:        Sat Mar 19 10:49:19 2016 +1100
summary:
  Switch to using test_support.gc_collect for test_weakset test, which should work much better for regrtest on slower machines

files:
  Lib/test/test_weakset.py |  30 ++++++++++++---------------
  1 files changed, 13 insertions(+), 17 deletions(-)


diff --git a/Lib/test/test_weakset.py b/Lib/test/test_weakset.py
--- a/Lib/test/test_weakset.py
+++ b/Lib/test/test_weakset.py
@@ -83,7 +83,7 @@
         self.assertEqual(len(self.s), len(self.d))
         self.assertEqual(len(self.fs), 1)
         del self.obj
-        gc.collect()
+        test_support.gc_collect()
         # len of weak collections is eventually consistent on
         # Jython. In practice this does not matter because of the
         # nature of weaksets - we cannot rely on what happens in the
@@ -97,7 +97,7 @@
         self.assertNotIn(1, self.s)
         self.assertIn(self.obj, self.fs)
         del self.obj
-        gc.collect()
+        test_support.gc_collect()
         self.assertNotIn(SomeClass('F'), self.fs)
 
     def test_union(self):
@@ -112,10 +112,10 @@
             c = C(self.items2)
             self.assertEqual(self.s.union(c), x)
             del c
-            gc.collect()
+            test_support.gc_collect()
         self.assertEqual(len(list(u)), len(list(self.items)) + len(list(self.items2)))
         self.items2.pop()
-        gc.collect()
+        test_support.gc_collect()
         self.assertEqual(len(list(u)), len(list(self.items)) + len(list(self.items2)))
 
     def test_or(self):
@@ -135,7 +135,7 @@
             self.assertEqual(i.intersection(C(self.items)), x)
         self.assertEqual(len(i), len(self.items2))
         self.items2.pop()
-        gc.collect()
+        test_support.gc_collect()
         self.assertEqual(len(list(i)), len(list(self.items2)))
 
     def test_isdisjoint(self):
@@ -169,7 +169,7 @@
         self.assertRaises(TypeError, self.s.symmetric_difference, [[]])
         self.assertEqual(len(i), len(self.items) + len(self.items2))
         self.items2.pop()
-        gc.collect()
+        test_support.gc_collect()
         self.assertEqual(len(list(i)), len(list(self.items)) + len(list(self.items2)))
 
     def test_xor(self):
@@ -258,7 +258,7 @@
         if not test_support.is_jython:  # Jython/JVM can weakly reference list and other objects
             self.assertRaises(TypeError, self.s.add, [])
         self.fs.add(Foo())
-        gc.collect()  # CPython assumes Foo() went out of scope and was collected, so ensure the same
+        test_support.gc_collect()  # CPython assumes Foo() went out of scope and was collected, so ensure the same
         self.assertEqual(len(list(self.fs)), 1)
         self.fs.add(self.obj)
         self.assertEqual(len(list(self.fs)), 1)
@@ -391,7 +391,7 @@
         next(it)             # Trigger internal iteration
         # Destroy an item
         del items[-1]
-        gc.collect()    # just in case
+        test_support.gc_collect()    # just in case
         # We have removed either the first consumed items, or another one
         self.assertIn(len(list(it)), [len(items), len(items) - 1])
         del it
@@ -410,13 +410,12 @@
                 next(it)
                 # Schedule an item for removal and recreate it
                 u = SomeClass(str(items.pop()))
-                gc.collect()      # just in case
+                test_support.gc_collect()      # just in case
                 yield u
             finally:
                 it = None           # should commit all removals
 
-        for _ in xrange(10):
-            gc.collect()
+        test_support.gc_collect()
 
         with testcontext() as u:
             self.assertNotIn(u, s)
@@ -440,8 +439,7 @@
         s = WeakSet(items)
         del items
         # do some gc
-        for _ in xrange(10):
-            gc.collect()
+        test_support.gc_collect()
         it = iter(s)
         try:
             next(it)
@@ -449,14 +447,12 @@
             pass
 
         # do some gc
-        for _ in xrange(10):
-            gc.collect()
+        test_support.gc_collect()
 
         n1 = len(s)
         del it
         # do some gc
-        for _ in xrange(10):
-            gc.collect()
+        test_support.gc_collect()
 
         n2 = len(s)
         # one item may be kept alive inside the iterator

-- 
Repository URL: https://hg.python.org/jython

From jython-checkins at python.org  Fri Mar 18 22:25:30 2016
From: jython-checkins at python.org (darjus.loktevic)
Date: Sat, 19 Mar 2016 02:25:30 +0000
Subject: [Jython-checkins] =?utf-8?q?jython=3A_Modify_test=5Fsupport=2Ebin?=
 =?utf-8?q?d=5Fport_to_do_a_gc_for_Jython_version_to_hopefully_fix?=
Message-ID: <20160319022529.61609.9394.800A8D41@psf.io>

https://hg.python.org/jython/rev/87534ec6252a
changeset:   7918:87534ec6252a
user:        Darjus Loktevic <darjus at gmail.com>
date:        Sat Mar 19 13:25:23 2016 +1100
summary:
  Modify test_support.bind_port to do a gc for Jython version to hopefully fix the test_timeout port issues for our CI systems

files:
  Lib/test/test_support.py |  1 +
  1 files changed, 1 insertions(+), 0 deletions(-)


diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py
--- a/Lib/test/test_support.py
+++ b/Lib/test/test_support.py
@@ -396,6 +396,7 @@
         port = tempsock.getsockname()[1]
         tempsock.close()
         del tempsock
+        gc_collect()
         sock.bind((host, port))
         return port
 

-- 
Repository URL: https://hg.python.org/jython

From darjus at gmail.com  Fri Mar 18 00:16:46 2016
From: darjus at gmail.com (Darjus Loktevic)
Date: Fri, 18 Mar 2016 04:16:46 +0000
Subject: [Jython-checkins] jython: Add some more gc.collect() to
 test_weakset, does not seem to be failing in
In-Reply-To: <0546DED2-AC7C-4FD4-AAC3-529CBD1C7BB7@underboss.org>
References: <20160122042819.64100.4715@psf.io>
 <trinity-d57de52d-422f-47da-932f-2e75ed443d6c-1453449874445@3capp-gmx-bs69>
 <0546DED2-AC7C-4FD4-AAC3-529CBD1C7BB7@underboss.org>
Message-ID: <CAFiRLmmtApPcp6_cDbGZfovOo_hj+O0T9A+A7-gYF=ojuwxmyA@mail.gmail.com>

Awesome, thanks!

Darjus

On Fri, Feb 5, 2016 at 10:44 AM Philip Jenvey <pjenvey at underboss.org> wrote:

> FYI there is a test_support.gc_collect which should be used instead. It
> calls collect multiple times with a pause in between for Jython.
>
> --
> Philip Jenvey
>
> > On Jan 22, 2016, at 12:04 AM, Stefan Richthofer <
> Stefan.Richthofer at gmx.de> wrote:
> >
> > Did you alternatively try to put some wait-time behind the collect call?
> It usually runs asynchronously and I suspect calling it twice only adds the
> sufficient wait time implicitly.
> >
> >
> >> Gesendet: Freitag, 22. Januar 2016 um 05:28 Uhr
> >> Von: "darjus.loktevic" <jython-checkins at python.org>
> >> An: jython-checkins at python.org
> >> Betreff: [Jython-checkins] jython: Add some more gc.collect() to
> test_weakset, does not seem to be failing in
> >>
> >> https://hg.python.org/jython/rev/f58d86b21716
> >> changeset:   7883:f58d86b21716
> >> user:        Darjus Loktevic <darjus at gmail.com>
> >> date:        Fri Jan 22 15:28:14 2016 +1100
> >> summary:
> >> Add some more gc.collect() to test_weakset, does not seem to be failing
> in circleci anymore under the matrix of JVMs
> >>
> >> files:
> >> Lib/test/test_weakset.py |  3 ++-
> >> 1 files changed, 2 insertions(+), 1 deletions(-)
> >>
> >>
> >> diff --git a/Lib/test/test_weakset.py b/Lib/test/test_weakset.py
> >> --- a/Lib/test/test_weakset.py
> >> +++ b/Lib/test/test_weakset.py
> >> @@ -415,7 +415,7 @@
> >>            finally:
> >>                it = None           # should commit all removals
> >>
> >> -        gc.collect()  # final before asserts
> >> +        gc.collect(); gc.collect()  # final before asserts
> >>        with testcontext() as u:
> >>            self.assertNotIn(u, s)
> >>        with testcontext() as u:
> >> @@ -447,6 +447,7 @@
> >>        n1 = len(s)
> >>        del it
> >>        gc.collect()
> >> +        gc.collect()
> >>        n2 = len(s)
> >>        # one item may be kept alive inside the iterator
> >>        self.assertIn(n1, (0, 1))
> >>
> >> --
> >> Repository URL: https://hg.python.org/jython
> >> _______________________________________________
> >> Jython-checkins mailing list
> >> Jython-checkins at python.org
> >> https://mail.python.org/mailman/listinfo/jython-checkins
> >>
> > _______________________________________________
> > Jython-checkins mailing list
> > Jython-checkins at python.org
> > https://mail.python.org/mailman/listinfo/jython-checkins
>
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mail.python.org/pipermail/jython-checkins/attachments/20160318/5dd0078e/attachment.html>