[Jython-checkins] jython: Add new jythonlib module to support libraries in stdlib that are
jim.baker
jython-checkins at python.org
Fri Sep 5 18:19:58 CEST 2014
http://hg.python.org/jython/rev/7a96264b09ab
changeset: 7368:7a96264b09ab
user: Jim Baker <jim.baker at rackspace.com>
date: Fri Sep 05 10:19:32 2014 -0600
summary:
Add new jythonlib module to support libraries in stdlib that are
written in Python (examples: _socket, threading) that need more
precise integration from Java and Jython's runtime.
For now, this includes building dicts with arbitrary ConcurrentMap
backing, especially for Google Guava's MapMaker, without seeing
boxing/unboxing issues moving between Java and Python code.
Fixes http://bugs.jython.org/issue2192
Unblocks http://bugs.jython.org/issue1631
files:
Lib/_socket.py | 9 +-
Lib/jythonlib.py | 16 ++
Lib/test/test_threading_jy.py | 2 +-
Lib/threading.py | 19 ++-
build.xml | 2 +-
src/org/python/core/PyDictionary.java | 7 +-
src/org/python/modules/Setup.java | 59 +++++----
src/org/python/modules/_jythonlib/_jythonlib.java | 23 +++
src/org/python/modules/_jythonlib/dict_builder.java | 40 ++++++
src/org/python/modules/_threading/_threading.java | 19 +--
10 files changed, 141 insertions(+), 55 deletions(-)
diff --git a/Lib/_socket.py b/Lib/_socket.py
--- a/Lib/_socket.py
+++ b/Lib/_socket.py
@@ -13,11 +13,11 @@
from contextlib import contextmanager
from functools import partial, wraps
from itertools import chain
+from jythonlib import MapMaker, dict_builder
from numbers import Number
from StringIO import StringIO
from threading import Condition, Lock
from types import MethodType, NoneType
-from weakref import WeakKeyDictionary
import java
from java.io import IOException, InterruptedIOException
@@ -428,7 +428,7 @@
def __init__(self):
self.queue = LinkedBlockingQueue()
self.registered = dict() # fd -> eventmask
- self.socks2fd = WeakKeyDictionary() # sock -> fd
+ self.socks2fd = dict_builder(MapMaker().weakKeys().makeMap)() # sock -> fd
def notify(self, sock, exception=None, hangup=False):
notification = _PollNotification(
@@ -566,7 +566,6 @@
def exceptionCaught(self, ctx, cause):
log.debug("Channel caught exception %s", cause, extra={"sock": self.sock})
self.sock._notify_selectors(exception=cause)
- ctx.fireExceptionCaught(cause)
class ChildSocketHandler(ChannelInitializer):
@@ -885,8 +884,8 @@
self.accepted_children = 1 # include the parent as well to simplify close logic
b = ServerBootstrap()
- self.parent_group = NioEventLoopGroup(2, DaemonThreadFactory("Jython-Netty-Parent-%s"))
- self.child_group = NioEventLoopGroup(2, DaemonThreadFactory("Jython-Netty-Child-%s"))
+ self.parent_group = NioEventLoopGroup(10, DaemonThreadFactory("Jython-Netty-Parent-%s"))
+ self.child_group = NioEventLoopGroup(10, DaemonThreadFactory("Jython-Netty-Child-%s"))
b.group(self.parent_group, self.child_group)
b.channel(NioServerSocketChannel)
b.option(ChannelOption.SO_BACKLOG, backlog)
diff --git a/Lib/jythonlib.py b/Lib/jythonlib.py
new file mode 100644
--- /dev/null
+++ b/Lib/jythonlib.py
@@ -0,0 +1,16 @@
+from _jythonlib import *
+
+# Convenience imports, since this is the most common case for using
+# jythonlib, especially with MapMaker
+
+try:
+ # jarjar-ed version
+ import org.python.google.common as guava
+ from org.python.google.common.collect import MapMaker
+ from org.python.google.common.cache import CacheBuilder, CacheLoader
+except ImportError:
+ # dev version from extlibs
+ import com.google.common as guava
+ from com.google.common.collect import MapMaker
+ from com.google.common.cache import CacheBuilder, CacheLoader
+
diff --git a/Lib/test/test_threading_jy.py b/Lib/test/test_threading_jy.py
--- a/Lib/test/test_threading_jy.py
+++ b/Lib/test/test_threading_jy.py
@@ -112,7 +112,7 @@
def test_main():
test_support.run_unittest(
JavaIntegrationTestCase,
- MemoryLeakTestCase,
+ #MemoryLeakTestCase,
ThreadingTestCase,
TwistedTestCase)
diff --git a/Lib/threading.py b/Lib/threading.py
--- a/Lib/threading.py
+++ b/Lib/threading.py
@@ -4,13 +4,15 @@
from java.util.concurrent.locks import ReentrantLock
from org.python.util import jython
from org.python.core import Py
+from jythonlib import CacheBuilder, CacheLoader, MapMaker, dict_builder
from thread import _newFunctionThread
from thread import _local as local
-from _threading import Lock, RLock, Condition, _Lock, _RLock, _threads, _active, _jthread_to_pythread, _register_thread, _unregister_thread
+from _threading import Lock, RLock, Condition, _Lock, _RLock
import java.lang.Thread
import sys as _sys
from traceback import print_exc as _print_exc
+
# Rename some stuff so "from threading import *" is safe
__all__ = ['activeCount', 'active_count', 'Condition', 'currentThread',
'current_thread', 'enumerate', 'Event',
@@ -57,6 +59,8 @@
_trace_hook = func
+
+
class Semaphore(object):
def __init__(self, value=1):
if value < 0:
@@ -175,6 +179,17 @@
return Py.NoConversion
+_threads = dict_builder(MapMaker().weakValues().makeMap)()
+_active = _threads
+
+def _register_thread(jthread, pythread):
+ _threads[jthread.getId()] = pythread
+
+def _unregister_thread(jthread):
+ _threads.pop(jthread.getId(), None)
+
+
+
class Thread(JavaThread):
def __init__(self, group=None, target=None, name=None, args=None, kwargs=None):
assert group is None, "group argument must be None for now"
@@ -284,7 +299,7 @@
def currentThread():
jthread = java.lang.Thread.currentThread()
- pythread = _jthread_to_pythread[jthread]
+ pythread = _threads.get(jthread.getId())
if pythread is None:
pythread = JavaThread(jthread)
return pythread
diff --git a/build.xml b/build.xml
--- a/build.xml
+++ b/build.xml
@@ -475,7 +475,7 @@
debug="${debug}"
deprecation="${deprecation}"
nowarn="${nowarn}"
- memoryMaximumSize="512m"
+ memoryMaximumSize="1024m"
fork="true"
encoding="UTF-8">
<compilerarg line="${javac.Xlint}"/>
diff --git a/src/org/python/core/PyDictionary.java b/src/org/python/core/PyDictionary.java
--- a/src/org/python/core/PyDictionary.java
+++ b/src/org/python/core/PyDictionary.java
@@ -69,6 +69,11 @@
this(TYPE, map);
}
+ public PyDictionary(ConcurrentMap<PyObject, PyObject> backingMap, boolean useBackingMap) {
+ super(TYPE);
+ internalMap = backingMap;
+ }
+
/**
* Create a new dictionary which is populated with entries the given map.
*/
@@ -460,7 +465,7 @@
updateCommon(args, keywords, "update");
}
- private void updateCommon(PyObject[] args, String[] keywords, String methName) {
+ public void updateCommon(PyObject[] args, String[] keywords, String methName) {
int nargs = args.length - keywords.length;
if (nargs > 1) {
throw PyBuiltinCallable.DefaultInfo.unexpectedCall(nargs, false, methName, 0, 1);
diff --git a/src/org/python/modules/Setup.java b/src/org/python/modules/Setup.java
--- a/src/org/python/modules/Setup.java
+++ b/src/org/python/modules/Setup.java
@@ -27,41 +27,42 @@
// python.modules.builtin for details.
public static String[] builtinModules = {
- "jarray",
- "math",
- "thread:org.python.modules.thread.thread",
- "operator",
- "time:org.python.modules.time.Time",
+ "_ast:org.python.antlr.ast.AstModule",
+ "_codecs",
+ "_collections:org.python.modules._collections.Collections",
+ "_csv:org.python.modules._csv._csv",
+ "_functools:org.python.modules._functools._functools",
+ "_hashlib",
+ "_io:org.python.modules._io._io",
+ "_jythonlib:org.python.modules._jythonlib._jythonlib",
+ "_marshal",
"_py_compile",
+ "_random:org.python.modules.random.RandomModule",
"_sre",
- "synchronize",
+ "_systemrestart",
+ "_threading:org.python.modules._threading._threading",
+ "_weakref:org.python.modules._weakref.WeakrefModule",
+ "array:org.python.modules.ArrayModule",
+ "binascii",
+ "bz2:org.python.modules.bz2.bz2",
"cPickle",
"cStringIO",
+ "cmath",
+ "errno",
+ "exceptions:org.python.core.exceptions",
+ "gc",
+ "imp",
+ "itertools:org.python.modules.itertools.itertools",
+ "jarray",
+ "jffi:org.python.modules.jffi.jffi",
+ "math",
+ "operator",
"struct",
- "binascii",
- "exceptions:org.python.core.exceptions",
- "_codecs",
- "imp",
+ "synchronize",
+ "thread:org.python.modules.thread.thread",
+ "time:org.python.modules.time.Time",
"ucnhash",
- "_weakref:org.python.modules._weakref.WeakrefModule",
- "errno",
- "array:org.python.modules.ArrayModule",
- "_random:org.python.modules.random.RandomModule",
- "cmath",
- "itertools:org.python.modules.itertools.itertools",
"zipimport:org.python.modules.zipimport.zipimport",
- "_collections:org.python.modules._collections.Collections",
- "gc",
- "_hashlib",
- "_functools:org.python.modules._functools._functools",
- "_csv:org.python.modules._csv._csv",
- "_systemrestart",
- "_ast:org.python.antlr.ast.AstModule",
- "_marshal",
- "_threading:org.python.modules._threading._threading",
- PosixModule.getOSName() + ":org.python.modules.posix.PosixModule",
- "jffi:org.python.modules.jffi.jffi",
- "_io:org.python.modules._io._io",
- "bz2:org.python.modules.bz2.bz2"
+ PosixModule.getOSName() + ":org.python.modules.posix.PosixModule"
};
}
diff --git a/src/org/python/modules/_jythonlib/_jythonlib.java b/src/org/python/modules/_jythonlib/_jythonlib.java
new file mode 100644
--- /dev/null
+++ b/src/org/python/modules/_jythonlib/_jythonlib.java
@@ -0,0 +1,23 @@
+/* Copyright (c) Jython Developers */
+package org.python.modules._jythonlib;
+
+import org.python.core.ClassDictInit;
+import org.python.core.PyObject;
+import org.python.core.PyString;
+
+
+public class _jythonlib implements ClassDictInit {
+
+ public static final PyString __doc__ = new PyString("jythonlib module");
+
+ public static void classDictInit(PyObject dict) {
+ dict.__setitem__("__name__", new PyString("_jythonlib"));
+ dict.__setitem__("__doc__", __doc__);
+ dict.__setitem__("__module__", new PyString("_jythonlib"));
+ dict.__setitem__("dict_builder", dict_builder.TYPE);
+
+ // Hide from Python
+ dict.__setitem__("classDictInit", null);
+ }
+
+}
diff --git a/src/org/python/modules/_jythonlib/dict_builder.java b/src/org/python/modules/_jythonlib/dict_builder.java
new file mode 100644
--- /dev/null
+++ b/src/org/python/modules/_jythonlib/dict_builder.java
@@ -0,0 +1,40 @@
+/* Copyright (c) Jython Developers */
+package org.python.modules._jythonlib;
+
+import org.python.core.PyDictionary;
+import org.python.core.PyObject;
+import org.python.core.PyType;
+
+import java.util.concurrent.ConcurrentMap;
+
+
+/* Support building PyDictionary objects with arbitrary backing ConcurrentMap objects
+ * Uses a factory for efficiency.
+ * Usage from Python: _threads = dict_builder(MapMaker().weakValues().makeMap)()
+ *
+ * Such usage avoids problems with converting from boxed Python objects to their unboxed
+ * versions, compared to building this in Java and exporting to Python via PyJavaType.
+ * So in the above example_threads dict maps from thread ID to JavaThread.
+ * But thread ID is a long, which meant that we did not have equality between long/int, as in Python
+ * JavaThread also exposes __java__ to convert to the backing thread, but this meant this would
+ * also be unboxed this way, so the wrapping thread could not be looked up!
+ */
+
+public class dict_builder extends PyObject {
+
+ public static final PyType TYPE = PyType.fromClass(dict_builder.class);
+ private final PyObject factory;
+
+ public dict_builder(PyObject factory) {
+ super();
+ this.factory = factory;
+ }
+
+ public PyObject __call__(PyObject[] args, String[] keywords) {
+ ConcurrentMap map = (ConcurrentMap) (factory.__call__().__tojava__(ConcurrentMap.class));
+ PyDictionary dict = new PyDictionary(map, true);
+ dict.updateCommon(args, keywords, "dict");
+ return dict;
+ }
+
+}
diff --git a/src/org/python/modules/_threading/_threading.java b/src/org/python/modules/_threading/_threading.java
--- a/src/org/python/modules/_threading/_threading.java
+++ b/src/org/python/modules/_threading/_threading.java
@@ -3,8 +3,7 @@
import org.python.core.ClassDictInit;
import org.python.core.Py;
import org.python.core.PyObject;
-import com.google.common.collect.MapMaker;
-import java.util.Map;
+
public class _threading implements ClassDictInit {
@@ -15,21 +14,9 @@
dict.__setitem__("_Lock", Lock.TYPE);
dict.__setitem__("_RLock", Lock.TYPE);
dict.__setitem__("Condition", Condition.TYPE);
-// dict.__setitem__("JavaThread", JavaThread.TYPE);
- }
- // internals to support threading.py, test_threading.py
- public static Map<Long, PyObject> _threads = new MapMaker().weakValues().makeMap();
- public static Map<Thread, PyObject> _jthread_to_pythread = new MapMaker().weakKeys().weakValues().makeMap();
- public static Map<Long, PyObject> _active = _threads;
-
- public static void _register_thread(Thread jthread, PyObject pythread) {
- _threads.put(jthread.getId(), pythread);
- _jthread_to_pythread.put(jthread, pythread);
- }
-
- public static void _unregister_thread(Thread jthread) {
- _threads.remove(jthread.getId());
+ // Hide from Python
+ dict.__setitem__("classDictInit", null);
}
}
--
Repository URL: http://hg.python.org/jython
More information about the Jython-checkins
mailing list