[Jython-checkins] jython: Reduce usage of files and threads in regrtest

[jim.baker]

https://hg.python.org/jython/rev/112f911545bb
changeset:   7396:112f911545bb
user:        Jim Baker <jim.baker at rackspace.com>
date:        Thu Oct 09 20:49:27 2014 -0600
summary:
  Reduce usage of files and threads in regrtest

Some of the tests, such as test_socket or test_xmlrpc, seem to
implicitly rely on deterministic destruction, given that using
gc.collect() will close out sockets and corresponding thread
pools. test_tarfile checks sys.platform for how many files to open,
but was not aware of running on sys.platform.startswith('java').

Lastly we have a resource leak on closing out the last thread in
thread pools for server sockets, and this causes issues with general
thread allocation after enough server sockets are opened. Need to
revisit with respect to Netty 4.1 which supports a better thread pool
and scheduling scheme via Fork Join. This probably is only seen in
testing however.

files:
  Lib/_socket.py            |  8 +++++---
  Lib/test/test_socket.py   |  5 +++++
  Lib/test/test_tarfile.py  |  5 +++++
  Lib/test/test_tempfile.py |  2 ++
  Lib/test/test_xmlrpc.py   |  9 +++++++++
  5 files changed, 26 insertions(+), 3 deletions(-)


diff --git a/Lib/_socket.py b/Lib/_socket.py
--- a/Lib/_socket.py
+++ b/Lib/_socket.py
@@ -189,6 +189,8 @@
 # Event loop management
 #######################
 
+_NUM_THREADS = 10
+
 # Use daemon threads for the event loop group. This is just fine
 # because these threads only handle ephemeral data, such as performing
 # SSL wrap/unwrap.
@@ -208,7 +210,7 @@
         return t
 
 
-NIO_GROUP = NioEventLoopGroup(10, DaemonThreadFactory("Jython-Netty-Client-%s"))
+NIO_GROUP = NioEventLoopGroup(_NUM_THREADS, DaemonThreadFactory("Jython-Netty-Client-%s"))
 
 
 def _check_threadpool_for_pending_threads(group):
@@ -882,8 +884,8 @@
         self.accepted_children = 1  # include the parent as well to simplify close logic
 
         b = ServerBootstrap()
-        self.parent_group = NioEventLoopGroup(10, DaemonThreadFactory("Jython-Netty-Parent-%s"))
-        self.child_group = NioEventLoopGroup(10, DaemonThreadFactory("Jython-Netty-Child-%s"))
+        self.parent_group = NioEventLoopGroup(_NUM_THREADS, DaemonThreadFactory("Jython-Netty-Parent-%s"))
+        self.child_group = NioEventLoopGroup(_NUM_THREADS, DaemonThreadFactory("Jython-Netty-Child-%s"))
         b.group(self.parent_group, self.child_group)
         b.channel(NioServerSocketChannel)
         b.option(ChannelOption.SO_BACKLOG, backlog)
diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py
--- a/Lib/test/test_socket.py
+++ b/Lib/test/test_socket.py
@@ -27,6 +27,11 @@
 is_bsd = os_name == 'Mac OS X' or 'BSD' in os_name
 is_solaris = os_name == 'SunOS'
 
+if test_support.is_jython:
+    import _socket
+    _socket._NUM_THREADS = 5
+
+
 class SocketTCPTest(unittest.TestCase):
 
     HOST = HOST
diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py
--- a/Lib/test/test_tarfile.py
+++ b/Lib/test/test_tarfile.py
@@ -6,6 +6,7 @@
 import StringIO
 from hashlib import md5
 import errno
+import gc
 
 import unittest
 import tarfile
@@ -46,6 +47,10 @@
 
     def tearDown(self):
         self.tar.close()
+        # Not all files are currently being closed in these tests,
+        # so to ensure something similar to CPython's deterministic cleanup,
+        # call gc and have finalization happen
+        gc.collect()
 
 
 class UstarReadTest(ReadTest):
diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py
--- a/Lib/test/test_tempfile.py
+++ b/Lib/test/test_tempfile.py
@@ -29,6 +29,8 @@
     TEST_FILES = 32
 elif sys.platform in ('openbsd3', 'openbsd4'):
     TEST_FILES = 48
+elif sys.platform.startswith("java"):
+    TEST_FILES = 32  # uniformly have a small number
 else:
     TEST_FILES = 100
 
diff --git a/Lib/test/test_xmlrpc.py b/Lib/test/test_xmlrpc.py
--- a/Lib/test/test_xmlrpc.py
+++ b/Lib/test/test_xmlrpc.py
@@ -1,5 +1,6 @@
 import base64
 import datetime
+import gc
 import sys
 import time
 import unittest
@@ -25,6 +26,10 @@
 else:
     have_unicode = True
 
+if test_support.is_jython:
+    import _socket
+    _socket._NUM_THREADS = 5
+
 alist = [{'astring': 'foo at bar.baz.spam',
           'afloat': 7283.43,
           'anint': 2**20,
@@ -443,6 +448,10 @@
         # disable traceback reporting
         SimpleXMLRPCServer.SimpleXMLRPCServer._send_traceback_header = False
 
+        # force finalization for tests that rely on deterministic
+        # destruction because of ref counting on CPython
+        gc.collect()
+
 # NOTE: The tests in SimpleServerTestCase will ignore failures caused by
 # "temporarily unavailable" exceptions raised in SimpleXMLRPCServer.  This
 # condition occurs infrequently on some platforms, frequently on others, and

-- 
Repository URL: https://hg.python.org/jython