[Python-checkins] cpython (3.4): Fix asyncio issue 235: Queue subclass bug caused by JoinableQueue merge.

[guido.van.rossum]

https://hg.python.org/cpython/rev/8e0dc4d3b49c
changeset:   95742:8e0dc4d3b49c
branch:      3.4
parent:      95740:7f1622478d17
user:        Guido van Rossum <guido at python.org>
date:        Mon Apr 20 09:24:24 2015 -0700
summary:
  Fix asyncio issue 235: Queue subclass bug caused by JoinableQueue merge.

files:
  Lib/asyncio/queues.py                |  19 ++++++++----
  Lib/test/test_asyncio/test_queues.py |  24 ++++++++++++---
  Misc/NEWS                            |   4 ++
  3 files changed, 36 insertions(+), 11 deletions(-)


diff --git a/Lib/asyncio/queues.py b/Lib/asyncio/queues.py
--- a/Lib/asyncio/queues.py
+++ b/Lib/asyncio/queues.py
@@ -54,6 +54,8 @@
         self._finished.set()
         self._init(maxsize)
 
+    # These three are overridable in subclasses.
+
     def _init(self, maxsize):
         self._queue = collections.deque()
 
@@ -62,6 +64,11 @@
 
     def _put(self, item):
         self._queue.append(item)
+
+    # End of the overridable methods.
+
+    def __put_internal(self, item):
+        self._put(item)
         self._unfinished_tasks += 1
         self._finished.clear()
 
@@ -133,7 +140,7 @@
                 'queue non-empty, why are getters waiting?')
 
             getter = self._getters.popleft()
-            self._put(item)
+            self.__put_internal(item)
 
             # getter cannot be cancelled, we just removed done getters
             getter.set_result(self._get())
@@ -145,7 +152,7 @@
             yield from waiter
 
         else:
-            self._put(item)
+            self.__put_internal(item)
 
     def put_nowait(self, item):
         """Put an item into the queue without blocking.
@@ -158,7 +165,7 @@
                 'queue non-empty, why are getters waiting?')
 
             getter = self._getters.popleft()
-            self._put(item)
+            self.__put_internal(item)
 
             # getter cannot be cancelled, we just removed done getters
             getter.set_result(self._get())
@@ -166,7 +173,7 @@
         elif self._maxsize > 0 and self._maxsize <= self.qsize():
             raise QueueFull
         else:
-            self._put(item)
+            self.__put_internal(item)
 
     @coroutine
     def get(self):
@@ -180,7 +187,7 @@
         if self._putters:
             assert self.full(), 'queue not full, why are putters waiting?'
             item, putter = self._putters.popleft()
-            self._put(item)
+            self.__put_internal(item)
 
             # When a getter runs and frees up a slot so this putter can
             # run, we need to defer the put for a tick to ensure that
@@ -207,7 +214,7 @@
         if self._putters:
             assert self.full(), 'queue not full, why are putters waiting?'
             item, putter = self._putters.popleft()
-            self._put(item)
+            self.__put_internal(item)
             # Wake putter on next tick.
 
             # getter cannot be cancelled, we just removed done putters
diff --git a/Lib/test/test_asyncio/test_queues.py b/Lib/test/test_asyncio/test_queues.py
--- a/Lib/test/test_asyncio/test_queues.py
+++ b/Lib/test/test_asyncio/test_queues.py
@@ -408,14 +408,16 @@
         self.assertEqual([1, 2, 3], items)
 
 
-class QueueJoinTests(_QueueTestBase):
+class _QueueJoinTestMixin:
+
+    q_class = None
 
     def test_task_done_underflow(self):
-        q = asyncio.Queue(loop=self.loop)
+        q = self.q_class(loop=self.loop)
         self.assertRaises(ValueError, q.task_done)
 
     def test_task_done(self):
-        q = asyncio.Queue(loop=self.loop)
+        q = self.q_class(loop=self.loop)
         for i in range(100):
             q.put_nowait(i)
 
@@ -452,7 +454,7 @@
         self.loop.run_until_complete(asyncio.wait(tasks, loop=self.loop))
 
     def test_join_empty_queue(self):
-        q = asyncio.Queue(loop=self.loop)
+        q = self.q_class(loop=self.loop)
 
         # Test that a queue join()s successfully, and before anything else
         # (done twice for insurance).
@@ -465,12 +467,24 @@
         self.loop.run_until_complete(join())
 
     def test_format(self):
-        q = asyncio.Queue(loop=self.loop)
+        q = self.q_class(loop=self.loop)
         self.assertEqual(q._format(), 'maxsize=0')
 
         q._unfinished_tasks = 2
         self.assertEqual(q._format(), 'maxsize=0 tasks=2')
 
 
+class QueueJoinTests(_QueueJoinTestMixin, _QueueTestBase):
+    q_class = asyncio.Queue
+
+
+class LifoQueueJoinTests(_QueueJoinTestMixin, _QueueTestBase):
+    q_class = asyncio.LifoQueue
+
+
+class PriorityQueueJoinTests(_QueueJoinTestMixin, _QueueTestBase):
+    q_class = asyncio.PriorityQueue
+
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -29,6 +29,10 @@
 Library
 -------
 
+- Fix asyncio issue 235: LifoQueue and PriorityQueue's put didn't
+  increment unfinished tasks (this bug was introduced in 3.4.3 when
+  JoinableQueue was merged with Queue).
+
 - Issue #23908: os functions now reject paths with embedded null character
   on Windows instead of silently truncate them.
 

-- 
Repository URL: https://hg.python.org/cpython