[Python-checkins] bpo-47015: Update test_os from asyncore to asyncio (GH-31876)
asvetlov
webhook-mailer at python.org
Sun Mar 20 10:39:34 EDT 2022
https://github.com/python/cpython/commit/3ae975f1ac880c47d51cca6c9e305547bd365be7
commit: 3ae975f1ac880c47d51cca6c9e305547bd365be7
branch: main
author: Oleg Iarygin <oleg at arhadthedev.net>
committer: asvetlov <andrew.svetlov at gmail.com>
date: 2022-03-20T16:39:12+02:00
summary:
bpo-47015: Update test_os from asyncore to asyncio (GH-31876)
files:
A Misc/NEWS.d/next/Tests/2022-03-13-23-43-40.bpo-47015.FjmCsz.rst
M Lib/test/test_os.py
diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py
index 426d2ae9373bc..9a60bf5f7fdd5 100644
--- a/Lib/test/test_os.py
+++ b/Lib/test/test_os.py
@@ -2,6 +2,7 @@
# does add tests for a few functions which have been determined to be more
# portable than they had been thought to be.
+import asyncio
import codecs
import contextlib
import decimal
@@ -23,7 +24,6 @@
import sys
import sysconfig
import tempfile
-import threading
import time
import types
import unittest
@@ -33,15 +33,9 @@
from test.support import import_helper
from test.support import os_helper
from test.support import socket_helper
-from test.support import threading_helper
from test.support import warnings_helper
from platform import win32_is_iot
-with warnings.catch_warnings():
- warnings.simplefilter('ignore', DeprecationWarning)
- import asynchat
- import asyncore
-
try:
import resource
except ImportError:
@@ -101,6 +95,10 @@ def create_file(filename, content=b'content'):
'on AIX, splice() only accepts sockets')
+def tearDownModule():
+ asyncio.set_event_loop_policy(None)
+
+
class MiscTests(unittest.TestCase):
def test_getcwd(self):
cwd = os.getcwd()
@@ -3228,94 +3226,8 @@ def test_set_get_priority(self):
raise
-class SendfileTestServer(asyncore.dispatcher, threading.Thread):
-
- class Handler(asynchat.async_chat):
-
- def __init__(self, conn):
- asynchat.async_chat.__init__(self, conn)
- self.in_buffer = []
- self.accumulate = True
- self.closed = False
- self.push(b"220 ready\r\n")
-
- def handle_read(self):
- data = self.recv(4096)
- if self.accumulate:
- self.in_buffer.append(data)
-
- def get_data(self):
- return b''.join(self.in_buffer)
-
- def handle_close(self):
- self.close()
- self.closed = True
-
- def handle_error(self):
- raise
-
- def __init__(self, address):
- threading.Thread.__init__(self)
- asyncore.dispatcher.__init__(self)
- self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
- self.bind(address)
- self.listen(5)
- self.host, self.port = self.socket.getsockname()[:2]
- self.handler_instance = None
- self._active = False
- self._active_lock = threading.Lock()
-
- # --- public API
-
- @property
- def running(self):
- return self._active
-
- def start(self):
- assert not self.running
- self.__flag = threading.Event()
- threading.Thread.start(self)
- self.__flag.wait()
-
- def stop(self):
- assert self.running
- self._active = False
- self.join()
-
- def wait(self):
- # wait for handler connection to be closed, then stop the server
- while not getattr(self.handler_instance, "closed", False):
- time.sleep(0.001)
- self.stop()
-
- # --- internals
-
- def run(self):
- self._active = True
- self.__flag.set()
- while self._active and asyncore.socket_map:
- self._active_lock.acquire()
- asyncore.loop(timeout=0.001, count=1)
- self._active_lock.release()
- asyncore.close_all()
-
- def handle_accept(self):
- conn, addr = self.accept()
- self.handler_instance = self.Handler(conn)
-
- def handle_connect(self):
- self.close()
- handle_read = handle_connect
-
- def writable(self):
- return 0
-
- def handle_error(self):
- raise
-
-
@unittest.skipUnless(hasattr(os, 'sendfile'), "test needs os.sendfile()")
-class TestSendfile(unittest.TestCase):
+class TestSendfile(unittest.IsolatedAsyncioTestCase):
DATA = b"12345abcde" * 16 * 1024 # 160 KiB
SUPPORT_HEADERS_TRAILERS = not sys.platform.startswith("linux") and \
@@ -3328,40 +3240,52 @@ class TestSendfile(unittest.TestCase):
@classmethod
def setUpClass(cls):
- cls.key = threading_helper.threading_setup()
create_file(os_helper.TESTFN, cls.DATA)
@classmethod
def tearDownClass(cls):
- threading_helper.threading_cleanup(*cls.key)
os_helper.unlink(os_helper.TESTFN)
- def setUp(self):
- self.server = SendfileTestServer((socket_helper.HOST, 0))
- self.server.start()
+ @staticmethod
+ async def chunks(reader):
+ while not reader.at_eof():
+ yield await reader.read()
+
+ async def handle_new_client(self, reader, writer):
+ self.server_buffer = b''.join([x async for x in self.chunks(reader)])
+ writer.close()
+ self.server.close() # The test server processes a single client only
+
+ async def asyncSetUp(self):
+ self.server_buffer = b''
+ self.server = await asyncio.start_server(self.handle_new_client,
+ socket_helper.HOSTv4)
+ server_name = self.server.sockets[0].getsockname()
self.client = socket.socket()
- self.client.connect((self.server.host, self.server.port))
- self.client.settimeout(1)
- # synchronize by waiting for "220 ready" response
- self.client.recv(1024)
+ self.client.setblocking(False)
+ await asyncio.get_running_loop().sock_connect(self.client, server_name)
self.sockno = self.client.fileno()
self.file = open(os_helper.TESTFN, 'rb')
self.fileno = self.file.fileno()
- def tearDown(self):
+ async def asyncTearDown(self):
self.file.close()
self.client.close()
- if self.server.running:
- self.server.stop()
- self.server = None
+ await self.server.wait_closed()
+
+ # Use the test subject instead of asyncio.loop.sendfile
+ @staticmethod
+ async def async_sendfile(*args, **kwargs):
+ return await asyncio.to_thread(os.sendfile, *args, **kwargs)
- def sendfile_wrapper(self, *args, **kwargs):
+ @staticmethod
+ async def sendfile_wrapper(*args, **kwargs):
"""A higher level wrapper representing how an application is
supposed to use sendfile().
"""
while True:
try:
- return os.sendfile(*args, **kwargs)
+ return await TestSendfile.async_sendfile(*args, **kwargs)
except OSError as err:
if err.errno == errno.ECONNRESET:
# disconnected
@@ -3372,13 +3296,14 @@ def sendfile_wrapper(self, *args, **kwargs):
else:
raise
- def test_send_whole_file(self):
+ async def test_send_whole_file(self):
# normal send
total_sent = 0
offset = 0
nbytes = 4096
while total_sent < len(self.DATA):
- sent = self.sendfile_wrapper(self.sockno, self.fileno, offset, nbytes)
+ sent = await self.sendfile_wrapper(self.sockno, self.fileno,
+ offset, nbytes)
if sent == 0:
break
offset += sent
@@ -3389,19 +3314,19 @@ def test_send_whole_file(self):
self.assertEqual(total_sent, len(self.DATA))
self.client.shutdown(socket.SHUT_RDWR)
self.client.close()
- self.server.wait()
- data = self.server.handler_instance.get_data()
- self.assertEqual(len(data), len(self.DATA))
- self.assertEqual(data, self.DATA)
+ await self.server.wait_closed()
+ self.assertEqual(len(self.server_buffer), len(self.DATA))
+ self.assertEqual(self.server_buffer, self.DATA)
- def test_send_at_certain_offset(self):
+ async def test_send_at_certain_offset(self):
# start sending a file at a certain offset
total_sent = 0
offset = len(self.DATA) // 2
must_send = len(self.DATA) - offset
nbytes = 4096
while total_sent < must_send:
- sent = self.sendfile_wrapper(self.sockno, self.fileno, offset, nbytes)
+ sent = await self.sendfile_wrapper(self.sockno, self.fileno,
+ offset, nbytes)
if sent == 0:
break
offset += sent
@@ -3410,18 +3335,18 @@ def test_send_at_certain_offset(self):
self.client.shutdown(socket.SHUT_RDWR)
self.client.close()
- self.server.wait()
- data = self.server.handler_instance.get_data()
+ await self.server.wait_closed()
expected = self.DATA[len(self.DATA) // 2:]
self.assertEqual(total_sent, len(expected))
- self.assertEqual(len(data), len(expected))
- self.assertEqual(data, expected)
+ self.assertEqual(len(self.server_buffer), len(expected))
+ self.assertEqual(self.server_buffer, expected)
- def test_offset_overflow(self):
+ async def test_offset_overflow(self):
# specify an offset > file size
offset = len(self.DATA) + 4096
try:
- sent = os.sendfile(self.sockno, self.fileno, offset, 4096)
+ sent = await self.async_sendfile(self.sockno, self.fileno,
+ offset, 4096)
except OSError as e:
# Solaris can raise EINVAL if offset >= file length, ignore.
if e.errno != errno.EINVAL:
@@ -3430,39 +3355,38 @@ def test_offset_overflow(self):
self.assertEqual(sent, 0)
self.client.shutdown(socket.SHUT_RDWR)
self.client.close()
- self.server.wait()
- data = self.server.handler_instance.get_data()
- self.assertEqual(data, b'')
+ await self.server.wait_closed()
+ self.assertEqual(self.server_buffer, b'')
- def test_invalid_offset(self):
+ async def test_invalid_offset(self):
with self.assertRaises(OSError) as cm:
- os.sendfile(self.sockno, self.fileno, -1, 4096)
+ await self.async_sendfile(self.sockno, self.fileno, -1, 4096)
self.assertEqual(cm.exception.errno, errno.EINVAL)
- def test_keywords(self):
+ async def test_keywords(self):
# Keyword arguments should be supported
- os.sendfile(out_fd=self.sockno, in_fd=self.fileno,
- offset=0, count=4096)
+ await self.async_sendfile(out_fd=self.sockno, in_fd=self.fileno,
+ offset=0, count=4096)
if self.SUPPORT_HEADERS_TRAILERS:
- os.sendfile(out_fd=self.sockno, in_fd=self.fileno,
- offset=0, count=4096,
- headers=(), trailers=(), flags=0)
+ await self.async_sendfile(out_fd=self.sockno, in_fd=self.fileno,
+ offset=0, count=4096,
+ headers=(), trailers=(), flags=0)
# --- headers / trailers tests
@requires_headers_trailers
- def test_headers(self):
+ async def test_headers(self):
total_sent = 0
expected_data = b"x" * 512 + b"y" * 256 + self.DATA[:-1]
- sent = os.sendfile(self.sockno, self.fileno, 0, 4096,
- headers=[b"x" * 512, b"y" * 256])
+ sent = await self.async_sendfile(self.sockno, self.fileno, 0, 4096,
+ headers=[b"x" * 512, b"y" * 256])
self.assertLessEqual(sent, 512 + 256 + 4096)
total_sent += sent
offset = 4096
while total_sent < len(expected_data):
nbytes = min(len(expected_data) - total_sent, 4096)
- sent = self.sendfile_wrapper(self.sockno, self.fileno,
- offset, nbytes)
+ sent = await self.sendfile_wrapper(self.sockno, self.fileno,
+ offset, nbytes)
if sent == 0:
break
self.assertLessEqual(sent, nbytes)
@@ -3471,12 +3395,11 @@ def test_headers(self):
self.assertEqual(total_sent, len(expected_data))
self.client.close()
- self.server.wait()
- data = self.server.handler_instance.get_data()
- self.assertEqual(hash(data), hash(expected_data))
+ await self.server.wait_closed()
+ self.assertEqual(hash(self.server_buffer), hash(expected_data))
@requires_headers_trailers
- def test_trailers(self):
+ async def test_trailers(self):
TESTFN2 = os_helper.TESTFN + "2"
file_data = b"abcdef"
@@ -3484,38 +3407,37 @@ def test_trailers(self):
create_file(TESTFN2, file_data)
with open(TESTFN2, 'rb') as f:
- os.sendfile(self.sockno, f.fileno(), 0, 5,
- trailers=[b"123456", b"789"])
+ await self.async_sendfile(self.sockno, f.fileno(), 0, 5,
+ trailers=[b"123456", b"789"])
self.client.close()
- self.server.wait()
- data = self.server.handler_instance.get_data()
- self.assertEqual(data, b"abcde123456789")
+ await self.server.wait_closed()
+ self.assertEqual(self.server_buffer, b"abcde123456789")
@requires_headers_trailers
@requires_32b
- def test_headers_overflow_32bits(self):
+ async def test_headers_overflow_32bits(self):
self.server.handler_instance.accumulate = False
with self.assertRaises(OSError) as cm:
- os.sendfile(self.sockno, self.fileno, 0, 0,
- headers=[b"x" * 2**16] * 2**15)
+ await self.async_sendfile(self.sockno, self.fileno, 0, 0,
+ headers=[b"x" * 2**16] * 2**15)
self.assertEqual(cm.exception.errno, errno.EINVAL)
@requires_headers_trailers
@requires_32b
- def test_trailers_overflow_32bits(self):
+ async def test_trailers_overflow_32bits(self):
self.server.handler_instance.accumulate = False
with self.assertRaises(OSError) as cm:
- os.sendfile(self.sockno, self.fileno, 0, 0,
- trailers=[b"x" * 2**16] * 2**15)
+ await self.async_sendfile(self.sockno, self.fileno, 0, 0,
+ trailers=[b"x" * 2**16] * 2**15)
self.assertEqual(cm.exception.errno, errno.EINVAL)
@requires_headers_trailers
@unittest.skipUnless(hasattr(os, 'SF_NODISKIO'),
'test needs os.SF_NODISKIO')
- def test_flags(self):
+ async def test_flags(self):
try:
- os.sendfile(self.sockno, self.fileno, 0, 4096,
- flags=os.SF_NODISKIO)
+ await self.async_sendfile(self.sockno, self.fileno, 0, 4096,
+ flags=os.SF_NODISKIO)
except OSError as err:
if err.errno not in (errno.EBUSY, errno.EAGAIN):
raise
diff --git a/Misc/NEWS.d/next/Tests/2022-03-13-23-43-40.bpo-47015.FjmCsz.rst b/Misc/NEWS.d/next/Tests/2022-03-13-23-43-40.bpo-47015.FjmCsz.rst
new file mode 100644
index 0000000000000..12c527ad1f23b
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2022-03-13-23-43-40.bpo-47015.FjmCsz.rst
@@ -0,0 +1,2 @@
+A test case for :func:`os.sendfile` is converted from deprecated
+:mod:`asyncore` (see :pep:`594`) to :mod:`asyncio`. Patch by Oleg Iarygin.
More information about the Python-checkins
mailing list