[Python-checkins] bpo-32410: Avoid blocking on file IO in sendfile fallback code (GH-7172)
Yury Selivanov
webhook-mailer at python.org
Mon May 28 18:31:57 EDT 2018
https://github.com/python/cpython/commit/7165754b6b5f3b7c07050d921fa1c58bba5f0ff1
commit: 7165754b6b5f3b7c07050d921fa1c58bba5f0ff1
branch: master
author: Yury Selivanov <yury at magic.io>
committer: GitHub <noreply at github.com>
date: 2018-05-28T18:31:55-04:00
summary:
bpo-32410: Avoid blocking on file IO in sendfile fallback code (GH-7172)
files:
A Misc/NEWS.d/next/Library/2018-05-28-16-19-35.bpo-32410.Z1DZaF.rst
M Lib/asyncio/base_events.py
M Lib/asyncio/constants.py
M Lib/test/test_asyncio/test_base_events.py
M Lib/test/test_asyncio/test_events.py
diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py
index a0243f5bac9a..ffd2513e33a6 100644
--- a/Lib/asyncio/base_events.py
+++ b/Lib/asyncio/base_events.py
@@ -800,7 +800,10 @@ def _getaddrinfo_debug(self, host, port, family, type, proto, flags):
async def _sock_sendfile_fallback(self, sock, file, offset, count):
if offset:
file.seek(offset)
- blocksize = min(count, 16384) if count else 16384
+ blocksize = (
+ min(count, constants.SENDFILE_FALLBACK_READBUFFER_SIZE)
+ if count else constants.SENDFILE_FALLBACK_READBUFFER_SIZE
+ )
buf = bytearray(blocksize)
total_sent = 0
try:
@@ -810,7 +813,7 @@ def _getaddrinfo_debug(self, host, port, family, type, proto, flags):
if blocksize <= 0:
break
view = memoryview(buf)[:blocksize]
- read = file.readinto(view)
+ read = await self.run_in_executor(None, file.readinto, view)
if not read:
break # EOF
await self.sock_sendall(sock, view)
diff --git a/Lib/asyncio/constants.py b/Lib/asyncio/constants.py
index 739b0a70c13e..d7ba49694289 100644
--- a/Lib/asyncio/constants.py
+++ b/Lib/asyncio/constants.py
@@ -14,6 +14,10 @@
# Number of seconds to wait for SSL handshake to complete
SSL_HANDSHAKE_TIMEOUT = 10.0
+# Used in sendfile fallback code. We use fallback for platforms
+# that don't support sendfile, or for TLS connections.
+SENDFILE_FALLBACK_READBUFFER_SIZE = 1024 * 256
+
# The enum should be here to break circular dependencies between
# base_events and sslproto
class _SendfileMode(enum.Enum):
diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py
index 8566a9d5504f..11e9465d3921 100644
--- a/Lib/test/test_asyncio/test_base_events.py
+++ b/Lib/test/test_asyncio/test_base_events.py
@@ -1818,12 +1818,15 @@ def connection_lost(self, exc):
@classmethod
def setUpClass(cls):
+ cls.__old_bufsize = constants.SENDFILE_FALLBACK_READBUFFER_SIZE
+ constants.SENDFILE_FALLBACK_READBUFFER_SIZE = 1024 * 16
with open(support.TESTFN, 'wb') as fp:
fp.write(cls.DATA)
super().setUpClass()
@classmethod
def tearDownClass(cls):
+ constants.SENDFILE_FALLBACK_READBUFFER_SIZE = cls.__old_bufsize
support.unlink(support.TESTFN)
super().tearDownClass()
diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py
index 39d85e8df07c..e7c4fa6cecec 100644
--- a/Lib/test/test_asyncio/test_events.py
+++ b/Lib/test/test_asyncio/test_events.py
@@ -2160,6 +2160,17 @@ def connection_lost(self, exc):
async def wait_closed(self):
await self.fut
+ @classmethod
+ def setUpClass(cls):
+ cls.__old_bufsize = constants.SENDFILE_FALLBACK_READBUFFER_SIZE
+ constants.SENDFILE_FALLBACK_READBUFFER_SIZE = 1024 * 16
+ super().setUpClass()
+
+ @classmethod
+ def tearDownClass(cls):
+ constants.SENDFILE_FALLBACK_READBUFFER_SIZE = cls.__old_bufsize
+ super().tearDownClass()
+
def set_socket_opts(self, sock):
# On macOS, SO_SNDBUF is reset by connect(). So this method
# should be called after the socket is connected.
diff --git a/Misc/NEWS.d/next/Library/2018-05-28-16-19-35.bpo-32410.Z1DZaF.rst b/Misc/NEWS.d/next/Library/2018-05-28-16-19-35.bpo-32410.Z1DZaF.rst
new file mode 100644
index 000000000000..2d7bb2032ac5
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2018-05-28-16-19-35.bpo-32410.Z1DZaF.rst
@@ -0,0 +1 @@
+Avoid blocking on file IO in sendfile fallback code
More information about the Python-checkins
mailing list