[Python-checkins] bpo-33353: test_asyncio use set_write_buffer_limits() (GH-7200)

[Victor Stinner]

https://github.com/python/cpython/commit/9551f7719213243fd96c4f284079243773c26b3c
commit: 9551f7719213243fd96c4f284079243773c26b3c
branch: master
author: Victor Stinner <vstinner at redhat.com>
committer: GitHub <noreply at github.com>
date: 2018-05-29T16:02:07+02:00
summary:

bpo-33353: test_asyncio use set_write_buffer_limits() (GH-7200)

Use transport.set_write_buffer_limits() in sendfile tests of
test_asyncio to make sure that the protocol is paused after sending
4 KiB. Previously,
test_sendfile_fallback_close_peer_in_the_middle_of_receiving() failed
on FreeBSD if the DATA was smaller than the default limit of 64 KiB.

files:
M Lib/asyncio/streams.py
M Lib/test/test_asyncio/test_events.py

diff --git a/Lib/asyncio/streams.py b/Lib/asyncio/streams.py
index 9a53ee482478..d6531f88a74d 100644
--- a/Lib/asyncio/streams.py
+++ b/Lib/asyncio/streams.py
@@ -16,7 +16,7 @@
 from .tasks import sleep
 
 
-_DEFAULT_LIMIT = 2 ** 16
+_DEFAULT_LIMIT = 2 ** 16  # 64 KiB
 
 
 class IncompleteReadError(EOFError):
diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py
index e7c4fa6cecec..01ed47b36495 100644
--- a/Lib/test/test_asyncio/test_events.py
+++ b/Lib/test/test_asyncio/test_events.py
@@ -2113,7 +2113,10 @@ def test_subprocess_shell_invalid_args(self):
 
 class SendfileBase:
 
-    DATA = b"12345abcde" * 64 * 1024  # 64 KiB (don't use smaller sizes)
+    DATA = b"SendfileBaseData" * (1024 * 8)  # 128 KiB
+
+    # Reduce socket buffer size to test on relative small data sets.
+    BUF_SIZE = 4 * 1024   # 4 KiB
 
     @classmethod
     def setUpClass(cls):
@@ -2171,12 +2174,6 @@ def tearDownClass(cls):
         constants.SENDFILE_FALLBACK_READBUFFER_SIZE = cls.__old_bufsize
         super().tearDownClass()
 
-    def set_socket_opts(self, sock):
-        # On macOS, SO_SNDBUF is reset by connect(). So this method
-        # should be called after the socket is connected.
-        sock.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 1024)
-        sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 1024)
-
     def make_socket(self, cleanup=True):
         sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         sock.setblocking(False)
@@ -2184,17 +2181,33 @@ def make_socket(self, cleanup=True):
             self.addCleanup(sock.close)
         return sock
 
+    def reduce_receive_buffer_size(self, sock):
+        # Reduce receive socket buffer size to test on relative
+        # small data sets.
+        sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, self.BUF_SIZE)
+
+    def reduce_send_buffer_size(self, sock, transport=None):
+        # Reduce send socket buffer size to test on relative small data sets.
+
+        # On macOS, SO_SNDBUF is reset by connect(). So this method
+        # should be called after the socket is connected.
+        sock.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, self.BUF_SIZE)
+
+        if transport is not None:
+            transport.set_write_buffer_limits(high=self.BUF_SIZE)
+
     def prepare_socksendfile(self):
-        sock = self.make_socket()
         proto = self.MyProto(self.loop)
         port = support.find_unused_port()
         srv_sock = self.make_socket(cleanup=False)
         srv_sock.bind((support.HOST, port))
         server = self.run_loop(self.loop.create_server(
             lambda: proto, sock=srv_sock))
-        self.set_socket_opts(srv_sock)
+        self.reduce_receive_buffer_size(srv_sock)
+
+        sock = self.make_socket()
         self.run_loop(self.loop.sock_connect(sock, ('127.0.0.1', port)))
-        self.set_socket_opts(sock)
+        self.reduce_send_buffer_size(sock)
 
         def cleanup():
             if proto.transport is not None:
@@ -2243,7 +2256,7 @@ def test_sock_sendfile_zero_size(self):
         self.assertEqual(self.file.tell(), 0)
 
     def test_sock_sendfile_mix_with_regular_send(self):
-        buf = b"X" * 160 * 1024  # 160 KiB
+        buf = b"mix_regular_send" * (4 * 1024)  # 64 KiB
         sock, proto = self.prepare_socksendfile()
         self.run_loop(self.loop.sock_sendall(sock, buf))
         ret = self.run_loop(self.loop.sock_sendfile(sock, self.file))
@@ -2288,11 +2301,10 @@ def prepare_sendfile(self, *, is_ssl=False, close_after=0):
             srv_ctx = None
             cli_ctx = None
         srv_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-        # reduce recv socket buffer size to test on relative small data sets
-        srv_sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 1024)
         srv_sock.bind((support.HOST, port))
         server = self.run_loop(self.loop.create_server(
             lambda: srv_proto, sock=srv_sock, ssl=srv_ctx))
+        self.reduce_receive_buffer_size(srv_sock)
 
         if is_ssl:
             server_hostname = support.HOST
@@ -2300,12 +2312,12 @@ def prepare_sendfile(self, *, is_ssl=False, close_after=0):
             server_hostname = None
         cli_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         cli_sock.connect((support.HOST, port))
-        # reduce send socket buffer size to test on relative small data sets
-        cli_sock.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 1024)
+
         cli_proto = self.MySendfileProto(loop=self.loop)
         tr, pr = self.run_loop(self.loop.create_connection(
             lambda: cli_proto, sock=cli_sock,
             ssl=cli_ctx, server_hostname=server_hostname))
+        self.reduce_send_buffer_size(cli_sock, transport=tr)
 
         def cleanup():
             srv_proto.transport.close()
@@ -2410,8 +2422,8 @@ def test_sendfile_for_closing_transp(self):
 
     def test_sendfile_pre_and_post_data(self):
         srv_proto, cli_proto = self.prepare_sendfile()
-        PREFIX = b'zxcvbnm' * 1024
-        SUFFIX = b'0987654321' * 1024
+        PREFIX = b'PREFIX__' * 1024  # 8 KiB
+        SUFFIX = b'--SUFFIX' * 1024  # 8 KiB
         cli_proto.transport.write(PREFIX)
         ret = self.run_loop(
             self.loop.sendfile(cli_proto.transport, self.file))