[Python-checkins] bpo-37935: Added tests for os.walk(), glob.iglob() and Path.glob() (GH-15956) (GH-16043)

[Gregory P. Smith]

https://github.com/python/cpython/commit/98a4a713d001cf2dfb04a9e318e4aea899bc8fbd
commit: 98a4a713d001cf2dfb04a9e318e4aea899bc8fbd
branch: 3.8
author: Miss Islington (bot) <31488909+miss-islington at users.noreply.github.com>
committer: Gregory P. Smith <greg at krypto.org>
date: 2019-09-12T16:07:47+01:00
summary:

bpo-37935: Added tests for os.walk(), glob.iglob() and Path.glob() (GH-15956) (GH-16043)

Test that they do not keep too many file descriptors open for the host OS in a reasonable test scenario.

See [bpo-37935](https://bugs.python.org/issue37935).
(cherry picked from commit f9dc2ad89032201427ed5f08061c703794627ad9)

Co-authored-by: Serhiy Storchaka <storchaka at gmail.com>

files:
M Lib/test/test_glob.py
M Lib/test/test_os.py
M Lib/test/test_pathlib.py

diff --git a/Lib/test/test_glob.py b/Lib/test/test_glob.py
index 767bd3764b89..cba8c7c60e21 100644
--- a/Lib/test/test_glob.py
+++ b/Lib/test/test_glob.py
@@ -264,6 +264,23 @@ def test_recursive_glob(self):
                 expect += [join('sym3', 'EF')]
             eq(glob.glob(join('**', 'EF'), recursive=True), expect)
 
+    def test_glob_many_open_files(self):
+        depth = 30
+        base = os.path.join(self.tempdir, 'deep')
+        p = os.path.join(base, *(['d']*depth))
+        os.makedirs(p)
+        pattern = os.path.join(base, *(['*']*depth))
+        iters = [glob.iglob(pattern, recursive=True) for j in range(100)]
+        for it in iters:
+            self.assertEqual(next(it), p)
+        pattern = os.path.join(base, '**', 'd')
+        iters = [glob.iglob(pattern, recursive=True) for j in range(100)]
+        p = base
+        for i in range(depth):
+            p = os.path.join(p, 'd')
+            for it in iters:
+                self.assertEqual(next(it), p)
+
 
 @skip_unless_symlink
 class SymlinkLoopGlobTests(unittest.TestCase):
diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py
index 8ff0296fad0b..4a076e3bbf54 100644
--- a/Lib/test/test_os.py
+++ b/Lib/test/test_os.py
@@ -1178,6 +1178,27 @@ def test_walk_bad_dir(self):
         finally:
             os.rename(path1new, path1)
 
+    def test_walk_many_open_files(self):
+        depth = 30
+        base = os.path.join(support.TESTFN, 'deep')
+        p = os.path.join(base, *(['d']*depth))
+        os.makedirs(p)
+
+        iters = [self.walk(base, topdown=False) for j in range(100)]
+        for i in range(depth + 1):
+            expected = (p, ['d'] if i else [], [])
+            for it in iters:
+                self.assertEqual(next(it), expected)
+            p = os.path.dirname(p)
+
+        iters = [self.walk(base, topdown=True) for j in range(100)]
+        p = base
+        for i in range(depth + 1):
+            expected = (p, ['d'] if i < depth else [], [])
+            for it in iters:
+                self.assertEqual(next(it), expected)
+            p = os.path.join(p, 'd')
+
 
 @unittest.skipUnless(hasattr(os, 'fwalk'), "Test needs os.fwalk()")
 class FwalkTests(WalkTests):
@@ -1247,6 +1268,10 @@ def test_fd_leak(self):
         self.addCleanup(os.close, newfd)
         self.assertEqual(newfd, minfd)
 
+    # fwalk() keeps file descriptors open
+    test_walk_many_open_files = None
+
+
 class BytesWalkTests(WalkTests):
     """Tests for os.walk() with bytes."""
     def walk(self, top, **kwargs):
diff --git a/Lib/test/test_pathlib.py b/Lib/test/test_pathlib.py
index ebc71ec219c8..95d0b09882f6 100644
--- a/Lib/test/test_pathlib.py
+++ b/Lib/test/test_pathlib.py
@@ -1475,6 +1475,23 @@ def test_rglob_symlink_loop(self):
                   }
         self.assertEqual(given, {p / x for x in expect})
 
+    def test_glob_many_open_files(self):
+        depth = 30
+        P = self.cls
+        base = P(BASE) / 'deep'
+        p = P(base, *(['d']*depth))
+        p.mkdir(parents=True)
+        pattern = '/'.join(['*'] * depth)
+        iters = [base.glob(pattern) for j in range(100)]
+        for it in iters:
+            self.assertEqual(next(it), p)
+        iters = [base.rglob('d') for j in range(100)]
+        p = base
+        for i in range(depth):
+            p = p / 'd'
+            for it in iters:
+                self.assertEqual(next(it), p)
+
     def test_glob_dotdot(self):
         # ".." is not special in globs.
         P = self.cls