[Python-checkins] cpython (3.4): Issue #23840: tokenize.open() now closes the temporary binary file on error to

[Terry Reedy]

On 5/25/2015 6:49 PM, victor.stinner wrote:
> https://hg.python.org/cpython/rev/623e07ea43df
> changeset:   96285:623e07ea43df
> branch:      3.4
> parent:      96280:a1bb4b18e3b5
> user:        Victor Stinner <victor.stinner at gmail.com>
> date:        Tue May 26 00:43:58 2015 +0200
> summary:
>    Issue #23840: tokenize.open() now closes the temporary binary file on error to
> fix a resource warning.
>
> files:
>    Lib/test/test_tokenize.py |  10 +++++++++-
>    Lib/tokenize.py           |  14 +++++++++-----
>    Misc/NEWS                 |   3 +++
>    3 files changed, 21 insertions(+), 6 deletions(-)
>
>
> diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
> --- a/Lib/test/test_tokenize.py
> +++ b/Lib/test/test_tokenize.py
> @@ -646,7 +646,7 @@
>                        STRING, ENDMARKER, ENCODING, tok_name, detect_encoding,
>                        open as tokenize_open, Untokenizer)
>   from io import BytesIO
> -from unittest import TestCase
> +from unittest import TestCase, mock
>   import os, sys, glob
>   import token
>
> @@ -1058,6 +1058,14 @@
>               ins = Bunk(lines, path)
>               detect_encoding(ins.readline)
>
> +    def test_open_error(self):
> +        # Issue #23840: open() must close the binary file on error
> +        m = BytesIO(b'#coding:xxx')
> +        with mock.patch('tokenize._builtin_open', return_value=m):
> +            self.assertRaises(SyntaxError, tokenize_open, 'foobar')
> +        self.assertTrue(m.closed)
> +
> +
>
>   class TestTokenize(TestCase):
>
> diff --git a/Lib/tokenize.py b/Lib/tokenize.py
> --- a/Lib/tokenize.py
> +++ b/Lib/tokenize.py
> @@ -435,11 +435,15 @@
>       detect_encoding().
>       """
>       buffer = _builtin_open(filename, 'rb')
> -    encoding, lines = detect_encoding(buffer.readline)
> -    buffer.seek(0)
> -    text = TextIOWrapper(buffer, encoding, line_buffering=True)
> -    text.mode = 'r'
> -    return text
> +    try:
> +        encoding, lines = detect_encoding(buffer.readline)
> +        buffer.seek(0)
> +        text = TextIOWrapper(buffer, encoding, line_buffering=True)
> +        text.mode = 'r'
> +        return text
> +    except:
> +        buffer.close()
> +        raise

Please do not add bare 'except:'.  If you mean 'except BaseException:', 
say so.

tjr