[Python-checkins] bpo-28806: Continue work: improve the netrc library (GH-26330)
asvetlov
webhook-mailer at python.org
Wed Nov 17 04:08:40 EST 2021
https://github.com/python/cpython/commit/15409c720be0503131713e3d3abc1acd0da07378
commit: 15409c720be0503131713e3d3abc1acd0da07378
branch: main
author: Emmanuel Arias <eamanu at yaerobi.com>
committer: asvetlov <andrew.svetlov at gmail.com>
date: 2021-11-17T11:07:54+02:00
summary:
bpo-28806: Continue work: improve the netrc library (GH-26330)
Continue with the improvement of the library netrc
Original work and report Xiang Zhang <angwerzx at 126.com>
* 📜🤖 Added by blurb_it.
Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com>
files:
A Misc/NEWS.d/next/Library/2021-05-24-13-48-34.bpo-28806.PkNw5D.rst
M Doc/library/netrc.rst
M Lib/netrc.py
M Lib/test/test_netrc.py
diff --git a/Doc/library/netrc.rst b/Doc/library/netrc.rst
index 4bf7de67c1d03..88265d9b9e9e9 100644
--- a/Doc/library/netrc.rst
+++ b/Doc/library/netrc.rst
@@ -41,6 +41,10 @@ the Unix :program:`ftp` program and other FTP clients.
.. versionchanged:: 3.10
:class:`netrc` try UTF-8 encoding before using locale specific
encoding.
+ The entry in the netrc file no longer needs to contain all tokens. The missing
+ tokens' value default to an empty string. All the tokens and their values now
+ can contain arbitrary characters, like whitespace and non-ASCII characters.
+ If the login name is anonymous, it won't trigger the security check.
.. exception:: NetrcParseError
@@ -85,10 +89,3 @@ Instances of :class:`~netrc.netrc` have public instance variables:
.. attribute:: netrc.macros
Dictionary mapping macro names to string lists.
-
-.. note::
-
- Passwords are limited to a subset of the ASCII character set. All ASCII
- punctuation is allowed in passwords, however, note that whitespace and
- non-printable characters are not allowed in passwords. This is a limitation
- of the way the .netrc file is parsed and may be removed in the future.
diff --git a/Lib/netrc.py b/Lib/netrc.py
index 734d94c8a6285..c1358aac6ae02 100644
--- a/Lib/netrc.py
+++ b/Lib/netrc.py
@@ -19,6 +19,50 @@ def __str__(self):
return "%s (%s, line %s)" % (self.msg, self.filename, self.lineno)
+class _netrclex:
+ def __init__(self, fp):
+ self.lineno = 1
+ self.instream = fp
+ self.whitespace = "\n\t\r "
+ self.pushback = []
+
+ def _read_char(self):
+ ch = self.instream.read(1)
+ if ch == "\n":
+ self.lineno += 1
+ return ch
+
+ def get_token(self):
+ if self.pushback:
+ return self.pushback.pop(0)
+ token = ""
+ fiter = iter(self._read_char, "")
+ for ch in fiter:
+ if ch in self.whitespace:
+ continue
+ if ch == '"':
+ for ch in fiter:
+ if ch == '"':
+ return token
+ elif ch == "\\":
+ ch = self._read_char()
+ token += ch
+ else:
+ if ch == "\\":
+ ch = self._read_char()
+ token += ch
+ for ch in fiter:
+ if ch in self.whitespace:
+ return token
+ elif ch == "\\":
+ ch = self._read_char()
+ token += ch
+ return token
+
+ def push_token(self, token):
+ self.pushback.append(token)
+
+
class netrc:
def __init__(self, file=None):
default_netrc = file is None
@@ -34,9 +78,7 @@ def __init__(self, file=None):
self._parse(file, fp, default_netrc)
def _parse(self, file, fp, default_netrc):
- lexer = shlex.shlex(fp)
- lexer.wordchars += r"""!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~"""
- lexer.commenters = lexer.commenters.replace('#', '')
+ lexer = _netrclex(fp)
while 1:
# Look for a machine, default, or macdef top-level keyword
saved_lineno = lexer.lineno
@@ -51,14 +93,19 @@ def _parse(self, file, fp, default_netrc):
entryname = lexer.get_token()
elif tt == 'default':
entryname = 'default'
- elif tt == 'macdef': # Just skip to end of macdefs
+ elif tt == 'macdef':
entryname = lexer.get_token()
self.macros[entryname] = []
- lexer.whitespace = ' \t'
while 1:
line = lexer.instream.readline()
- if not line or line == '\012':
- lexer.whitespace = ' \t\r\n'
+ if not line:
+ raise NetrcParseError(
+ "Macro definition missing null line terminator.",
+ file, lexer.lineno)
+ if line == '\n':
+ # a macro definition finished with consecutive new-line
+ # characters. The first \n is encountered by the
+ # readline() method and this is the second \n.
break
self.macros[entryname].append(line)
continue
@@ -66,53 +113,55 @@ def _parse(self, file, fp, default_netrc):
raise NetrcParseError(
"bad toplevel token %r" % tt, file, lexer.lineno)
+ if not entryname:
+ raise NetrcParseError("missing %r name" % tt, file, lexer.lineno)
+
# We're looking at start of an entry for a named machine or default.
- login = ''
- account = password = None
+ login = account = password = ''
self.hosts[entryname] = {}
while 1:
+ prev_lineno = lexer.lineno
tt = lexer.get_token()
- if (tt.startswith('#') or
- tt in {'', 'machine', 'default', 'macdef'}):
- if password:
- self.hosts[entryname] = (login, account, password)
- lexer.push_token(tt)
- break
- else:
- raise NetrcParseError(
- "malformed %s entry %s terminated by %s"
- % (toplevel, entryname, repr(tt)),
- file, lexer.lineno)
+ if tt.startswith('#'):
+ if lexer.lineno == prev_lineno:
+ lexer.instream.readline()
+ continue
+ if tt in {'', 'machine', 'default', 'macdef'}:
+ self.hosts[entryname] = (login, account, password)
+ lexer.push_token(tt)
+ break
elif tt == 'login' or tt == 'user':
login = lexer.get_token()
elif tt == 'account':
account = lexer.get_token()
elif tt == 'password':
- if os.name == 'posix' and default_netrc:
- prop = os.fstat(fp.fileno())
- if prop.st_uid != os.getuid():
- import pwd
- try:
- fowner = pwd.getpwuid(prop.st_uid)[0]
- except KeyError:
- fowner = 'uid %s' % prop.st_uid
- try:
- user = pwd.getpwuid(os.getuid())[0]
- except KeyError:
- user = 'uid %s' % os.getuid()
- raise NetrcParseError(
- ("~/.netrc file owner (%s) does not match"
- " current user (%s)") % (fowner, user),
- file, lexer.lineno)
- if (prop.st_mode & (stat.S_IRWXG | stat.S_IRWXO)):
- raise NetrcParseError(
- "~/.netrc access too permissive: access"
- " permissions must restrict access to only"
- " the owner", file, lexer.lineno)
password = lexer.get_token()
else:
raise NetrcParseError("bad follower token %r" % tt,
file, lexer.lineno)
+ self._security_check(fp, default_netrc, self.hosts[entryname][0])
+
+ def _security_check(self, fp, default_netrc, login):
+ if os.name == 'posix' and default_netrc and login != "anonymous":
+ prop = os.fstat(fp.fileno())
+ if prop.st_uid != os.getuid():
+ import pwd
+ try:
+ fowner = pwd.getpwuid(prop.st_uid)[0]
+ except KeyError:
+ fowner = 'uid %s' % prop.st_uid
+ try:
+ user = pwd.getpwuid(os.getuid())[0]
+ except KeyError:
+ user = 'uid %s' % os.getuid()
+ raise NetrcParseError(
+ (f"~/.netrc file owner ({fowner}, {user}) does not match"
+ " current user"))
+ if (prop.st_mode & (stat.S_IRWXG | stat.S_IRWXO)):
+ raise NetrcParseError(
+ "~/.netrc access too permissive: access"
+ " permissions must restrict access to only"
+ " the owner")
def authenticators(self, host):
"""Return a (user, account, password) tuple for given host."""
diff --git a/Lib/test/test_netrc.py b/Lib/test/test_netrc.py
index 90ef5cd363b3f..a6b4bc47a32c1 100644
--- a/Lib/test/test_netrc.py
+++ b/Lib/test/test_netrc.py
@@ -1,7 +1,7 @@
-import netrc, os, unittest, sys, tempfile, textwrap
-from test import support
-from test.support import os_helper
+import netrc, os, unittest, sys, textwrap
+from test.support import os_helper, run_unittest
+temp_filename = os_helper.TESTFN
class NetrcTestCase(unittest.TestCase):
@@ -10,26 +10,32 @@ def make_nrc(self, test_data):
mode = 'w'
if sys.platform != 'cygwin':
mode += 't'
- temp_fd, temp_filename = tempfile.mkstemp()
- with os.fdopen(temp_fd, mode=mode) as fp:
+ with open(temp_filename, mode) as fp:
fp.write(test_data)
- self.addCleanup(os.unlink, temp_filename)
- return netrc.netrc(temp_filename)
+ try:
+ nrc = netrc.netrc(temp_filename)
+ finally:
+ os.unlink(temp_filename)
+ return nrc
- def test_default(self):
+ def test_toplevel_non_ordered_tokens(self):
nrc = self.make_nrc("""\
- machine host1.domain.com login log1 password pass1 account acct1
- default login log2 password pass2
+ machine host.domain.com password pass1 login log1 account acct1
+ default login log2 password pass2 account acct2
""")
- self.assertEqual(nrc.hosts['host1.domain.com'],
- ('log1', 'acct1', 'pass1'))
- self.assertEqual(nrc.hosts['default'], ('log2', None, 'pass2'))
+ self.assertEqual(nrc.hosts['host.domain.com'], ('log1', 'acct1', 'pass1'))
+ self.assertEqual(nrc.hosts['default'], ('log2', 'acct2', 'pass2'))
- nrc2 = self.make_nrc(nrc.__repr__())
- self.assertEqual(nrc.hosts, nrc2.hosts)
+ def test_toplevel_tokens(self):
+ nrc = self.make_nrc("""\
+ machine host.domain.com login log1 password pass1 account acct1
+ default login log2 password pass2 account acct2
+ """)
+ self.assertEqual(nrc.hosts['host.domain.com'], ('log1', 'acct1', 'pass1'))
+ self.assertEqual(nrc.hosts['default'], ('log2', 'acct2', 'pass2'))
def test_macros(self):
- nrc = self.make_nrc("""\
+ data = """\
macdef macro1
line1
line2
@@ -37,33 +43,151 @@ def test_macros(self):
macdef macro2
line3
line4
- """)
+
+ """
+ nrc = self.make_nrc(data)
self.assertEqual(nrc.macros, {'macro1': ['line1\n', 'line2\n'],
'macro2': ['line3\n', 'line4\n']})
+ # strip the last \n
+ self.assertRaises(netrc.NetrcParseError, self.make_nrc,
+ data.rstrip(' ')[:-1])
+
+ def test_optional_tokens(self):
+ data = (
+ "machine host.domain.com",
+ "machine host.domain.com login",
+ "machine host.domain.com account",
+ "machine host.domain.com password",
+ "machine host.domain.com login \"\" account",
+ "machine host.domain.com login \"\" password",
+ "machine host.domain.com account \"\" password"
+ )
+ for item in data:
+ nrc = self.make_nrc(item)
+ self.assertEqual(nrc.hosts['host.domain.com'], ('', '', ''))
+ data = (
+ "default",
+ "default login",
+ "default account",
+ "default password",
+ "default login \"\" account",
+ "default login \"\" password",
+ "default account \"\" password"
+ )
+ for item in data:
+ nrc = self.make_nrc(item)
+ self.assertEqual(nrc.hosts['default'], ('', '', ''))
- def _test_passwords(self, nrc, passwd):
+ def test_invalid_tokens(self):
+ data = (
+ "invalid host.domain.com",
+ "machine host.domain.com invalid",
+ "machine host.domain.com login log password pass account acct invalid",
+ "default host.domain.com invalid",
+ "default host.domain.com login log password pass account acct invalid"
+ )
+ for item in data:
+ self.assertRaises(netrc.NetrcParseError, self.make_nrc, item)
+
+ def _test_token_x(self, nrc, token, value):
nrc = self.make_nrc(nrc)
- self.assertEqual(nrc.hosts['host.domain.com'], ('log', 'acct', passwd))
+ if token == 'login':
+ self.assertEqual(nrc.hosts['host.domain.com'], (value, 'acct', 'pass'))
+ elif token == 'account':
+ self.assertEqual(nrc.hosts['host.domain.com'], ('log', value, 'pass'))
+ elif token == 'password':
+ self.assertEqual(nrc.hosts['host.domain.com'], ('log', 'acct', value))
+
+ def test_token_value_quotes(self):
+ self._test_token_x("""\
+ machine host.domain.com login "log" password pass account acct
+ """, 'login', 'log')
+ self._test_token_x("""\
+ machine host.domain.com login log password pass account "acct"
+ """, 'account', 'acct')
+ self._test_token_x("""\
+ machine host.domain.com login log password "pass" account acct
+ """, 'password', 'pass')
+
+ def test_token_value_escape(self):
+ self._test_token_x("""\
+ machine host.domain.com login \\"log password pass account acct
+ """, 'login', '"log')
+ self._test_token_x("""\
+ machine host.domain.com login "\\"log" password pass account acct
+ """, 'login', '"log')
+ self._test_token_x("""\
+ machine host.domain.com login log password pass account \\"acct
+ """, 'account', '"acct')
+ self._test_token_x("""\
+ machine host.domain.com login log password pass account "\\"acct"
+ """, 'account', '"acct')
+ self._test_token_x("""\
+ machine host.domain.com login log password \\"pass account acct
+ """, 'password', '"pass')
+ self._test_token_x("""\
+ machine host.domain.com login log password "\\"pass" account acct
+ """, 'password', '"pass')
- def test_password_with_leading_hash(self):
- self._test_passwords("""\
+ def test_token_value_whitespace(self):
+ self._test_token_x("""\
+ machine host.domain.com login "lo g" password pass account acct
+ """, 'login', 'lo g')
+ self._test_token_x("""\
+ machine host.domain.com login log password "pas s" account acct
+ """, 'password', 'pas s')
+ self._test_token_x("""\
+ machine host.domain.com login log password pass account "acc t"
+ """, 'account', 'acc t')
+
+ def test_token_value_non_ascii(self):
+ self._test_token_x("""\
+ machine host.domain.com login \xa1\xa2 password pass account acct
+ """, 'login', '\xa1\xa2')
+ self._test_token_x("""\
+ machine host.domain.com login log password pass account \xa1\xa2
+ """, 'account', '\xa1\xa2')
+ self._test_token_x("""\
+ machine host.domain.com login log password \xa1\xa2 account acct
+ """, 'password', '\xa1\xa2')
+
+ def test_token_value_leading_hash(self):
+ self._test_token_x("""\
+ machine host.domain.com login #log password pass account acct
+ """, 'login', '#log')
+ self._test_token_x("""\
+ machine host.domain.com login log password pass account #acct
+ """, 'account', '#acct')
+ self._test_token_x("""\
machine host.domain.com login log password #pass account acct
- """, '#pass')
+ """, 'password', '#pass')
- def test_password_with_trailing_hash(self):
- self._test_passwords("""\
+ def test_token_value_trailing_hash(self):
+ self._test_token_x("""\
+ machine host.domain.com login log# password pass account acct
+ """, 'login', 'log#')
+ self._test_token_x("""\
+ machine host.domain.com login log password pass account acct#
+ """, 'account', 'acct#')
+ self._test_token_x("""\
machine host.domain.com login log password pass# account acct
- """, 'pass#')
+ """, 'password', 'pass#')
- def test_password_with_internal_hash(self):
- self._test_passwords("""\
+ def test_token_value_internal_hash(self):
+ self._test_token_x("""\
+ machine host.domain.com login lo#g password pass account acct
+ """, 'login', 'lo#g')
+ self._test_token_x("""\
+ machine host.domain.com login log password pass account ac#ct
+ """, 'account', 'ac#ct')
+ self._test_token_x("""\
machine host.domain.com login log password pa#ss account acct
- """, 'pa#ss')
+ """, 'password', 'pa#ss')
def _test_comment(self, nrc, passwd='pass'):
nrc = self.make_nrc(nrc)
- self.assertEqual(nrc.hosts['foo.domain.com'], ('bar', None, passwd))
- self.assertEqual(nrc.hosts['bar.domain.com'], ('foo', None, 'pass'))
+ self.assertEqual(nrc.hosts['foo.domain.com'], ('bar', '', passwd))
+ self.assertEqual(nrc.hosts['bar.domain.com'], ('foo', '', 'pass'))
def test_comment_before_machine_line(self):
self._test_comment("""\
@@ -86,6 +210,42 @@ def test_comment_before_machine_line_hash_only(self):
machine bar.domain.com login foo password pass
""")
+ def test_comment_after_machine_line(self):
+ self._test_comment("""\
+ machine foo.domain.com login bar password pass
+ # comment
+ machine bar.domain.com login foo password pass
+ """)
+ self._test_comment("""\
+ machine foo.domain.com login bar password pass
+ machine bar.domain.com login foo password pass
+ # comment
+ """)
+
+ def test_comment_after_machine_line_no_space(self):
+ self._test_comment("""\
+ machine foo.domain.com login bar password pass
+ #comment
+ machine bar.domain.com login foo password pass
+ """)
+ self._test_comment("""\
+ machine foo.domain.com login bar password pass
+ machine bar.domain.com login foo password pass
+ #comment
+ """)
+
+ def test_comment_after_machine_line_hash_only(self):
+ self._test_comment("""\
+ machine foo.domain.com login bar password pass
+ #
+ machine bar.domain.com login foo password pass
+ """)
+ self._test_comment("""\
+ machine foo.domain.com login bar password pass
+ machine bar.domain.com login foo password pass
+ #
+ """)
+
def test_comment_at_end_of_machine_line(self):
self._test_comment("""\
machine foo.domain.com login bar password pass # comment
@@ -109,57 +269,40 @@ def test_comment_at_end_of_machine_line_pass_has_hash(self):
def test_security(self):
# This test is incomplete since we are normally not run as root and
# therefore can't test the file ownership being wrong.
- with os_helper.temp_cwd(None) as d:
- fn = os.path.join(d, '.netrc')
- with open(fn, 'wt') as f:
- f.write("""\
- machine foo.domain.com login bar password pass
- default login foo password pass
- """)
- with os_helper.EnvironmentVarGuard() as environ:
- environ.set('HOME', d)
- os.chmod(fn, 0o600)
- nrc = netrc.netrc()
- self.assertEqual(nrc.hosts['foo.domain.com'],
- ('bar', None, 'pass'))
- os.chmod(fn, 0o622)
- self.assertRaises(netrc.NetrcParseError, netrc.netrc)
-
- def test_file_not_found_in_home(self):
- with os_helper.temp_cwd(None) as d:
- with os_helper.EnvironmentVarGuard() as environ:
- environ.set('HOME', d)
- self.assertRaises(FileNotFoundError, netrc.netrc)
-
- def test_file_not_found_explicit(self):
- self.assertRaises(FileNotFoundError, netrc.netrc,
- file='unlikely_netrc')
-
- def test_home_not_set(self):
- with os_helper.temp_cwd(None) as fake_home:
- fake_netrc_path = os.path.join(fake_home, '.netrc')
- with open(fake_netrc_path, 'w') as f:
- f.write('machine foo.domain.com login bar password pass')
- os.chmod(fake_netrc_path, 0o600)
-
- orig_expanduser = os.path.expanduser
- called = []
-
- def fake_expanduser(s):
- called.append(s)
- with os_helper.EnvironmentVarGuard() as environ:
- environ.set('HOME', fake_home)
- environ.set('USERPROFILE', fake_home)
- result = orig_expanduser(s)
- return result
-
- with support.swap_attr(os.path, 'expanduser', fake_expanduser):
- nrc = netrc.netrc()
- login, account, password = nrc.authenticators('foo.domain.com')
- self.assertEqual(login, 'bar')
-
- self.assertTrue(called)
+ d = os_helper.TESTFN
+ os.mkdir(d)
+ self.addCleanup(os_helper.rmtree, d)
+ fn = os.path.join(d, '.netrc')
+ with open(fn, 'wt') as f:
+ f.write("""\
+ machine foo.domain.com login bar password pass
+ default login foo password pass
+ """)
+ with os_helper.EnvironmentVarGuard() as environ:
+ environ.set('HOME', d)
+ os.chmod(fn, 0o600)
+ nrc = netrc.netrc()
+ self.assertEqual(nrc.hosts['foo.domain.com'],
+ ('bar', '', 'pass'))
+ os.chmod(fn, 0o622)
+ self.assertRaises(netrc.NetrcParseError, netrc.netrc)
+ with open(fn, 'wt') as f:
+ f.write("""\
+ machine foo.domain.com login anonymous password pass
+ default login foo password pass
+ """)
+ with os_helper.EnvironmentVarGuard() as environ:
+ environ.set('HOME', d)
+ os.chmod(fn, 0o600)
+ nrc = netrc.netrc()
+ self.assertEqual(nrc.hosts['foo.domain.com'],
+ ('anonymous', '', 'pass'))
+ os.chmod(fn, 0o622)
+ self.assertEqual(nrc.hosts['foo.domain.com'],
+ ('anonymous', '', 'pass'))
+def test_main():
+ run_unittest(NetrcTestCase)
if __name__ == "__main__":
- unittest.main()
+ test_main()
diff --git a/Misc/NEWS.d/next/Library/2021-05-24-13-48-34.bpo-28806.PkNw5D.rst b/Misc/NEWS.d/next/Library/2021-05-24-13-48-34.bpo-28806.PkNw5D.rst
new file mode 100644
index 0000000000000..7783845efbcc5
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2021-05-24-13-48-34.bpo-28806.PkNw5D.rst
@@ -0,0 +1 @@
+Improve netrc library. netrc file no longer needs to contain all tokens. And if the login name is anonymous, security check is no longer need.
\ No newline at end of file
More information about the Python-checkins
mailing list