From python-checkins at python.org Sun Sep 12 11:30:22 2010
From: python-checkins at python.org (martin.von.loewis)
Date: Sun, 12 Sep 2010 11:30:22 +0200 (CEST)
Subject: [Pypi-checkins] r864 - trunk/pypi
Message-ID: <20100912093022.9445DF05E@mail.python.org>
Author: martin.von.loewis
Date: Sun Sep 12 11:30:22 2010
New Revision: 864
Modified:
trunk/pypi/webui.py
Log:
Encode OpenID fields with UTF-8 before checking signature.
Modified: trunk/pypi/webui.py
==============================================================================
--- trunk/pypi/webui.py (original)
+++ trunk/pypi/webui.py Sun Sep 12 11:30:22 2010
@@ -2546,7 +2546,7 @@
# Recheck OpenID response
qs = {}
for key, value in self.form.items():
- qs[key] = [value]
+ qs[key] = [value.encode('utf-8')]
session = self.store.get_session_by_handle(self.form['openid.assoc_handle'])
if not session:
raise FormError, "Invalid session"
From python-checkins at python.org Wed Sep 15 16:23:50 2010
From: python-checkins at python.org (martin.von.loewis)
Date: Wed, 15 Sep 2010 16:23:50 +0200 (CEST)
Subject: [Pypi-checkins] r865 - trunk/pypi
Message-ID: <20100915142350.C72BFEE9E0@mail.python.org>
Author: martin.von.loewis
Date: Wed Sep 15 16:23:44 2010
New Revision: 865
Modified:
trunk/pypi/store.py
trunk/pypi/webui.py
Log:
Speedup simple index, by not decoding and encoding.
Modified: trunk/pypi/store.py
==============================================================================
--- trunk/pypi/store.py (original)
+++ trunk/pypi/store.py Wed Sep 15 16:23:44 2010
@@ -586,6 +586,13 @@
safe_execute(cursor, 'select name,stable_version from packages order by name')
return Result(None, cursor.fetchall(), self._Packages)
+ def get_packages_utf8(self):
+ '''Fetch the complete list of package names, UTF-8 encoded
+ '''
+ cursor = self.get_cursor()
+ cursor.execute('select name from packages order by name')
+ return [p[0] for p in cursor.fetchall()]
+
_Journal = FastResultRow('action submitted_date! submitted_by submitted_from')
def get_journal(self, name, version):
''' Retrieve info about the package from the database.
Modified: trunk/pypi/webui.py
==============================================================================
--- trunk/pypi/webui.py (original)
+++ trunk/pypi/webui.py Wed Sep 15 16:23:44 2010
@@ -610,12 +610,12 @@
html = []
html.append("Simple Index")
html.append("\n")
- for name,stable_version in self.store.get_packages():
- qname = urllib.quote(name.encode("utf-8"))
+ for name in self.store.get_packages_utf8():
+ qname = urllib.quote(name)
ename = cgi.escape(name)
html.append("%s \n" % (qname,ename))
html.append("")
- html = ''.join(html).encode('utf-8')
+ html = ''.join(html)
self.handler.send_response(200, 'OK')
self.handler.set_content_type('text/html; charset=utf-8')
self.handler.end_headers()
From python-checkins at python.org Mon Sep 20 10:46:00 2010
From: python-checkins at python.org (martin.von.loewis)
Date: Mon, 20 Sep 2010 10:46:00 +0200 (CEST)
Subject: [Pypi-checkins] r866 - trunk/pypi/tools
Message-ID: <20100920084600.47BC0EE984@mail.python.org>
Author: martin.von.loewis
Date: Mon Sep 20 10:46:00 2010
New Revision: 866
Added:
trunk/pypi/tools/mirrorlib.py (contents, props changed)
Log:
Create library to deal with PyPI mirrors.
Added: trunk/pypi/tools/mirrorlib.py
==============================================================================
--- (empty file)
+++ trunk/pypi/tools/mirrorlib.py Mon Sep 20 10:46:00 2010
@@ -0,0 +1,158 @@
+'''Library to support tools that access PyPI mirrors. The following
+functional areas are covered:
+- mirror selection (find_mirror)
+- mirror verification
+- key rollover
+'''
+
+################## Mirror Selection ##############################
+import socket, time, datetime, errno, select
+
+def _mirror_list(first):
+ '''Generator producing all mirror names'''
+ ord_a = ord('a')
+ last = socket.gethostbyname_ex('last.pypi.python.org')
+ cur_index = ord(first)-ord_a
+ cur = first+'.pypi.python.org'
+ while last[0] != cur:
+ yield cur, socket.gethostbyname(cur)
+ cur_index += 1
+ if cur_index < 26:
+ # a..z
+ cur = chr(ord_a+cur_index)
+ elif cur_index > 701:
+ raise ValueError, 'too many mirrors'
+ else:
+ # aa, ab, ... zz
+ cur = divmod(cur_index, 26)
+ cur = chr(ord_a-1+cur[0])+chr(ord_a+cur[1])
+ cur += '.pypi.python.org'
+ yield last[0], last[2][0]
+
+class _Mirror:
+ # status values:
+ # 0: wants to send
+ # 1: wants to recv
+ # 2: completed, ok
+ # 3: completed, failed
+ def __init__(self, name, ip):
+ self.name = name
+ self.ip = ip
+ self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ self.socket.setblocking(0)
+ self.started = time.time()
+ try:
+ self.socket.connect((name, 80))
+ except socket.error, e:
+ if e.errno != errno.EINPROGRESS:
+ raise
+ # now need to select for writing
+ self.status = 0
+
+ def write(self):
+ self.socket.send('GET /last-modified HTTP/1.0\r\n'
+ 'Host: %s\r\n'
+ '\r\n' % self.name)
+ self.status = 1
+
+ def read(self):
+ data = self.socket.recv(1200)
+ self.response_time = time.time()-self.started
+ # response should be much shorter
+ assert len(data) < 1200
+ self.socket.close()
+ data = data.splitlines()
+ if data[0].split()[1] == '200':
+ # ok
+ data = data[-1]
+ try:
+ self.last_modified = datetime.datetime.strptime(data, "%Y%m%dT%H:%M:%S")
+ self.status = 2 # complete
+ except ValueError:
+ self.status = 3 # failed
+ else:
+ self.status = 3
+
+ def failed(self):
+ self.socket.close()
+ self.status = failed()
+
+ def results(self):
+ return self.name, self.ip, self.response_time, self.last_modified
+
+def _select(mirrors):
+ # perform select call on mirrors dictionary
+ rlist = []
+ wlist = []
+ xlist = []
+ for m in mirrors.values():
+ if m.status == 0:
+ wlist.append(m.socket)
+ xlist.append(m.socket)
+ elif m.status == 1:
+ rlist.append(m.socket)
+ xlist.append(m.socket)
+ rlist, wlist, xlist = select.select(rlist, wlist, xlist, 0)
+ completed = []
+ for s in wlist:
+ mirrors[s].write()
+ for s in rlist:
+ m = mirrors[s]
+ del mirrors[s]
+ m.read()
+ if m.status == 2:
+ completed.append(m)
+ for s in xlist:
+ mirrors[s].failed()
+ del mirrors[s]
+ return completed
+
+def _close(mirrors):
+ for m in mirrors:
+ m.close()
+
+def _newest(mirrors):
+ if not mirrors:
+ raise ValueError, "no mirrors found"
+ mirrors.sort(key=lambda m:m.last_modified)
+ return mirrors[-1].results()
+
+def find_mirror(start_with='a',
+ good_response_time = 1,
+ good_age = 30*60,
+ max_wait = 5):
+ '''find_mirror(start_with, good_response_time, good_age, max_wait) -> name, IP, response_time, last_modified
+ Find a PyPI mirror matching given criteria.
+ start_with indicates the first mirror that should be considered (defaults to 'a').
+ good_response_time is the maximum response time which lets this algorithm look no further;
+ likewise, good_age is the maximum age acceptable to the caller.
+ If this procedure goes on for longer than max_wait (default 5s), return even if
+ not all mirrors have been responding.
+ If no matching mirror can be found, the newest one that did response is returned.'''
+ started = time.time()
+ good_mirrors = []
+ pending_mirrors = {} # socket:mirror
+ good_last_modified = datetime.datetime.utcnow()-datetime.timedelta(seconds=good_age)
+ for host, ip in _mirror_list(start_with):
+ m = _Mirror(host, ip)
+ pending_mirrors[m.socket] = m
+ for m in _select(pending_mirrors):
+ if m.response_time < good_response_time and m.last_modified > good_last_modified:
+ _close(pending_mirrors)
+ return m.results()
+ else:
+ good_mirrors.append(m)
+
+ while pending_mirrors:
+ if time.time() > started+max_wait and good_mirrors:
+ # if we have looked for 5s for a mirror, and we already have one
+ # return the newest one
+ _close(pending)
+ return _newest(good_mirrors)
+ for m in _select(pending_mirrors):
+ if m.response_time < good_response_time and m.last_modified > good_last_modified:
+ _close(pending_mirrors)
+ return m.results()
+ else:
+ good_mirrors.append(m)
+ return _newest(good_mirrors)
From python-checkins at python.org Mon Sep 20 10:57:58 2010
From: python-checkins at python.org (martin.von.loewis)
Date: Mon, 20 Sep 2010 10:57:58 +0200 (CEST)
Subject: [Pypi-checkins] r867 - trunk/pypi/tools
Message-ID: <20100920085758.F353EEE9A6@mail.python.org>
Author: martin.von.loewis
Date: Mon Sep 20 10:57:58 2010
New Revision: 867
Modified:
trunk/pypi/tools/mirrorlib.py
Log:
Rename max_wait to slow_mirrors_wait.
Modified: trunk/pypi/tools/mirrorlib.py
==============================================================================
--- trunk/pypi/tools/mirrorlib.py (original)
+++ trunk/pypi/tools/mirrorlib.py Mon Sep 20 10:57:58 2010
@@ -120,15 +120,16 @@
def find_mirror(start_with='a',
good_response_time = 1,
good_age = 30*60,
- max_wait = 5):
- '''find_mirror(start_with, good_response_time, good_age, max_wait) -> name, IP, response_time, last_modified
+ slow_mirrors_wait = 5):
+ '''find_mirror(start_with, good_response_time, good_age, slow_mirrors_wait) -> name, IP, response_time, last_modified
Find a PyPI mirror matching given criteria.
start_with indicates the first mirror that should be considered (defaults to 'a').
good_response_time is the maximum response time which lets this algorithm look no further;
likewise, good_age is the maximum age acceptable to the caller.
- If this procedure goes on for longer than max_wait (default 5s), return even if
+ If this procedure goes on for longer than slow_mirrors_wait (default 5s), return even if
not all mirrors have been responding.
- If no matching mirror can be found, the newest one that did response is returned.'''
+ If no matching mirror can be found, the newest one that did response is returned.
+ If no mirror can be found at all, ValueError is raised'''
started = time.time()
good_mirrors = []
pending_mirrors = {} # socket:mirror
@@ -144,7 +145,7 @@
good_mirrors.append(m)
while pending_mirrors:
- if time.time() > started+max_wait and good_mirrors:
+ if time.time() > started+slow_mirrors_wait and good_mirrors:
# if we have looked for 5s for a mirror, and we already have one
# return the newest one
_close(pending)
From python-checkins at python.org Mon Sep 20 21:41:22 2010
From: python-checkins at python.org (martin.von.loewis)
Date: Mon, 20 Sep 2010 21:41:22 +0200 (CEST)
Subject: [Pypi-checkins] r868 - trunk/pypi/tools
Message-ID: <20100920194122.86D20EE9A9@mail.python.org>
Author: martin.von.loewis
Date: Mon Sep 20 21:41:22 2010
New Revision: 868
Modified:
trunk/pypi/tools/mirrorlib.py
Log:
Add IPv6 support.
Modified: trunk/pypi/tools/mirrorlib.py
==============================================================================
--- trunk/pypi/tools/mirrorlib.py (original)
+++ trunk/pypi/tools/mirrorlib.py Mon Sep 20 21:41:22 2010
@@ -14,8 +14,11 @@
last = socket.gethostbyname_ex('last.pypi.python.org')
cur_index = ord(first)-ord_a
cur = first+'.pypi.python.org'
- while last[0] != cur:
- yield cur, socket.gethostbyname(cur)
+ while True:
+ for family, _, _, _, sockaddr in socket.getaddrinfo(cur, 0, 0, socket.SOCK_STREAM):
+ yield cur, family, sockaddr
+ if last[0] == cur:
+ break
cur_index += 1
if cur_index < 26:
# a..z
@@ -27,7 +30,6 @@
cur = divmod(cur_index, 26)
cur = chr(ord_a-1+cur[0])+chr(ord_a+cur[1])
cur += '.pypi.python.org'
- yield last[0], last[2][0]
class _Mirror:
# status values:
@@ -35,10 +37,11 @@
# 1: wants to recv
# 2: completed, ok
# 3: completed, failed
- def __init__(self, name, ip):
+ def __init__(self, name, family, ip):
self.name = name
+ self.family = family
self.ip = ip
- self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ self.socket = socket.socket(family, socket.SOCK_STREAM)
self.socket.setblocking(0)
self.started = time.time()
try:
@@ -78,7 +81,7 @@
self.status = failed()
def results(self):
- return self.name, self.ip, self.response_time, self.last_modified
+ return self.name, self.family, self.ip, self.response_time, self.last_modified
def _select(mirrors):
# perform select call on mirrors dictionary
@@ -121,7 +124,9 @@
good_response_time = 1,
good_age = 30*60,
slow_mirrors_wait = 5):
- '''find_mirror(start_with, good_response_time, good_age, slow_mirrors_wait) -> name, IP, response_time, last_modified
+ '''find_mirror(start_with, good_response_time, good_age, slow_mirrors_wait)
+ -> name, family, IP, response_time, last_modified
+
Find a PyPI mirror matching given criteria.
start_with indicates the first mirror that should be considered (defaults to 'a').
good_response_time is the maximum response time which lets this algorithm look no further;
@@ -134,8 +139,8 @@
good_mirrors = []
pending_mirrors = {} # socket:mirror
good_last_modified = datetime.datetime.utcnow()-datetime.timedelta(seconds=good_age)
- for host, ip in _mirror_list(start_with):
- m = _Mirror(host, ip)
+ for host, family, ip in _mirror_list(start_with):
+ m = _Mirror(host, family, ip)
pending_mirrors[m.socket] = m
for m in _select(pending_mirrors):
if m.response_time < good_response_time and m.last_modified > good_last_modified:
From python-checkins at python.org Mon Sep 20 21:46:27 2010
From: python-checkins at python.org (martin.von.loewis)
Date: Mon, 20 Sep 2010 21:46:27 +0200 (CEST)
Subject: [Pypi-checkins] r869 - trunk/pypi
Message-ID: <20100920194627.A1AE2EE996@mail.python.org>
Author: martin.von.loewis
Date: Mon Sep 20 21:46:27 2010
New Revision: 869
Modified:
trunk/pypi/standalone.py
trunk/pypi/webui.py
Log:
Add /daytime URL.
Modified: trunk/pypi/standalone.py
==============================================================================
--- trunk/pypi/standalone.py (original)
+++ trunk/pypi/standalone.py Mon Sep 20 21:46:27 2010
@@ -9,7 +9,7 @@
self.send_header('Content-Type', content_type)
def run(self):
- for scriptname in ('/mirrors', '/simple', '/pypi', '/serversig'):
+ for scriptname in ('/mirrors', '/simple', '/pypi', '/serversig', '/daytime'):
if self.path.startswith(scriptname):
rest = self.path[len(scriptname):]
break
Modified: trunk/pypi/webui.py
==============================================================================
--- trunk/pypi/webui.py (original)
+++ trunk/pypi/webui.py Mon Sep 20 21:46:27 2010
@@ -458,6 +458,8 @@
return self.run_simple_sign()
if script_name == '/mirrors':
return self.mirrors()
+ if script_name == '/daytime':
+ return self.daytime()
# see if the user has provided a username/password
auth = self.env.get('HTTP_CGI_AUTHORIZATION', '').strip()
if auth:
@@ -2749,6 +2751,16 @@
options = {'title': 'PyPI mirrors'}
self.write_template('mirrors.pt', **options)
+ def daytime(self):
+ # Mirrors are supposed to provide /last-modified,
+ # but it doesn't make sense to do so for the master server
+ '''display the current server time.
+ '''
+ self.handler.send_response(200, 'OK')
+ self.handler.set_content_type('text/plain')
+ self.handler.end_headers()
+ self.wfile.write(time.strftime("%Y%m%dT%H:%M:%S\n", time.gmtime(time.time())))
+
def openid(self):
self.write_template('openid.pt', title='OpenID Login')
From python-checkins at python.org Mon Sep 20 21:50:49 2010
From: python-checkins at python.org (martin.von.loewis)
Date: Mon, 20 Sep 2010 21:50:49 +0200 (CEST)
Subject: [Pypi-checkins] r870 - trunk/pypi/tools
Message-ID: <20100920195049.7FB5BEE996@mail.python.org>
Author: martin.von.loewis
Date: Mon Sep 20 21:50:49 2010
New Revision: 870
Modified:
trunk/pypi/tools/mirrorlib.py
Log:
Use /daytime on master server.
Modified: trunk/pypi/tools/mirrorlib.py
==============================================================================
--- trunk/pypi/tools/mirrorlib.py (original)
+++ trunk/pypi/tools/mirrorlib.py Mon Sep 20 21:50:49 2010
@@ -53,9 +53,15 @@
self.status = 0
def write(self):
- self.socket.send('GET /last-modified HTTP/1.0\r\n'
+ url = 'last-modified'
+ if self.name == 'a.pypi.python.org':
+ # the master server doesn't provide last-modified,
+ # as that would be pointless. Instead, /daytime can be
+ # used as an indication of currency and responsiveness.
+ url = 'daytime'
+ self.socket.send('GET /%s HTTP/1.0\r\n'
'Host: %s\r\n'
- '\r\n' % self.name)
+ '\r\n' % (url, self.name))
self.status = 1
def read(self):
From python-checkins at python.org Tue Sep 21 05:01:33 2010
From: python-checkins at python.org (martin.von.loewis)
Date: Tue, 21 Sep 2010 05:01:33 +0200 (CEST)
Subject: [Pypi-checkins] r871 - trunk/pypi
Message-ID: <20100921030133.2EBE2EE9DF@mail.python.org>
Author: martin.von.loewis
Date: Tue Sep 21 05:01:32 2010
New Revision: 871
Modified:
trunk/pypi/webui.py
Log:
Reject empty strings as content or filetype.
Fixes #3072282.
Modified: trunk/pypi/webui.py
==============================================================================
--- trunk/pypi/webui.py (original)
+++ trunk/pypi/webui.py Tue Sep 21 05:01:32 2010
@@ -2257,7 +2257,7 @@
filetype = self.form['filetype']
if filetype == 'sdist':
self.form['pyversion'] = 'source'
- if content is None or filetype is None:
+ if not content or not filetype:
raise FormError, 'Both content and filetype are required'
md5_digest = self.form['md5_digest']
From python-checkins at python.org Sun Sep 26 08:52:37 2010
From: python-checkins at python.org (martin.von.loewis)
Date: Sun, 26 Sep 2010 08:52:37 +0200 (CEST)
Subject: [Pypi-checkins] r872 - trunk/pypi/templates
Message-ID: <20100926065237.9A5B0F335@mail.python.org>
Author: martin.von.loewis
Date: Sun Sep 26 08:52:37 2010
New Revision: 872
Modified:
trunk/pypi/templates/register.pt
Log:
Explain OpenID registration.
Modified: trunk/pypi/templates/register.pt
==============================================================================
--- trunk/pypi/templates/register.pt (original)
+++ trunk/pypi/templates/register.pt Sun Sep 26 08:52:37 2010
@@ -5,6 +5,12 @@
metal:use-macro="standard_template/macros/page">
+
+ This form allows "traditional" registration (using a password).
+ Users who want to register with their OpenID (e.g. Google,
+ myOpenID or Launchpad account) should follow one of the links
+ to the right.
+