From python-checkins at python.org Fri Jul 1 02:00:22 2011
From: python-checkins at python.org (richard)
Date: Fri, 1 Jul 2011 02:00:22 +0200 (CEST)
Subject: [Pypi-checkins] r926 - trunk/pypi/templates
Message-ID: <3R4yYL27mvzLqN@mail.python.org>
Author: richard
Date: Fri Jul 1 02:00:22 2011
New Revision: 926
Modified:
trunk/pypi/templates/standard_template.pt
Log:
reverse ordering for more useful display in browser tabs
Modified: trunk/pypi/templates/standard_template.pt
==============================================================================
--- trunk/pypi/templates/standard_template.pt (original)
+++ trunk/pypi/templates/standard_template.pt Fri Jul 1 02:00:22 2011
@@ -8,7 +8,7 @@
-
+
From python-checkins at python.org Sun Jul 10 20:33:26 2011
From: python-checkins at python.org (martin.von.loewis)
Date: Sun, 10 Jul 2011 20:33:26 +0200 (CEST)
Subject: [Pypi-checkins] r927 - in trunk: appengine pypi
Message-ID: <3RByqV2pwgzMRJ@mail.python.org>
Author: martin.von.loewis
Date: Sun Jul 10 20:33:26 2011
New Revision: 927
Modified:
trunk/appengine/handlers.py
trunk/pypi/gae.py
Log:
Skip over deleted files.
Modified: trunk/appengine/handlers.py
==============================================================================
--- trunk/appengine/handlers.py (original)
+++ trunk/appengine/handlers.py Sun Jul 10 20:33:26 2011
@@ -47,6 +47,8 @@
f.put()
f.contents = self.get_uploads()[0]
f.put()
+ if self.request.get('deleted'):
+ fetch.delete_file(f)
if m.current_upload != path:
logging.error('Unexpected upload of '+path)
else:
Modified: trunk/pypi/gae.py
==============================================================================
--- trunk/pypi/gae.py (original)
+++ trunk/pypi/gae.py Sun Jul 10 20:33:26 2011
@@ -31,7 +31,14 @@
try:
data = open(srcdir+"/"+path).read()
except IOError, e:
- return
+ if errno == errno.ENOENT:
+ # file has been deleted
+ session += '&deleted=1'
+ data = ''
+ else:
+ # some other problem with file. GAE will request transfer
+ # again
+ return
boundary = ""
while boundary in data:
boundary = binascii.hexlify(os.urandom(10))
From python-checkins at python.org Sun Jul 10 21:21:18 2011
From: python-checkins at python.org (martin.von.loewis)
Date: Sun, 10 Jul 2011 21:21:18 +0200 (CEST)
Subject: [Pypi-checkins] r928 - trunk/pypi
Message-ID: <3RBztk1JK9zMSn@mail.python.org>
Author: martin.von.loewis
Date: Sun Jul 10 21:21:18 2011
New Revision: 928
Modified:
trunk/pypi/gae.py
Log:
Fix various bugs.
Modified: trunk/pypi/gae.py
==============================================================================
--- trunk/pypi/gae.py (original)
+++ trunk/pypi/gae.py Sun Jul 10 21:21:18 2011
@@ -2,7 +2,7 @@
# GAE GETs an action gae_file, giving GAE host and a secret
# PyPI GETs /mkupload/secret, learning path and upload session
# PyPI POSTs to upload session
-import urllib2, httplib, threading, os, binascii, urlparse
+import urllib2, httplib, threading, os, binascii, urlparse, errno
POST="""\
--%(boundary)s
@@ -14,6 +14,10 @@
%(path)s
--%(boundary)s
+Content-Disposition: form-data; name="%(presence)s"
+
+1
+--%(boundary)s
Content-Disposition: form-data; name="file"; filename="%(path)s"
Content-Type: application/octet-stream
@@ -30,10 +34,11 @@
host, session = urlparse.urlsplit(url)[1:3]
try:
data = open(srcdir+"/"+path).read()
+ presence = "present"
except IOError, e:
- if errno == errno.ENOENT:
+ if e.errno == errno.ENOENT:
# file has been deleted
- session += '&deleted=1'
+ presence = "deleted"
data = ''
else:
# some other problem with file. GAE will request transfer
From python-checkins at python.org Sun Jul 10 21:57:15 2011
From: python-checkins at python.org (martin.von.loewis)
Date: Sun, 10 Jul 2011 21:57:15 +0200 (CEST)
Subject: [Pypi-checkins] r929 - trunk/pypi
Message-ID: <3RC0hC0kWhzMJL@mail.python.org>
Author: martin.von.loewis
Date: Sun Jul 10 21:57:08 2011
New Revision: 929
Modified:
trunk/pypi/rpc.py
trunk/pypi/store.py
Log:
Add changed_packages RPC.
Modified: trunk/pypi/rpc.py
==============================================================================
--- trunk/pypi/rpc.py (original)
+++ trunk/pypi/rpc.py Sun Jul 10 21:57:08 2011
@@ -23,6 +23,7 @@
self.register_function(search)
self.register_function(updated_releases)
self.register_function(changelog)
+ self.register_function(changed_packages)
self.register_function(post_cheesecake_for_release)
self.register_introspection_functions()
self.register_multicall_functions()
@@ -112,6 +113,9 @@
row['action'])
for row in result]
+def changed_packages(store, since):
+ return store.changed_packages(since)
+
def post_cheesecake_for_release(store, name, version, score_data, password):
if password != store.config.cheesecake_password:
raise ValuError("Bad password.")
Modified: trunk/pypi/store.py
==============================================================================
--- trunk/pypi/store.py (original)
+++ trunk/pypi/store.py Sun Jul 10 21:57:08 2011
@@ -841,6 +841,14 @@
return Result(None, cursor.fetchall(), self._Changelog)
+ def changed_packages(self, since):
+ "Fetch names of packages changed 'since'"
+ assert isinstance(since, int)
+ cursor = self.get_cursor()
+ safe_execute(cursor, 'select distinct(name) from journals where submitted_date > %s',
+ (time.gmtime(since),))
+ return cursor.fetchall()
+
def changelog_last_hour(self):
return self.changelog(int(time.time())-3600)
From python-checkins at python.org Sun Jul 10 21:59:46 2011
From: python-checkins at python.org (martin.von.loewis)
Date: Sun, 10 Jul 2011 21:59:46 +0200 (CEST)
Subject: [Pypi-checkins] r930 - trunk/pypi
Message-ID: <3RC0l60LBMzMJL@mail.python.org>
Author: martin.von.loewis
Date: Sun Jul 10 21:59:45 2011
New Revision: 930
Modified:
trunk/pypi/store.py
Log:
Properly query for timestamps.
Modified: trunk/pypi/store.py
==============================================================================
--- trunk/pypi/store.py (original)
+++ trunk/pypi/store.py Sun Jul 10 21:59:45 2011
@@ -846,7 +846,7 @@
assert isinstance(since, int)
cursor = self.get_cursor()
safe_execute(cursor, 'select distinct(name) from journals where submitted_date > %s',
- (time.gmtime(since),))
+ (time.strftime('%Y-%m-%d %H:%M:%S +0000', time.gmtime(since)),))
return cursor.fetchall()
def changelog_last_hour(self):
From python-checkins at python.org Sun Jul 10 22:01:51 2011
From: python-checkins at python.org (martin.von.loewis)
Date: Sun, 10 Jul 2011 22:01:51 +0200 (CEST)
Subject: [Pypi-checkins] r931 - trunk/pypi
Message-ID: <3RC0nW1cMlzMNC@mail.python.org>
Author: martin.von.loewis
Date: Sun Jul 10 22:01:51 2011
New Revision: 931
Modified:
trunk/pypi/store.py
Log:
Unwrap single-valued records.
Modified: trunk/pypi/store.py
==============================================================================
--- trunk/pypi/store.py (original)
+++ trunk/pypi/store.py Sun Jul 10 22:01:51 2011
@@ -842,12 +842,12 @@
return Result(None, cursor.fetchall(), self._Changelog)
def changed_packages(self, since):
- "Fetch names of packages changed 'since'"
+ "Fetch list of names of packages changed 'since'"
assert isinstance(since, int)
cursor = self.get_cursor()
safe_execute(cursor, 'select distinct(name) from journals where submitted_date > %s',
(time.strftime('%Y-%m-%d %H:%M:%S +0000', time.gmtime(since)),))
- return cursor.fetchall()
+ return [r[0] for r in cursor.fetchall()]
def changelog_last_hour(self):
return self.changelog(int(time.time())-3600)
From python-checkins at python.org Mon Jul 11 17:35:19 2011
From: python-checkins at python.org (martin.von.loewis)
Date: Mon, 11 Jul 2011 17:35:19 +0200 (CEST)
Subject: [Pypi-checkins] r932 - trunk/appengine
Message-ID: <3RCVqW6LctzMWx@mail.python.org>
Author: martin.von.loewis
Date: Mon Jul 11 17:35:19 2011
New Revision: 932
Modified:
trunk/appengine/fetch.py
Log:
Use changed_packages RPC.
Modified: trunk/appengine/fetch.py
==============================================================================
--- trunk/appengine/fetch.py (original)
+++ trunk/appengine/fetch.py Mon Jul 11 17:35:19 2011
@@ -205,11 +205,11 @@
def check_modifications(m, todo):
now = int(time.time())
try:
- modified = rpc().changelog(m.last_modified-1)
+ modified = rpc().changed_packages(m.last_modified-1)
except DownloadError, e:
- logging.warning('changelog call failed: '+str(e))
+ logging.warning('changed_packages call failed: '+str(e))
return
- for name, version, date, action in modified:
+ for name in modified:
if ('package', name) in todo:
continue
todo.append(('package', name))
From python-checkins at python.org Mon Jul 11 23:30:54 2011
From: python-checkins at python.org (martin.von.loewis)
Date: Mon, 11 Jul 2011 23:30:54 +0200 (CEST)
Subject: [Pypi-checkins] r933 - trunk/appengine
Message-ID: <3RCfjp15KlzMYx@mail.python.org>
Author: martin.von.loewis
Date: Mon Jul 11 23:30:48 2011
New Revision: 933
Modified:
trunk/appengine/app.yaml
Log:
Upgrade to new GAE version. Enable datastore admin.
Modified: trunk/appengine/app.yaml
==============================================================================
--- trunk/appengine/app.yaml (original)
+++ trunk/appengine/app.yaml Mon Jul 11 23:30:48 2011
@@ -3,11 +3,11 @@
runtime: python
api_version: 1
-handlers:
-- url: /remote_api
- script: $PYTHON_LIB/google/appengine/ext/remote_api/handler.py
- login: admin
+builtins:
+- remote_api: on
+- datastore_admin: on
+handlers:
- url: /cron
script: mirror.py
login: admin
From python-checkins at python.org Mon Jul 11 23:31:35 2011
From: python-checkins at python.org (martin.von.loewis)
Date: Mon, 11 Jul 2011 23:31:35 +0200 (CEST)
Subject: [Pypi-checkins] r934 - trunk/appengine
Message-ID: <3RCfkb5shwzMYx@mail.python.org>
Author: martin.von.loewis
Date: Mon Jul 11 23:31:35 2011
New Revision: 934
Modified:
trunk/appengine/stats.py
Log:
Deal with delayed downloads.
Modified: trunk/appengine/stats.py
==============================================================================
--- trunk/appengine/stats.py (original)
+++ trunk/appengine/stats.py Mon Jul 11 23:31:35 2011
@@ -1,5 +1,5 @@
# stdlib
-import datetime, bz2, csv, re, cStringIO, cPickle
+import datetime, bz2, csv, re, cStringIO, cPickle, logging
from collections import defaultdict
# GAE
from google.appengine.api.labs import taskqueue
@@ -30,6 +30,10 @@
now = datetime.datetime.utcnow()
return "%s-%.2d-%.2d" % (now.year, now.month, now.day)
+def nextday(day):
+ day = datetime.date(*map(int,day.split('-')))+datetime.timedelta(days=1)
+ return "%s-%.2d-%.2d" % (day.year, day.month, day.day)
+
def mkbz2(entries):
downloads = entries.items()
downloads.sort()
@@ -52,6 +56,17 @@
old = model.Stats.all().filter('day = ', day).fetch(1)
if old:
old = old[0]
+ if old.partial is None:
+ # stats have already been integrated.
+ # assign downloads to the next day
+ todo = model.Download.all().filter('day = ', day).fetch(100)
+ newday = nextday(old.day)
+ logging.warning('%d extra downloads for %s, reassigning to %s' %
+ (len(todo), old.day, newday))
+ for d in todo:
+ d.day = newday
+ d.put()
+ return
if len(old.partial) > 500000:
# argh. need to make multiple files to fit into Google blob limits
partno = 1
@@ -71,6 +86,7 @@
key = download.project,download.name,agent
entries[key] += 1
deletable.append(download)
+ logging.info("integrated %d stats for %s" % (len(todo), day))
if len(todo) == 100:
# Partial results. Save them
data = mkbz2(entries)
From python-checkins at python.org Mon Jul 11 23:33:24 2011
From: python-checkins at python.org (martin.von.loewis)
Date: Mon, 11 Jul 2011 23:33:24 +0200 (CEST)
Subject: [Pypi-checkins] r935 - trunk/appengine
Message-ID: <3RCfmh6gDmzMZ5@mail.python.org>
Author: martin.von.loewis
Date: Mon Jul 11 23:33:24 2011
New Revision: 935
Modified:
trunk/appengine/stats.py
Log:
Finalize partial stats with no more downloads.
Modified: trunk/appengine/stats.py
==============================================================================
--- trunk/appengine/stats.py (original)
+++ trunk/appengine/stats.py Mon Jul 11 23:33:24 2011
@@ -114,6 +114,11 @@
# find a day that isn't integrated yet
d = model.Download.all().filter('day != ', today()).fetch(1)
if not d:
+ # Finalize pending stats
+ for p in model.Stats.all().filter('data = ', None):
+ p.data = p.partial
+ p.partial = None
+ p.save()
return "Done"
integrate_one_day(d[0].day)
taskqueue.add(url='/step/integrate')
From python-checkins at python.org Fri Jul 15 08:44:44 2011
From: python-checkins at python.org (richard)
Date: Fri, 15 Jul 2011 08:44:44 +0200 (CEST)
Subject: [Pypi-checkins] r936 - trunk/pypi
Message-ID: <3RFksS3hgkzMgp@mail.python.org>
Author: richard
Date: Fri Jul 15 08:44:44 2011
New Revision: 936
Modified:
trunk/pypi/admin.py
Log:
add test-run capability to nested list nuker
Modified: trunk/pypi/admin.py
==============================================================================
--- trunk/pypi/admin.py (original)
+++ trunk/pypi/admin.py Fri Jul 15 08:44:44 2011
@@ -145,7 +145,7 @@
c.execute('update comments_journal set submitted_by=%s where submitted_by=%s', (new, old))
c.execute('delete from users where name=%s', (old,))
-def nuke_nested_lists(store):
+def nuke_nested_lists(store, confirm):
c = store.get_cursor()
c.execute("""select name, version, summary from releases
where summary like '%nested lists%'""")
@@ -168,9 +168,13 @@
if 'def print_lol' in f.read():
hits[name] = summary
for name in hits:
- store.remove_package(name)
+ if confirm:
+ store.remove_package(name)
print '%s: %s' % (name, hits[name])
- print 'removed %d packages' % len(hits)
+ if confirm:
+ print 'removed %d packages' % len(hits)
+ else:
+ print 'WOULD HAVE removed %d packages' % len(hits)
if __name__ == '__main__':
config = config.Config('/data/pypi/config.ini')
From python-checkins at python.org Fri Jul 15 08:44:53 2011
From: python-checkins at python.org (richard)
Date: Fri, 15 Jul 2011 08:44:53 +0200 (CEST)
Subject: [Pypi-checkins] r937 - trunk/pypi
Message-ID: <3RFksd1wKhzMgp@mail.python.org>
Author: richard
Date: Fri Jul 15 08:44:53 2011
New Revision: 937
Modified:
trunk/pypi/webui.py
Log:
bug fix
Modified: trunk/pypi/webui.py
==============================================================================
--- trunk/pypi/webui.py (original)
+++ trunk/pypi/webui.py Fri Jul 15 08:44:53 2011
@@ -588,7 +588,7 @@
if enc in result:
prio = result[enc]
elif '*' in result:
- prio = result[enc]
+ prio = result['*']
else:
prio = 0
if prio > best_prio:
From python-checkins at python.org Fri Jul 15 08:45:44 2011
From: python-checkins at python.org (richard)
Date: Fri, 15 Jul 2011 08:45:44 +0200 (CEST)
Subject: [Pypi-checkins] r938 - trunk/pypi
Message-ID: <3RFktc2zB4zMgp@mail.python.org>
Author: richard
Date: Fri Jul 15 08:45:44 2011
New Revision: 938
Modified:
trunk/pypi/admin.py
Log:
duh
Modified: trunk/pypi/admin.py
==============================================================================
--- trunk/pypi/admin.py (original)
+++ trunk/pypi/admin.py Fri Jul 15 08:45:44 2011
@@ -145,7 +145,7 @@
c.execute('update comments_journal set submitted_by=%s where submitted_by=%s', (new, old))
c.execute('delete from users where name=%s', (old,))
-def nuke_nested_lists(store, confirm):
+def nuke_nested_lists(store, confirm=False):
c = store.get_cursor()
c.execute("""select name, version, summary from releases
where summary like '%nested lists%'""")