+ <title tal:content="string:${data/title} : Python Package Index" /> <meta tal:attributes="content data/keywords" /> <meta tal:attributes="content data/description" /> <link rel="alternate" type="application/rss+xml" title="RSS: 30 latest updates" href="http://www.python.org/pypi?:action=rss"/> From python-checkins at python.org Sun Jul 10 20:33:26 2011 From: python-checkins at python.org (martin.von.loewis) Date: Sun, 10 Jul 2011 20:33:26 +0200 (CEST) Subject: [Pypi-checkins] r927 - in trunk: appengine pypi Message-ID: <3RByqV2pwgzMRJ@mail.python.org> Author: martin.von.loewis Date: Sun Jul 10 20:33:26 2011 New Revision: 927 Modified: trunk/appengine/handlers.py trunk/pypi/gae.py Log: Skip over deleted files. Modified: trunk/appengine/handlers.py ============================================================================== --- trunk/appengine/handlers.py (original) +++ trunk/appengine/handlers.py Sun Jul 10 20:33:26 2011 @@ -47,6 +47,8 @@ f.put() f.contents = self.get_uploads()[0] f.put() + if self.request.get('deleted'): + fetch.delete_file(f) if m.current_upload != path: logging.error('Unexpected upload of '+path) else: Modified: trunk/pypi/gae.py ============================================================================== --- trunk/pypi/gae.py (original) +++ trunk/pypi/gae.py Sun Jul 10 20:33:26 2011 @@ -31,7 +31,14 @@ try: data = open(srcdir+"/"+path).read() except IOError, e: - return + if errno == errno.ENOENT: + # file has been deleted + session += '&deleted=1' + data = '' + else: + # some other problem with file. GAE will request transfer + # again + return boundary = "" while boundary in data: boundary = binascii.hexlify(os.urandom(10)) From python-checkins at python.org Sun Jul 10 21:21:18 2011 From: python-checkins at python.org (martin.von.loewis) Date: Sun, 10 Jul 2011 21:21:18 +0200 (CEST) Subject: [Pypi-checkins] r928 - trunk/pypi Message-ID: <3RBztk1JK9zMSn@mail.python.org> Author: martin.von.loewis Date: Sun Jul 10 21:21:18 2011 New Revision: 928 Modified: trunk/pypi/gae.py Log: Fix various bugs. Modified: trunk/pypi/gae.py ============================================================================== --- trunk/pypi/gae.py (original) +++ trunk/pypi/gae.py Sun Jul 10 21:21:18 2011 @@ -2,7 +2,7 @@ # GAE GETs an action gae_file, giving GAE host and a secret # PyPI GETs /mkupload/secret, learning path and upload session # PyPI POSTs to upload session -import urllib2, httplib, threading, os, binascii, urlparse +import urllib2, httplib, threading, os, binascii, urlparse, errno POST="""\ --%(boundary)s @@ -14,6 +14,10 @@ %(path)s --%(boundary)s +Content-Disposition: form-data; name="%(presence)s" + +1 +--%(boundary)s Content-Disposition: form-data; name="file"; filename="%(path)s" Content-Type: application/octet-stream @@ -30,10 +34,11 @@ host, session = urlparse.urlsplit(url)[1:3] try: data = open(srcdir+"/"+path).read() + presence = "present" except IOError, e: - if errno == errno.ENOENT: + if e.errno == errno.ENOENT: # file has been deleted - session += '&deleted=1' + presence = "deleted" data = '' else: # some other problem with file. GAE will request transfer From python-checkins at python.org Sun Jul 10 21:57:15 2011 From: python-checkins at python.org (martin.von.loewis) Date: Sun, 10 Jul 2011 21:57:15 +0200 (CEST) Subject: [Pypi-checkins] r929 - trunk/pypi Message-ID: <3RC0hC0kWhzMJL@mail.python.org> Author: martin.von.loewis Date: Sun Jul 10 21:57:08 2011 New Revision: 929 Modified: trunk/pypi/rpc.py trunk/pypi/store.py Log: Add changed_packages RPC. Modified: trunk/pypi/rpc.py ============================================================================== --- trunk/pypi/rpc.py (original) +++ trunk/pypi/rpc.py Sun Jul 10 21:57:08 2011 @@ -23,6 +23,7 @@ self.register_function(search) self.register_function(updated_releases) self.register_function(changelog) + self.register_function(changed_packages) self.register_function(post_cheesecake_for_release) self.register_introspection_functions() self.register_multicall_functions() @@ -112,6 +113,9 @@ row['action']) for row in result] +def changed_packages(store, since): + return store.changed_packages(since) + def post_cheesecake_for_release(store, name, version, score_data, password): if password != store.config.cheesecake_password: raise ValuError("Bad password.") Modified: trunk/pypi/store.py ============================================================================== --- trunk/pypi/store.py (original) +++ trunk/pypi/store.py Sun Jul 10 21:57:08 2011 @@ -841,6 +841,14 @@ return Result(None, cursor.fetchall(), self._Changelog) + def changed_packages(self, since): + "Fetch names of packages changed 'since'" + assert isinstance(since, int) + cursor = self.get_cursor() + safe_execute(cursor, 'select distinct(name) from journals where submitted_date > %s', + (time.gmtime(since),)) + return cursor.fetchall() + def changelog_last_hour(self): return self.changelog(int(time.time())-3600) From python-checkins at python.org Sun Jul 10 21:59:46 2011 From: python-checkins at python.org (martin.von.loewis) Date: Sun, 10 Jul 2011 21:59:46 +0200 (CEST) Subject: [Pypi-checkins] r930 - trunk/pypi Message-ID: <3RC0l60LBMzMJL@mail.python.org> Author: martin.von.loewis Date: Sun Jul 10 21:59:45 2011 New Revision: 930 Modified: trunk/pypi/store.py Log: Properly query for timestamps. Modified: trunk/pypi/store.py ============================================================================== --- trunk/pypi/store.py (original) +++ trunk/pypi/store.py Sun Jul 10 21:59:45 2011 @@ -846,7 +846,7 @@ assert isinstance(since, int) cursor = self.get_cursor() safe_execute(cursor, 'select distinct(name) from journals where submitted_date > %s', - (time.gmtime(since),)) + (time.strftime('%Y-%m-%d %H:%M:%S +0000', time.gmtime(since)),)) return cursor.fetchall() def changelog_last_hour(self): From python-checkins at python.org Sun Jul 10 22:01:51 2011 From: python-checkins at python.org (martin.von.loewis) Date: Sun, 10 Jul 2011 22:01:51 +0200 (CEST) Subject: [Pypi-checkins] r931 - trunk/pypi Message-ID: <3RC0nW1cMlzMNC@mail.python.org> Author: martin.von.loewis Date: Sun Jul 10 22:01:51 2011 New Revision: 931 Modified: trunk/pypi/store.py Log: Unwrap single-valued records. Modified: trunk/pypi/store.py ============================================================================== --- trunk/pypi/store.py (original) +++ trunk/pypi/store.py Sun Jul 10 22:01:51 2011 @@ -842,12 +842,12 @@ return Result(None, cursor.fetchall(), self._Changelog) def changed_packages(self, since): - "Fetch names of packages changed 'since'" + "Fetch list of names of packages changed 'since'" assert isinstance(since, int) cursor = self.get_cursor() safe_execute(cursor, 'select distinct(name) from journals where submitted_date > %s', (time.strftime('%Y-%m-%d %H:%M:%S +0000', time.gmtime(since)),)) - return cursor.fetchall() + return [r[0] for r in cursor.fetchall()] def changelog_last_hour(self): return self.changelog(int(time.time())-3600) From python-checkins at python.org Mon Jul 11 17:35:19 2011 From: python-checkins at python.org (martin.von.loewis) Date: Mon, 11 Jul 2011 17:35:19 +0200 (CEST) Subject: [Pypi-checkins] r932 - trunk/appengine Message-ID: <3RCVqW6LctzMWx@mail.python.org> Author: martin.von.loewis Date: Mon Jul 11 17:35:19 2011 New Revision: 932 Modified: trunk/appengine/fetch.py Log: Use changed_packages RPC. Modified: trunk/appengine/fetch.py ============================================================================== --- trunk/appengine/fetch.py (original) +++ trunk/appengine/fetch.py Mon Jul 11 17:35:19 2011 @@ -205,11 +205,11 @@ def check_modifications(m, todo): now = int(time.time()) try: - modified = rpc().changelog(m.last_modified-1) + modified = rpc().changed_packages(m.last_modified-1) except DownloadError, e: - logging.warning('changelog call failed: '+str(e)) + logging.warning('changed_packages call failed: '+str(e)) return - for name, version, date, action in modified: + for name in modified: if ('package', name) in todo: continue todo.append(('package', name)) From python-checkins at python.org Mon Jul 11 23:30:54 2011 From: python-checkins at python.org (martin.von.loewis) Date: Mon, 11 Jul 2011 23:30:54 +0200 (CEST) Subject: [Pypi-checkins] r933 - trunk/appengine Message-ID: <3RCfjp15KlzMYx@mail.python.org> Author: martin.von.loewis Date: Mon Jul 11 23:30:48 2011 New Revision: 933 Modified: trunk/appengine/app.yaml Log: Upgrade to new GAE version. Enable datastore admin. Modified: trunk/appengine/app.yaml ============================================================================== --- trunk/appengine/app.yaml (original) +++ trunk/appengine/app.yaml Mon Jul 11 23:30:48 2011 @@ -3,11 +3,11 @@ runtime: python api_version: 1 -handlers: -- url: /remote_api - script: $PYTHON_LIB/google/appengine/ext/remote_api/handler.py - login: admin +builtins: +- remote_api: on +- datastore_admin: on +handlers: - url: /cron script: mirror.py login: admin From python-checkins at python.org Mon Jul 11 23:31:35 2011 From: python-checkins at python.org (martin.von.loewis) Date: Mon, 11 Jul 2011 23:31:35 +0200 (CEST) Subject: [Pypi-checkins] r934 - trunk/appengine Message-ID: <3RCfkb5shwzMYx@mail.python.org> Author: martin.von.loewis Date: Mon Jul 11 23:31:35 2011 New Revision: 934 Modified: trunk/appengine/stats.py Log: Deal with delayed downloads. Modified: trunk/appengine/stats.py ============================================================================== --- trunk/appengine/stats.py (original) +++ trunk/appengine/stats.py Mon Jul 11 23:31:35 2011 @@ -1,5 +1,5 @@ # stdlib -import datetime, bz2, csv, re, cStringIO, cPickle +import datetime, bz2, csv, re, cStringIO, cPickle, logging from collections import defaultdict # GAE from google.appengine.api.labs import taskqueue @@ -30,6 +30,10 @@ now = datetime.datetime.utcnow() return "%s-%.2d-%.2d" % (now.year, now.month, now.day) +def nextday(day): + day = datetime.date(*map(int,day.split('-')))+datetime.timedelta(days=1) + return "%s-%.2d-%.2d" % (day.year, day.month, day.day) + def mkbz2(entries): downloads = entries.items() downloads.sort() @@ -52,6 +56,17 @@ old = model.Stats.all().filter('day = ', day).fetch(1) if old: old = old[0] + if old.partial is None: + # stats have already been integrated. + # assign downloads to the next day + todo = model.Download.all().filter('day = ', day).fetch(100) + newday = nextday(old.day) + logging.warning('%d extra downloads for %s, reassigning to %s' % + (len(todo), old.day, newday)) + for d in todo: + d.day = newday + d.put() + return if len(old.partial) > 500000: # argh. need to make multiple files to fit into Google blob limits partno = 1 @@ -71,6 +86,7 @@ key = download.project,download.name,agent entries[key] += 1 deletable.append(download) + logging.info("integrated %d stats for %s" % (len(todo), day)) if len(todo) == 100: # Partial results. Save them data = mkbz2(entries) From python-checkins at python.org Mon Jul 11 23:33:24 2011 From: python-checkins at python.org (martin.von.loewis) Date: Mon, 11 Jul 2011 23:33:24 +0200 (CEST) Subject: [Pypi-checkins] r935 - trunk/appengine Message-ID: <3RCfmh6gDmzMZ5@mail.python.org> Author: martin.von.loewis Date: Mon Jul 11 23:33:24 2011 New Revision: 935 Modified: trunk/appengine/stats.py Log: Finalize partial stats with no more downloads. Modified: trunk/appengine/stats.py ============================================================================== --- trunk/appengine/stats.py (original) +++ trunk/appengine/stats.py Mon Jul 11 23:33:24 2011 @@ -114,6 +114,11 @@ # find a day that isn't integrated yet d = model.Download.all().filter('day != ', today()).fetch(1) if not d: + # Finalize pending stats + for p in model.Stats.all().filter('data = ', None): + p.data = p.partial + p.partial = None + p.save() return "Done" integrate_one_day(d[0].day) taskqueue.add(url='/step/integrate') From python-checkins at python.org Fri Jul 15 08:44:44 2011 From: python-checkins at python.org (richard) Date: Fri, 15 Jul 2011 08:44:44 +0200 (CEST) Subject: [Pypi-checkins] r936 - trunk/pypi Message-ID: <3RFksS3hgkzMgp@mail.python.org> Author: richard Date: Fri Jul 15 08:44:44 2011 New Revision: 936 Modified: trunk/pypi/admin.py Log: add test-run capability to nested list nuker Modified: trunk/pypi/admin.py ============================================================================== --- trunk/pypi/admin.py (original) +++ trunk/pypi/admin.py Fri Jul 15 08:44:44 2011 @@ -145,7 +145,7 @@ c.execute('update comments_journal set submitted_by=%s where submitted_by=%s', (new, old)) c.execute('delete from users where name=%s', (old,)) -def nuke_nested_lists(store): +def nuke_nested_lists(store, confirm): c = store.get_cursor() c.execute("""select name, version, summary from releases where summary like '%nested lists%'""") @@ -168,9 +168,13 @@ if 'def print_lol' in f.read(): hits[name] = summary for name in hits: - store.remove_package(name) + if confirm: + store.remove_package(name) print '%s: %s' % (name, hits[name]) - print 'removed %d packages' % len(hits) + if confirm: + print 'removed %d packages' % len(hits) + else: + print 'WOULD HAVE removed %d packages' % len(hits) if __name__ == '__main__': config = config.Config('/data/pypi/config.ini') From python-checkins at python.org Fri Jul 15 08:44:53 2011 From: python-checkins at python.org (richard) Date: Fri, 15 Jul 2011 08:44:53 +0200 (CEST) Subject: [Pypi-checkins] r937 - trunk/pypi Message-ID: <3RFksd1wKhzMgp@mail.python.org> Author: richard Date: Fri Jul 15 08:44:53 2011 New Revision: 937 Modified: trunk/pypi/webui.py Log: bug fix Modified: trunk/pypi/webui.py ============================================================================== --- trunk/pypi/webui.py (original) +++ trunk/pypi/webui.py Fri Jul 15 08:44:53 2011 @@ -588,7 +588,7 @@ if enc in result: prio = result[enc] elif '*' in result: - prio = result[enc] + prio = result['*'] else: prio = 0 if prio > best_prio: From python-checkins at python.org Fri Jul 15 08:45:44 2011 From: python-checkins at python.org (richard) Date: Fri, 15 Jul 2011 08:45:44 +0200 (CEST) Subject: [Pypi-checkins] r938 - trunk/pypi Message-ID: <3RFktc2zB4zMgp@mail.python.org> Author: richard Date: Fri Jul 15 08:45:44 2011 New Revision: 938 Modified: trunk/pypi/admin.py Log: duh Modified: trunk/pypi/admin.py ============================================================================== --- trunk/pypi/admin.py (original) +++ trunk/pypi/admin.py Fri Jul 15 08:45:44 2011 @@ -145,7 +145,7 @@ c.execute('update comments_journal set submitted_by=%s where submitted_by=%s', (new, old)) c.execute('delete from users where name=%s', (old,)) -def nuke_nested_lists(store, confirm): +def nuke_nested_lists(store, confirm=False): c = store.get_cursor() c.execute("""select name, version, summary from releases where summary like '%nested lists%'""")

From python-checkins at python.org Fri Jul 1 02:00:22 2011 From: python-checkins at python.org (richard) Date: Fri, 1 Jul 2011 02:00:22 +0200 (CEST) Subject: [Pypi-checkins] r926 - trunk/pypi/templates Message-ID: <3R4yYL27mvzLqN@mail.python.org> Author: richard Date: Fri Jul 1 02:00:22 2011 New Revision: 926 Modified: trunk/pypi/templates/standard_template.pt Log: reverse ordering for more useful display in browser tabs Modified: trunk/pypi/templates/standard_template.pt ============================================================================== --- trunk/pypi/templates/standard_template.pt (original) +++ trunk/pypi/templates/standard_template.pt Fri Jul 1 02:00:22 2011 @@ -8,7 +8,7 @@ - + <title tal:content="string:${data/title} : Python Package Index" /> <meta tal:attributes="content data/keywords" /> <meta tal:attributes="content data/description" /> <link rel="alternate" type="application/rss+xml" title="RSS: 30 latest updates" href="http://www.python.org/pypi?:action=rss"/> From python-checkins at python.org Sun Jul 10 20:33:26 2011 From: python-checkins at python.org (martin.von.loewis) Date: Sun, 10 Jul 2011 20:33:26 +0200 (CEST) Subject: [Pypi-checkins] r927 - in trunk: appengine pypi Message-ID: <3RByqV2pwgzMRJ@mail.python.org> Author: martin.von.loewis Date: Sun Jul 10 20:33:26 2011 New Revision: 927 Modified: trunk/appengine/handlers.py trunk/pypi/gae.py Log: Skip over deleted files. Modified: trunk/appengine/handlers.py ============================================================================== --- trunk/appengine/handlers.py (original) +++ trunk/appengine/handlers.py Sun Jul 10 20:33:26 2011 @@ -47,6 +47,8 @@ f.put() f.contents = self.get_uploads()[0] f.put() + if self.request.get('deleted'): + fetch.delete_file(f) if m.current_upload != path: logging.error('Unexpected upload of '+path) else: Modified: trunk/pypi/gae.py ============================================================================== --- trunk/pypi/gae.py (original) +++ trunk/pypi/gae.py Sun Jul 10 20:33:26 2011 @@ -31,7 +31,14 @@ try: data = open(srcdir+"/"+path).read() except IOError, e: - return + if errno == errno.ENOENT: + # file has been deleted + session += '&deleted=1' + data = '' + else: + # some other problem with file. GAE will request transfer + # again + return boundary = "" while boundary in data: boundary = binascii.hexlify(os.urandom(10)) From python-checkins at python.org Sun Jul 10 21:21:18 2011 From: python-checkins at python.org (martin.von.loewis) Date: Sun, 10 Jul 2011 21:21:18 +0200 (CEST) Subject: [Pypi-checkins] r928 - trunk/pypi Message-ID: <3RBztk1JK9zMSn@mail.python.org> Author: martin.von.loewis Date: Sun Jul 10 21:21:18 2011 New Revision: 928 Modified: trunk/pypi/gae.py Log: Fix various bugs. Modified: trunk/pypi/gae.py ============================================================================== --- trunk/pypi/gae.py (original) +++ trunk/pypi/gae.py Sun Jul 10 21:21:18 2011 @@ -2,7 +2,7 @@ # GAE GETs an action gae_file, giving GAE host and a secret # PyPI GETs /mkupload/secret, learning path and upload session # PyPI POSTs to upload session -import urllib2, httplib, threading, os, binascii, urlparse +import urllib2, httplib, threading, os, binascii, urlparse, errno POST="""\ --%(boundary)s @@ -14,6 +14,10 @@ %(path)s --%(boundary)s +Content-Disposition: form-data; name="%(presence)s" + +1 +--%(boundary)s Content-Disposition: form-data; name="file"; filename="%(path)s" Content-Type: application/octet-stream @@ -30,10 +34,11 @@ host, session = urlparse.urlsplit(url)[1:3] try: data = open(srcdir+"/"+path).read() + presence = "present" except IOError, e: - if errno == errno.ENOENT: + if e.errno == errno.ENOENT: # file has been deleted - session += '&deleted=1' + presence = "deleted" data = '' else: # some other problem with file. GAE will request transfer From python-checkins at python.org Sun Jul 10 21:57:15 2011 From: python-checkins at python.org (martin.von.loewis) Date: Sun, 10 Jul 2011 21:57:15 +0200 (CEST) Subject: [Pypi-checkins] r929 - trunk/pypi Message-ID: <3RC0hC0kWhzMJL@mail.python.org> Author: martin.von.loewis Date: Sun Jul 10 21:57:08 2011 New Revision: 929 Modified: trunk/pypi/rpc.py trunk/pypi/store.py Log: Add changed_packages RPC. Modified: trunk/pypi/rpc.py ============================================================================== --- trunk/pypi/rpc.py (original) +++ trunk/pypi/rpc.py Sun Jul 10 21:57:08 2011 @@ -23,6 +23,7 @@ self.register_function(search) self.register_function(updated_releases) self.register_function(changelog) + self.register_function(changed_packages) self.register_function(post_cheesecake_for_release) self.register_introspection_functions() self.register_multicall_functions() @@ -112,6 +113,9 @@ row['action']) for row in result] +def changed_packages(store, since): + return store.changed_packages(since) + def post_cheesecake_for_release(store, name, version, score_data, password): if password != store.config.cheesecake_password: raise ValuError("Bad password.") Modified: trunk/pypi/store.py ============================================================================== --- trunk/pypi/store.py (original) +++ trunk/pypi/store.py Sun Jul 10 21:57:08 2011 @@ -841,6 +841,14 @@ return Result(None, cursor.fetchall(), self._Changelog) + def changed_packages(self, since): + "Fetch names of packages changed 'since'" + assert isinstance(since, int) + cursor = self.get_cursor() + safe_execute(cursor, 'select distinct(name) from journals where submitted_date > %s', + (time.gmtime(since),)) + return cursor.fetchall() + def changelog_last_hour(self): return self.changelog(int(time.time())-3600) From python-checkins at python.org Sun Jul 10 21:59:46 2011 From: python-checkins at python.org (martin.von.loewis) Date: Sun, 10 Jul 2011 21:59:46 +0200 (CEST) Subject: [Pypi-checkins] r930 - trunk/pypi Message-ID: <3RC0l60LBMzMJL@mail.python.org> Author: martin.von.loewis Date: Sun Jul 10 21:59:45 2011 New Revision: 930 Modified: trunk/pypi/store.py Log: Properly query for timestamps. Modified: trunk/pypi/store.py ============================================================================== --- trunk/pypi/store.py (original) +++ trunk/pypi/store.py Sun Jul 10 21:59:45 2011 @@ -846,7 +846,7 @@ assert isinstance(since, int) cursor = self.get_cursor() safe_execute(cursor, 'select distinct(name) from journals where submitted_date > %s', - (time.gmtime(since),)) + (time.strftime('%Y-%m-%d %H:%M:%S +0000', time.gmtime(since)),)) return cursor.fetchall() def changelog_last_hour(self): From python-checkins at python.org Sun Jul 10 22:01:51 2011 From: python-checkins at python.org (martin.von.loewis) Date: Sun, 10 Jul 2011 22:01:51 +0200 (CEST) Subject: [Pypi-checkins] r931 - trunk/pypi Message-ID: <3RC0nW1cMlzMNC@mail.python.org> Author: martin.von.loewis Date: Sun Jul 10 22:01:51 2011 New Revision: 931 Modified: trunk/pypi/store.py Log: Unwrap single-valued records. Modified: trunk/pypi/store.py ============================================================================== --- trunk/pypi/store.py (original) +++ trunk/pypi/store.py Sun Jul 10 22:01:51 2011 @@ -842,12 +842,12 @@ return Result(None, cursor.fetchall(), self._Changelog) def changed_packages(self, since): - "Fetch names of packages changed 'since'" + "Fetch list of names of packages changed 'since'" assert isinstance(since, int) cursor = self.get_cursor() safe_execute(cursor, 'select distinct(name) from journals where submitted_date > %s', (time.strftime('%Y-%m-%d %H:%M:%S +0000', time.gmtime(since)),)) - return cursor.fetchall() + return [r[0] for r in cursor.fetchall()] def changelog_last_hour(self): return self.changelog(int(time.time())-3600) From python-checkins at python.org Mon Jul 11 17:35:19 2011 From: python-checkins at python.org (martin.von.loewis) Date: Mon, 11 Jul 2011 17:35:19 +0200 (CEST) Subject: [Pypi-checkins] r932 - trunk/appengine Message-ID: <3RCVqW6LctzMWx@mail.python.org> Author: martin.von.loewis Date: Mon Jul 11 17:35:19 2011 New Revision: 932 Modified: trunk/appengine/fetch.py Log: Use changed_packages RPC. Modified: trunk/appengine/fetch.py ============================================================================== --- trunk/appengine/fetch.py (original) +++ trunk/appengine/fetch.py Mon Jul 11 17:35:19 2011 @@ -205,11 +205,11 @@ def check_modifications(m, todo): now = int(time.time()) try: - modified = rpc().changelog(m.last_modified-1) + modified = rpc().changed_packages(m.last_modified-1) except DownloadError, e: - logging.warning('changelog call failed: '+str(e)) + logging.warning('changed_packages call failed: '+str(e)) return - for name, version, date, action in modified: + for name in modified: if ('package', name) in todo: continue todo.append(('package', name)) From python-checkins at python.org Mon Jul 11 23:30:54 2011 From: python-checkins at python.org (martin.von.loewis) Date: Mon, 11 Jul 2011 23:30:54 +0200 (CEST) Subject: [Pypi-checkins] r933 - trunk/appengine Message-ID: <3RCfjp15KlzMYx@mail.python.org> Author: martin.von.loewis Date: Mon Jul 11 23:30:48 2011 New Revision: 933 Modified: trunk/appengine/app.yaml Log: Upgrade to new GAE version. Enable datastore admin. Modified: trunk/appengine/app.yaml ============================================================================== --- trunk/appengine/app.yaml (original) +++ trunk/appengine/app.yaml Mon Jul 11 23:30:48 2011 @@ -3,11 +3,11 @@ runtime: python api_version: 1 -handlers: -- url: /remote_api - script: $PYTHON_LIB/google/appengine/ext/remote_api/handler.py - login: admin +builtins: +- remote_api: on +- datastore_admin: on +handlers: - url: /cron script: mirror.py login: admin From python-checkins at python.org Mon Jul 11 23:31:35 2011 From: python-checkins at python.org (martin.von.loewis) Date: Mon, 11 Jul 2011 23:31:35 +0200 (CEST) Subject: [Pypi-checkins] r934 - trunk/appengine Message-ID: <3RCfkb5shwzMYx@mail.python.org> Author: martin.von.loewis Date: Mon Jul 11 23:31:35 2011 New Revision: 934 Modified: trunk/appengine/stats.py Log: Deal with delayed downloads. Modified: trunk/appengine/stats.py ============================================================================== --- trunk/appengine/stats.py (original) +++ trunk/appengine/stats.py Mon Jul 11 23:31:35 2011 @@ -1,5 +1,5 @@ # stdlib -import datetime, bz2, csv, re, cStringIO, cPickle +import datetime, bz2, csv, re, cStringIO, cPickle, logging from collections import defaultdict # GAE from google.appengine.api.labs import taskqueue @@ -30,6 +30,10 @@ now = datetime.datetime.utcnow() return "%s-%.2d-%.2d" % (now.year, now.month, now.day) +def nextday(day): + day = datetime.date(*map(int,day.split('-')))+datetime.timedelta(days=1) + return "%s-%.2d-%.2d" % (day.year, day.month, day.day) + def mkbz2(entries): downloads = entries.items() downloads.sort() @@ -52,6 +56,17 @@ old = model.Stats.all().filter('day = ', day).fetch(1) if old: old = old[0] + if old.partial is None: + # stats have already been integrated. + # assign downloads to the next day + todo = model.Download.all().filter('day = ', day).fetch(100) + newday = nextday(old.day) + logging.warning('%d extra downloads for %s, reassigning to %s' % + (len(todo), old.day, newday)) + for d in todo: + d.day = newday + d.put() + return if len(old.partial) > 500000: # argh. need to make multiple files to fit into Google blob limits partno = 1 @@ -71,6 +86,7 @@ key = download.project,download.name,agent entries[key] += 1 deletable.append(download) + logging.info("integrated %d stats for %s" % (len(todo), day)) if len(todo) == 100: # Partial results. Save them data = mkbz2(entries) From python-checkins at python.org Mon Jul 11 23:33:24 2011 From: python-checkins at python.org (martin.von.loewis) Date: Mon, 11 Jul 2011 23:33:24 +0200 (CEST) Subject: [Pypi-checkins] r935 - trunk/appengine Message-ID: <3RCfmh6gDmzMZ5@mail.python.org> Author: martin.von.loewis Date: Mon Jul 11 23:33:24 2011 New Revision: 935 Modified: trunk/appengine/stats.py Log: Finalize partial stats with no more downloads. Modified: trunk/appengine/stats.py ============================================================================== --- trunk/appengine/stats.py (original) +++ trunk/appengine/stats.py Mon Jul 11 23:33:24 2011 @@ -114,6 +114,11 @@ # find a day that isn't integrated yet d = model.Download.all().filter('day != ', today()).fetch(1) if not d: + # Finalize pending stats + for p in model.Stats.all().filter('data = ', None): + p.data = p.partial + p.partial = None + p.save() return "Done" integrate_one_day(d[0].day) taskqueue.add(url='/step/integrate') From python-checkins at python.org Fri Jul 15 08:44:44 2011 From: python-checkins at python.org (richard) Date: Fri, 15 Jul 2011 08:44:44 +0200 (CEST) Subject: [Pypi-checkins] r936 - trunk/pypi Message-ID: <3RFksS3hgkzMgp@mail.python.org> Author: richard Date: Fri Jul 15 08:44:44 2011 New Revision: 936 Modified: trunk/pypi/admin.py Log: add test-run capability to nested list nuker Modified: trunk/pypi/admin.py ============================================================================== --- trunk/pypi/admin.py (original) +++ trunk/pypi/admin.py Fri Jul 15 08:44:44 2011 @@ -145,7 +145,7 @@ c.execute('update comments_journal set submitted_by=%s where submitted_by=%s', (new, old)) c.execute('delete from users where name=%s', (old,)) -def nuke_nested_lists(store): +def nuke_nested_lists(store, confirm): c = store.get_cursor() c.execute("""select name, version, summary from releases where summary like '%nested lists%'""") @@ -168,9 +168,13 @@ if 'def print_lol' in f.read(): hits[name] = summary for name in hits: - store.remove_package(name) + if confirm: + store.remove_package(name) print '%s: %s' % (name, hits[name]) - print 'removed %d packages' % len(hits) + if confirm: + print 'removed %d packages' % len(hits) + else: + print 'WOULD HAVE removed %d packages' % len(hits) if __name__ == '__main__': config = config.Config('/data/pypi/config.ini') From python-checkins at python.org Fri Jul 15 08:44:53 2011 From: python-checkins at python.org (richard) Date: Fri, 15 Jul 2011 08:44:53 +0200 (CEST) Subject: [Pypi-checkins] r937 - trunk/pypi Message-ID: <3RFksd1wKhzMgp@mail.python.org> Author: richard Date: Fri Jul 15 08:44:53 2011 New Revision: 937 Modified: trunk/pypi/webui.py Log: bug fix Modified: trunk/pypi/webui.py ============================================================================== --- trunk/pypi/webui.py (original) +++ trunk/pypi/webui.py Fri Jul 15 08:44:53 2011 @@ -588,7 +588,7 @@ if enc in result: prio = result[enc] elif '*' in result: - prio = result[enc] + prio = result['*'] else: prio = 0 if prio > best_prio: From python-checkins at python.org Fri Jul 15 08:45:44 2011 From: python-checkins at python.org (richard) Date: Fri, 15 Jul 2011 08:45:44 +0200 (CEST) Subject: [Pypi-checkins] r938 - trunk/pypi Message-ID: <3RFktc2zB4zMgp@mail.python.org> Author: richard Date: Fri Jul 15 08:45:44 2011 New Revision: 938 Modified: trunk/pypi/admin.py Log: duh Modified: trunk/pypi/admin.py ============================================================================== --- trunk/pypi/admin.py (original) +++ trunk/pypi/admin.py Fri Jul 15 08:45:44 2011 @@ -145,7 +145,7 @@ c.execute('update comments_journal set submitted_by=%s where submitted_by=%s', (new, old)) c.execute('delete from users where name=%s', (old,)) -def nuke_nested_lists(store, confirm): +def nuke_nested_lists(store, confirm=False): c = store.get_cursor() c.execute("""select name, version, summary from releases where summary like '%nested lists%'""")