From python-checkins at python.org Sun Jul 4 20:36:29 2010 From: python-checkins at python.org (martin.von.loewis) Date: Sun, 4 Jul 2010 20:36:29 +0200 (CEST) Subject: [Pypi-checkins] r765 - trunk/pypi/tools Message-ID: <20100704183629.2679BEE988@mail.python.org> Author: martin.von.loewis Date: Sun Jul 4 20:36:28 2010 New Revision: 765 Modified: trunk/pypi/tools/apache_stats.py Log: Integrate change from pep381client: use two-digits months and days. Modified: trunk/pypi/tools/apache_stats.py ============================================================================== --- trunk/pypi/tools/apache_stats.py (original) +++ trunk/pypi/tools/apache_stats.py Sun Jul 4 20:36:28 2010 @@ -112,7 +112,7 @@ def build_local_stats(self, year, month, day, logfile, directory=None): """builds local stats with default values""" - filename = '%d-%d-%d.bz2' % (year, month, day) + filename = '%d-%.2d-%.2d.bz2' % (year, month, day) if directory is not None: filename = os.path.join(directory, filename) From python-checkins at python.org Sun Jul 4 20:45:13 2010 From: python-checkins at python.org (martin.von.loewis) Date: Sun, 4 Jul 2010 20:45:13 +0200 (CEST) Subject: [Pypi-checkins] r766 - trunk/pypi/tools Message-ID: <20100704184513.AF20FEE988@mail.python.org> Author: martin.von.loewis Date: Sun Jul 4 20:45:13 2010 New Revision: 766 Added: trunk/pypi/tools/downloadstats (contents, props changed) Log: Adapt script from pep381client. Added: trunk/pypi/tools/downloadstats ============================================================================== --- (empty file) +++ trunk/pypi/tools/downloadstats Sun Jul 4 20:45:13 2010 @@ -0,0 +1,24 @@ +#!/usr/bin/python +# Generates download stats for all days in the given log files, +# except for the oldest and the newest day. +import sys, os, csv +import apache_reader, apache_stats + +statsdir = '/data/pypi/local-stats/' + +days = set() +records = [] +for fn in sys.argv[1:]: + for record in apache_reader.ApacheLogReader(fn, '/packages'): + days.add((record['year'], record['month'], record['day'])) + records.append(record) + +days = sorted(days)[1:-1] + +class Stats(apache_stats.LocalStats): + def _get_logs(self, logfile, files_url): + return records +stats = Stats() +for year,month,day in days: + stats.build_local_stats(year, month, day, None, statsdir+'days') + From python-checkins at python.org Fri Jul 9 01:24:58 2010 From: python-checkins at python.org (martin.von.loewis) Date: Fri, 9 Jul 2010 01:24:58 +0200 (CEST) Subject: [Pypi-checkins] r767 - trunk/pypi/tools Message-ID: <20100708232458.30296EE991@mail.python.org> Author: martin.von.loewis Date: Fri Jul 9 01:24:57 2010 New Revision: 767 Added: trunk/pypi/tools/integratestats (contents, props changed) Modified: trunk/pypi/tools/apache_stats.py Log: Add integratestats tool. Modified: trunk/pypi/tools/apache_stats.py ============================================================================== --- trunk/pypi/tools/apache_stats.py (original) +++ trunk/pypi/tools/apache_stats.py Fri Jul 9 01:24:57 2010 @@ -37,13 +37,6 @@ current line. if the callable returns True, the line is not included """ - if isinstance(fileobj, str): - fileobj = self._get_file_obj(fileobj, 'w', compression) - file_created = True - else: - file_created = False - - writer = csv.writer(fileobj) downloads = {} for log in self._get_logs(logfile, files_url): if filter is not None: @@ -58,6 +51,16 @@ downloads[key] += count else: downloads[key] = count + self._write_stats(fileobj, downloads) + + def _write_stats(self, fileobj, downloads, compression=None): + if isinstance(fileobj, str): + fileobj = self._get_file_obj(fileobj, 'w', compression) + file_created = True + else: + file_created = False + + writer = csv.writer(fileobj) filenames = downloads.keys() filenames.sort() for key in filenames: @@ -107,9 +110,17 @@ yield {'packagename': line[0], 'filename': line[1], 'useragent': line[2], - 'count': line[3]} + 'count': int(line[3])} #reader.close() + def read_stats_dict(self, stats_file): + res = {} + for r in self.read_stats(stats_file): + key = (r['packagename'], r['filename'], r['useragent']) + value = r['count'] + res[key] = value + return res + def build_local_stats(self, year, month, day, logfile, directory=None): """builds local stats with default values""" filename = '%d-%.2d-%.2d.bz2' % (year, month, day) @@ -119,6 +130,27 @@ self.build_daily_stats(year, month, day, logfile, filename, compression='bz2') + def integrate_stats(self, targetdir, year, month, day, fd): + new = self.read_stats_dict(fd) + oldpath = "%s/days/%s-%.2s-%.2s.bz2" % (targetdir, year, month, day) + if os.path.exists(oldpath): + old = self.read_stats_dict(oldpath) + for k, v in new.items(): + old[k] = old.get(k, 0) + v + else: + old = new + self._write_stats(oldpath, old, 'bz2') + monthpath = "%s/months/%s-%.2s.bz2" % (targetdir, year, month) + if os.path.exists(monthpath): + old = self.read_stats_dict(monthpath) + for k, v in new.items(): + old[k] = old.get(k, 0) + v + else: + old = new + self._write_stats(monthpath, old, 'bz2') + return new + + class ApacheLocalStats(LocalStats): """concrete class that uses the ApacheLogReader""" def _get_logs(self, logfile, files_url): Added: trunk/pypi/tools/integratestats ============================================================================== --- (empty file) +++ trunk/pypi/tools/integratestats Fri Jul 9 01:24:57 2010 @@ -0,0 +1,65 @@ +#!/usr/bin/python +import sys, os, socket, psycopg2, urllib, re, bz2, cStringIO, ConfigParser +sys.path.append(os.path.dirname(__file__)+"/..") +import apache_stats + +statsdir = '/data/pypi/stats/' + +def integrate(config, data): + # Setup database connection + c = ConfigParser.ConfigParser({'user':'', 'password':''}) + c.read(config) + dbname = c.get('database', 'name') + dbuser = c.get('database', 'user') + dbpass = c.get('database', 'password') + dbconn = psycopg2.connect(database=dbname, user=dbuser, password=dbpass) + cursor = dbconn.cursor() + for (package, filename, browser), count in data.items(): + cursor.execute('update release_files set downloads=downloads+%s where filename=%s', + (count, filename)) + dbconn.commit() + dbconn.close() + +def integrate_remote(config, host, dbupdate=True): + index = urllib.urlopen('http://%s.pypi.python.org/local-stats/days' % host).read() + files = set(re.findall('href=.(20..-..-..).bz2', index)) + try: + integrated = open('/data/pypi/stats/integrated/'+host).readlines() + integrated = set([x.strip() for x in integrated]) + except IOError: + integrated = set() + missing = files-integrated + stats = apache_stats.LocalStats() + for m in missing: + data = urllib.urlopen('http://%s.pypi.python.org/local-stats/days/%s.bz2' % (host, m)).read() + data = bz2.decompress(data) + data = cStringIO.StringIO(data) + year, month, day = m.split('-') + # index integration + delta = stats.integrate_stats(statsdir, year, month, day, data) + if dbupdate: + # database integration + integrate(config, delta) + integrated.add(m) + open('/data/pypi/stats/integrated/'+host, 'w').write('\n'.join(sorted(integrated))) + +def main(): + lasts = socket.gethostbyname_ex('last.pypi.python.org') + # look for name X.pypi.python.org + lasts = [lasts[0]] + lasts[1] + for last in lasts: + if last[1:] == '.pypi.python.org': + break + else: + raise ValueError, "Could not properly resolve last mirror name" + last = last.split('.')[0] + integrate_remote(None, 'a', False) + host = 'b' + while True: + integrate_remote(sys.argv[1], host) + host = chr(ord(host)+1) + if host == last: + break + +main() + From python-checkins at python.org Fri Jul 9 01:28:37 2010 From: python-checkins at python.org (martin.von.loewis) Date: Fri, 9 Jul 2010 01:28:37 +0200 (CEST) Subject: [Pypi-checkins] r768 - trunk/pypi/tools Message-ID: <20100708232837.1185EEE995@mail.python.org> Author: martin.von.loewis Date: Fri Jul 9 01:28:36 2010 New Revision: 768 Modified: trunk/pypi/tools/integratestats Log: Fix stats path. Modified: trunk/pypi/tools/integratestats ============================================================================== --- trunk/pypi/tools/integratestats (original) +++ trunk/pypi/tools/integratestats Fri Jul 9 01:28:36 2010 @@ -3,7 +3,7 @@ sys.path.append(os.path.dirname(__file__)+"/..") import apache_stats -statsdir = '/data/pypi/stats/' +statsdir = '/data/www/pypi/stats/' def integrate(config, data): # Setup database connection From python-checkins at python.org Fri Jul 9 01:34:19 2010 From: python-checkins at python.org (martin.von.loewis) Date: Fri, 9 Jul 2010 01:34:19 +0200 (CEST) Subject: [Pypi-checkins] r769 - trunk/pypi/tools Message-ID: <20100708233419.3217DEEAF6@mail.python.org> Author: martin.von.loewis Date: Fri Jul 9 01:34:19 2010 New Revision: 769 Modified: trunk/pypi/tools/integratestats Log: Fix integrated path. Modified: trunk/pypi/tools/integratestats ============================================================================== --- trunk/pypi/tools/integratestats (original) +++ trunk/pypi/tools/integratestats Fri Jul 9 01:34:19 2010 @@ -3,7 +3,7 @@ sys.path.append(os.path.dirname(__file__)+"/..") import apache_stats -statsdir = '/data/www/pypi/stats/' +statsdir = '/data/www/pypi/stats' def integrate(config, data): # Setup database connection @@ -24,7 +24,7 @@ index = urllib.urlopen('http://%s.pypi.python.org/local-stats/days' % host).read() files = set(re.findall('href=.(20..-..-..).bz2', index)) try: - integrated = open('/data/pypi/stats/integrated/'+host).readlines() + integrated = open(statsdir+'/integrated/'+host).readlines() integrated = set([x.strip() for x in integrated]) except IOError: integrated = set() @@ -41,7 +41,7 @@ # database integration integrate(config, delta) integrated.add(m) - open('/data/pypi/stats/integrated/'+host, 'w').write('\n'.join(sorted(integrated))) + open(statsdir+'/integrated/'+host, 'w').write('\n'.join(sorted(integrated))) def main(): lasts = socket.gethostbyname_ex('last.pypi.python.org') From python-checkins at python.org Fri Jul 9 09:09:00 2010 From: python-checkins at python.org (martin.von.loewis) Date: Fri, 9 Jul 2010 09:09:00 +0200 (CEST) Subject: [Pypi-checkins] r770 - trunk/pypi/tools Message-ID: <20100709070900.477DCEE990@mail.python.org> Author: martin.von.loewis Date: Fri Jul 9 09:09:00 2010 New Revision: 770 Modified: trunk/pypi/tools/apache_stats.py Log: Restore missing parameter. Modified: trunk/pypi/tools/apache_stats.py ============================================================================== --- trunk/pypi/tools/apache_stats.py (original) +++ trunk/pypi/tools/apache_stats.py Fri Jul 9 09:09:00 2010 @@ -51,7 +51,7 @@ downloads[key] += count else: downloads[key] = count - self._write_stats(fileobj, downloads) + self._write_stats(fileobj, downloads, compression=compression) def _write_stats(self, fileobj, downloads, compression=None): if isinstance(fileobj, str): From python-checkins at python.org Sat Jul 10 12:51:13 2010 From: python-checkins at python.org (martin.von.loewis) Date: Sat, 10 Jul 2010 12:51:13 +0200 (CEST) Subject: [Pypi-checkins] r771 - trunk/pypi Message-ID: <20100710105113.60ECBEEA5A@mail.python.org> Author: martin.von.loewis Date: Sat Jul 10 12:51:13 2010 New Revision: 771 Added: trunk/pypi/mirrors.txt (contents, props changed) Log: Document mirrors in this text file for now. Added: trunk/pypi/mirrors.txt ============================================================================== --- (empty file) +++ trunk/pypi/mirrors.txt Sat Jul 10 12:51:13 2010 @@ -0,0 +1,4 @@ +b: 141.89.226.2, martin at v.loewis.de +c: h1606280.stratoserver.net, pypi at zopyx.com +d: pypi.websushi.org, jezdez+pypi at enn.io + From python-checkins at python.org Wed Jul 21 00:09:03 2010 From: python-checkins at python.org (martin.von.loewis) Date: Wed, 21 Jul 2010 00:09:03 +0200 (CEST) Subject: [Pypi-checkins] r773 - trunk/pypi Message-ID: <20100720220903.66128EE995@mail.python.org> Author: martin.von.loewis Date: Wed Jul 21 00:09:03 2010 New Revision: 773 Modified: trunk/pypi/store.py trunk/pypi/webui.py Log: Cleanup imports. Modified: trunk/pypi/store.py ============================================================================== --- trunk/pypi/store.py (original) +++ trunk/pypi/store.py Wed Jul 21 00:09:03 2010 @@ -1,6 +1,6 @@ ''' Implements a store of disutils PKG-INFO entries, keyed off name, version. ''' -import sys, os, re, psycopg2, time, sha, random, types, math, stat, errno +import sys, os, re, psycopg2, time, hashlib, random, types, math, stat, errno import logging, cStringIO, string, datetime, calendar, binascii, urllib2, cgi from xml.parsers import expat from distutils.version import LooseVersion @@ -1295,7 +1295,7 @@ if self.has_user(name): if password: # update existing user, including password - password = sha.sha(password).hexdigest() + password = hashlib.sha1(password).hexdigest() safe_execute(cursor, 'update users set password=%s, email=%s where name=%s', (password, email, name)) @@ -1314,7 +1314,7 @@ if cursor.fetchone()[0] > 0: raise ValueError, "Email address already belongs to a different user" - password = sha.sha(password).hexdigest() + password = hashlib.sha1(password).hexdigest() # new user safe_execute(cursor, @@ -1943,7 +1943,7 @@ self.userip = userip def setpasswd(self, username, password): - password = sha.sha(password).hexdigest() + password = hashlib.sha1(password).hexdigest() self.get_cursor().execute(''' update users set password=%s where name=%s ''', (password, username)) Modified: trunk/pypi/webui.py ============================================================================== --- trunk/pypi/webui.py (original) +++ trunk/pypi/webui.py Wed Jul 21 00:09:03 2010 @@ -1,7 +1,7 @@ # system imports -import sys, os, urllib, cStringIO, traceback, cgi, binascii, getopt, md5 -import time, random, smtplib, base64, sha, email, types, stat, urlparse -import re, zipfile, logging, pprint, sets, shutil, Cookie, subprocess +import sys, os, urllib, cStringIO, traceback, cgi, binascii +import time, random, smtplib, base64, email, types, urlparse +import re, zipfile, logging, shutil, Cookie, subprocess, hashlib from zope.pagetemplate.pagetemplatefile import PageTemplateFile from distutils.util import rfc822_escape from distutils2.metadata import DistributionMetadata @@ -14,7 +14,7 @@ from xml.etree import cElementTree # local imports -import store, config, trove, versionpredicate, verify_filetype, rpc +import store, config, versionpredicate, verify_filetype, rpc import MailingLogger, openid2rp from mini_pkg_resources import safe_name @@ -433,7 +433,7 @@ # Invalid base64, or not exactly one colon un = pw = '' if self.store.has_user(un): - pw = sha.sha(pw).hexdigest() + pw = hashlib.sha1(pw).hexdigest() user = self.store.get_user(un) if pw != user['password']: raise Unauthorised, 'Incorrect password' @@ -989,7 +989,7 @@ write_element(pelem, person, 'foaf:name') email = info[person+'_email'] if email and email != 'UNKNOWN': - obj = sha.new(email) + obj = hashlib.sha1(email) email = binascii.b2a_hex(obj.digest()) elem = SE(pelem, 'foaf:mbox_sha1sum') elem.text = email @@ -1326,7 +1326,7 @@ self.write_template('index.pt', title="Index of Packages", matches=l) - STOPWORDS = sets.Set([ + STOPWORDS = set([ "a", "and", "are", "as", "at", "be", "but", "by", "for", "if", "in", "into", "is", "it", "no", "not", "of", "on", "or", "such", @@ -2218,7 +2218,7 @@ raise FormError, "signature is not ASCII-armored" # digest content - m = md5.new() + m = hashlib.md5() m.update(content) calc_digest = m.hexdigest() @@ -2228,7 +2228,7 @@ self.fail(heading='MD5 digest mismatch', message='''The MD5 digest supplied does not match a digest calculated from the uploaded file (m = - md5.new(); m.update(content); digest = + hashlib.md5(); m.update(content); digest = m.hexdigest())''') return From python-checkins at python.org Wed Jul 21 14:48:11 2010 From: python-checkins at python.org (martin.von.loewis) Date: Wed, 21 Jul 2010 14:48:11 +0200 (CEST) Subject: [Pypi-checkins] r774 - trunk/pypi Message-ID: <20100721124811.C9016E9E6@mail.python.org> Author: martin.von.loewis Date: Wed Jul 21 14:48:11 2010 New Revision: 774 Modified: trunk/pypi/pkgbase_schema.sql Log: Fix schema so that postgres can import it. Modified: trunk/pypi/pkgbase_schema.sql ============================================================================== --- trunk/pypi/pkgbase_schema.sql (original) +++ trunk/pypi/pkgbase_schema.sql Wed Jul 21 14:48:11 2010 @@ -1,3 +1,4 @@ +begin; -- Table structure for table: users CREATE TABLE users ( name TEXT PRIMARY KEY, @@ -35,7 +36,7 @@ created TIMESTAMP, nonce TEXT ); -CREATE INDEX openid_nonces_nonce ON openid_nonces(created); +CREATE INDEX openid_nonces_created ON openid_nonces(created); CREATE INDEX openid_nonces_nonce ON openid_nonces(nonce); CREATE TABLE cookies ( @@ -51,7 +52,6 @@ key TEXT ); CREATE INDEX sshkeys_name ON sshkeys(name); -CREATE INDEX rego_otk_otk_idx ON rego_otk(otk); -- Table structure for table: rego_otk CREATE TABLE rego_otk ( @@ -59,7 +59,7 @@ otk TEXT, date TIMESTAMP ); CREATE INDEX rego_otk_name_idx ON rego_otk(name); - +CREATE INDEX rego_otk_otk_idx ON rego_otk(otk); -- Table structure for table: journals CREATE TABLE journals ( @@ -102,25 +102,6 @@ PRIMARY KEY (main_index_id, name) ); --- Table structure for table: cheesecake_main_indices -CREATE TABLE cheesecake_main_indices ( - id SERIAL, - absolute INTEGER NOT NULL, - relative INTEGER NOT NULL, - PRIMARY KEY (id) -); - - --- Table structure for table: cheesecake_subindices -CREATE TABLE cheesecake_subindices ( - main_index_id INTEGER REFERENCES cheesecake_main_indices, - name TEXT, - value INTEGER NOT NULL, - details TEXT NOT NULL, - PRIMARY KEY (main_index_id, name) -); - - -- Table structure for table: releases CREATE TABLE releases ( name TEXT REFERENCES packages ON UPDATE CASCADE, @@ -203,7 +184,7 @@ ); CREATE INDEX rel_req_name_idx ON release_requires(name); CREATE INDEX rel_req_version_id_idx ON release_requires(version); -CREATE INDEX rel_req_name_version_idx ON release_obsoletes (name,version); +CREATE INDEX rel_req_name_version_idx ON release_requires(name,version); -- Table structure for table: release_obsoletes CREATE TABLE release_obsoletes ( @@ -349,14 +330,6 @@ ); -- ratings -CREATE TABLE comments( - id SERIAL PRIMARY KEY, - rating INTEGER REFERENCES ratings(id) ON DELETE CASCADE, - user_name TEXT REFERENCES users ON DELETE CASCADE, - date TIMESTAMP, - message TEXT, - in_reply_to INTEGER REFERENCES comments ON DELETE CASCADE -); CREATE TABLE ratings( id SERIAL UNIQUE, name TEXT, @@ -368,6 +341,14 @@ FOREIGN KEY (name, version) REFERENCES releases ON UPDATE CASCADE ON DELETE CASCADE ); CREATE INDEX rating_name_version ON ratings(name, version); +CREATE TABLE comments( + id SERIAL PRIMARY KEY, + rating INTEGER REFERENCES ratings(id) ON DELETE CASCADE, + user_name TEXT REFERENCES users ON DELETE CASCADE, + date TIMESTAMP, + message TEXT, + in_reply_to INTEGER REFERENCES comments ON DELETE CASCADE +); CREATE TABLE comments_journal( name text, version text, @@ -378,3 +359,4 @@ FOREIGN KEY (name, version) REFERENCES releases (name, version) ON DELETE CASCADE ); +commit; \ No newline at end of file From python-checkins at python.org Thu Jul 22 18:55:41 2010 From: python-checkins at python.org (martin.von.loewis) Date: Thu, 22 Jul 2010 18:55:41 +0200 (CEST) Subject: [Pypi-checkins] r775 - trunk/pypi Message-ID: <20100722165541.29D4EEE98E@mail.python.org> Author: martin.von.loewis Date: Thu Jul 22 18:55:40 2010 New Revision: 775 Added: trunk/pypi/config.ini.template - copied unchanged from r774, trunk/pypi/config.ini Removed: trunk/pypi/config.ini Log: Make config.ini a template file only. Deleted: /trunk/pypi/config.ini ============================================================================== --- /trunk/pypi/config.ini Thu Jul 22 18:55:40 2010 +++ (empty file) @@ -1,32 +0,0 @@ -[database] -name = packages -user = pypi -files_dir = /MacDev/svn.python.org/pypi-pep345/files -docs_dir = /MacDev/svn.python.org/pypi-pep345/docs - -[webui] -mailhost = mail.commonground.com.au -adminemail = richard at commonground.com.au -replyto = richard at commonground.com.au -url = http://localhost/cgi-bin/pypi.cgi -pydotorg = http://www.python.org/ - -simple_script = /simple -files_url = http://localhost/pypi_files -rss_file = /tmp/pypi_rss.xml -debug_mode = yes -cheesecake_password = secret -privkey = privkey -simple_sign_script = /serversig - -[logging] -file = -mailhost = -fromaddr = -toaddrs = - -[mirrors] -folder = mirrors -local-stats = local-stats -global-stats = global-stats - From python-checkins at python.org Thu Jul 22 18:57:40 2010 From: python-checkins at python.org (martin.von.loewis) Date: Thu, 22 Jul 2010 18:57:40 +0200 (CEST) Subject: [Pypi-checkins] r776 - trunk/pypi Message-ID: <20100722165740.66A25EE98E@mail.python.org> Author: martin.von.loewis Date: Thu Jul 22 18:57:40 2010 New Revision: 776 Modified: trunk/pypi/README Log: Mention config.ini template. Modified: trunk/pypi/README ============================================================================== --- trunk/pypi/README (original) +++ trunk/pypi/README Thu Jul 22 18:57:40 2010 @@ -20,8 +20,9 @@ Make sure you read http://wiki.python.org/moin/CheeseShopDev#DevelopmentEnvironmentHints and you have a working PostgreSQL DB. -Make sure your config.ini is up-to-date. Change CONFIG_FILE at the begining -of pypi.wsgi, so it looks like this:: +Make sure your config.ini is up-to-date, initially copying from +config.ini.template. Change CONFIG_FILE at the begining of pypi.wsgi, +so it looks like this:: CONFIG_FILE = 'config.ini' From python-checkins at python.org Thu Jul 22 23:47:37 2010 From: python-checkins at python.org (martin.von.loewis) Date: Thu, 22 Jul 2010 23:47:37 +0200 (CEST) Subject: [Pypi-checkins] r777 - in trunk/pypi: . tools Message-ID: <20100722214737.C7476F591@mail.python.org> Author: martin.von.loewis Date: Thu Jul 22 23:47:37 2010 New Revision: 777 Added: trunk/pypi/tools/demodata trunk/pypi/tools/mksqlite (contents, props changed) Modified: trunk/pypi/README trunk/pypi/admin.py trunk/pypi/config.ini.template trunk/pypi/config.py trunk/pypi/pkgbase_schema.sql trunk/pypi/store.py trunk/pypi/webui.py Log: Add support for running PyPI on sqlite3. Modified: trunk/pypi/README ============================================================================== --- trunk/pypi/README (original) +++ trunk/pypi/README Thu Jul 22 23:47:37 2010 @@ -10,7 +10,7 @@ - zope.tal - zope.tales - zope.i18nmessageid -- psycopg2 +- psycopg2 (for testing, sqlite3 might be sufficient) - docutils - distutils2 @@ -32,7 +32,7 @@ $ virtualenv --no-site-packages --distribute . $ bin/easy_install cElementTree zope.interface zope.pagetemplate $ bin/easy_install zope.tal zope.tales zope.i18nmessageid psycopg2 - $ bin/easy_install M2Crypto BeautifulSoup docutils + $ bin/easy_install docutils Then you can launch the server using the pypi.wsgi script:: @@ -41,3 +41,12 @@ PyPI will be available in your browser at http://localhost:8000 +Database Setup +-------------- + +To fill a database, run pkgbase_schema.sql on an empty Postgres database. +Then run tools/demodata to populate the database with dummy data. + +For testing purposes, run tools/mksqlite to create packages.db. Set +[database]driver to sqlite3, and [database]name to packages.db, then +run tools/demodata to populate the database. \ No newline at end of file Modified: trunk/pypi/admin.py ============================================================================== --- trunk/pypi/admin.py (original) +++ trunk/pypi/admin.py Thu Jul 22 23:47:37 2010 @@ -41,27 +41,30 @@ raise ValueError, "user is not currently owner" store.delete_role(owner, 'Owner', package) -def add_classifier(store, classifier): +def add_classifier(st, classifier): ''' Add a classifier to the trove_classifiers list ''' - cursor = store.get_cursor() + cursor = st.get_cursor() cursor.execute("select max(id) from trove_classifiers") - id = int(cursor.fetchone()[0]) + 1 + id = cursor.fetchone()[0] + if id: + id = int(id) + 1 + else: + id = 1 fields = [f.strip() for f in classifier.split('::')] for f in fields: assert ':' not in f levels = [] for l in range(2, len(fields)): c2 = ' :: '.join(fields[:l]) - cursor.execute('select id from trove_classifiers where classifier=%s', (c2,)) + store.safe_execute(cursor, 'select id from trove_classifiers where classifier=%s', (c2,)) l = cursor.fetchone() if not l: raise ValueError, c2 + " is not a known classifier" levels.append(l[0]) levels += [id] + [0]*(3-len(levels)) - cursor.execute('insert into trove_classifiers (id, classifier, l2, l3, l4, l5) ' + store.safe_execute(cursor, 'insert into trove_classifiers (id, classifier, l2, l3, l4, l5) ' 'values (%s,%s,%s,%s,%s,%s)', [id, classifier]+levels) - print 'done' def rename_package(store, old, new): ''' Rename a package. ''' @@ -135,6 +138,7 @@ remove_package(*args) elif command == 'addclass': add_classifier(*args) + print 'done' elif command == 'addowner': add_owner(*args) elif command == 'delowner': Modified: trunk/pypi/config.ini.template ============================================================================== --- trunk/pypi/config.ini.template (original) +++ trunk/pypi/config.ini.template Thu Jul 22 23:47:37 2010 @@ -1,4 +1,5 @@ [database] +driver = postgresql2 name = packages user = pypi files_dir = /MacDev/svn.python.org/pypi-pep345/files @@ -8,7 +9,7 @@ mailhost = mail.commonground.com.au adminemail = richard at commonground.com.au replyto = richard at commonground.com.au -url = http://localhost/cgi-bin/pypi.cgi +url = http://localhost:8000/pypi pydotorg = http://www.python.org/ simple_script = /simple Modified: trunk/pypi/config.py ============================================================================== --- trunk/pypi/config.py (original) +++ trunk/pypi/config.py Thu Jul 22 23:47:37 2010 @@ -8,6 +8,10 @@ c.read(configfile) self.database_name = c.get('database', 'name') self.database_user = c.get('database', 'user') + if c.has_option('database', 'driver'): + self.database_driver = c.get('database', 'driver') + else: + self.database_driver = 'psycopg2' if c.has_option('database', 'password'): self.database_pw = c.get('database', 'password') else: Modified: trunk/pypi/pkgbase_schema.sql ============================================================================== --- trunk/pypi/pkgbase_schema.sql (original) +++ trunk/pypi/pkgbase_schema.sql Thu Jul 22 23:47:37 2010 @@ -72,9 +72,11 @@ ); CREATE INDEX journals_name_idx ON journals(name); CREATE INDEX journals_version_idx ON journals(version); +-- nosqlite CREATE INDEX journals_latest_releases ON journals(submitted_date, name, version) WHERE version IS NOT NULL AND action='new release'; +-- nosqlite-end CREATE INDEX journals_changelog ON journals(submitted_date, name, version, action); @@ -145,10 +147,12 @@ -- trove ids sequence +-- nosqlite CREATE TABLE dual (dummy INTEGER); INSERT INTO dual VALUES (1); CREATE SEQUENCE trove_ids; SELECT setval('trove_ids', 1000) FROM dual; +-- nosqlite-end -- Table structure for table: release_classifiers @@ -312,9 +316,9 @@ name TEXT PRIMARY KEY, value TIMESTAMP ); -INSERT INTO timestamps(name, value) VALUES('http','1970-01-01'); -INSERT INTO timestamps(name, value) VALUES('ftp','1970-01-01'); -INSERT INTO timestamps(name, value) VALUES('browse_tally','1970-01-01'); +INSERT INTO timestamps(name, value) VALUES('http','1970-01-01 00:00:00'); +INSERT INTO timestamps(name, value) VALUES('ftp','1970-01-01 00:00:00'); +INSERT INTO timestamps(name, value) VALUES('browse_tally','1970-01-01 00:00:00'); -- Table structure for table: timestamps -- Note: stamp_name is ftp, http Modified: trunk/pypi/store.py ============================================================================== --- trunk/pypi/store.py (original) +++ trunk/pypi/store.py Thu Jul 22 23:47:37 2010 @@ -1,7 +1,16 @@ ''' Implements a store of disutils PKG-INFO entries, keyed off name, version. ''' -import sys, os, re, psycopg2, time, hashlib, random, types, math, stat, errno +import sys, os, re, time, hashlib, random, types, math, stat, errno import logging, cStringIO, string, datetime, calendar, binascii, urllib2, cgi +try: + import psycopg2 +except ImportError: + pass +try: + import sqlite3 + sqlite3_cursor = sqlite3.Cursor +except ImportError: + sqlite3_cursor = type(None) from xml.parsers import expat from distutils.version import LooseVersion import trove, openid2rp @@ -156,6 +165,9 @@ if params is None: return cursor.execute(sql) + if isinstance(cursor, sqlite3_cursor): + sql = sql.replace('%s', "?") + # Encode every incoming param to UTF-8 if it's a string safe_params = [] for param in params: @@ -181,6 +193,12 @@ self._conn = None self._cursor = None self._trove = None + if self.config.database_driver == 'sqlite3': + self.true, self.false = '1', '0' + self.can_lock = False + else: + self.true, self.false = 'TRUE', 'FALSE' + self.can_lock = True def trove(self): if not self._trove: @@ -339,7 +357,7 @@ # hide all other releases of this package if thus configured if self.get_package_autohide(name): safe_execute(cursor, 'update releases set _pypi_hidden=%s where ' - 'name=%s and version <> %s', ('TRUE', name, version)) + 'name=%s and version <> %s', (self.true, name, version)) # add description urls if html: @@ -594,8 +612,8 @@ where = ' %s '%operator.join(where) if '_pypi_hidden' in spec: - if spec['_pypi_hidden'] in ('1', 1): v = 'TRUE' - else: v = 'FALSE' + if spec['_pypi_hidden'] in ('1', 1): v = self.true + else: v = self.false if where: where += ' AND _pypi_hidden = %s'%v else: @@ -762,7 +780,7 @@ where j.version is not NULL and j.action = 'new release' and j.name = r.name and j.version = r.version - and r._pypi_hidden = FALSE + and r._pypi_hidden = '''+self.false+''' and j.submitted_date > %s order by submitted_date desc ''', (time.strftime('%Y-%m-%d %H:%M:%S +0000', time.gmtime(since)),)) @@ -833,7 +851,7 @@ from journals j, releases r where j.version is not NULL and j.name = r.name and j.version = r.version - and r._pypi_hidden = FALSE + and r._pypi_hidden = '''+self.false+''' order by submitted_date desc ''') @@ -1026,13 +1044,13 @@ '''Add a user rating of a release; message is optional''' cursor = self.get_cursor() safe_execute(cursor, '''insert into ratings (name, version, user_name, date, rating) - values(%s, %s, %s, now(), %s)''', (name, version, self.username, rating)) + values(%s, %s, %s, current_timestamp, %s)''', (name, version, self.username, rating)) if message: safe_execute(cursor, '''insert into comments(rating, user_name, date, message, in_reply_to) - values(currval('ratings_id_seq'), %s, now(), %s, NULL)''', + values(currval('ratings_id_seq'), %s, current_timestamp, %s, NULL)''', (self.username, message)) safe_execute(cursor, '''insert into comments_journal(name, version, id, submitted_by, date, action) - values(%s,%s,currval('comments_id_seq'),%s,now(),%s)''', + values(%s,%s,currval('comments_id_seq'),%s,current_timestamp,%s)''', (name, version, self.username, 'add_rating %r' % message)) def copy_rating(self, name, fromversion, toversion): @@ -1040,7 +1058,7 @@ return the comment if any''' cursor = self.get_cursor() safe_execute(cursor, '''insert into ratings(name,version,user_name,date,rating) - select name,%s,user_name,now(),rating from ratings + select name,%s,user_name,current_timestamp,rating from ratings where name=%s and version=%s and user_name=%s''', (toversion, name, fromversion, self.username)) # only copy comment, not follow-ups @@ -1052,7 +1070,7 @@ select currval('ratings_id_seq'), user_name, date, message, in_reply_to from comments where id=%s''', (cid,)) safe_execute(cursor, '''insert into comments_journal(name, version, id, submitted_by, date, action) - values(%s, %s, currval('comments_id_seq'), %s, now(), %s)''', (name, toversion, + values(%s, %s, currval('comments_id_seq'), %s, current_timestamp, %s)''', (name, toversion, self.username, 'copied %s' % cid)) safe_execute(cursor, '''select message from comments @@ -1064,7 +1082,7 @@ '''Remove a rating for the current user''' cursor = self.get_cursor() safe_execute(cursor, """insert into comments_journal(name, version, id, submitted_by, date, action) - select %s, %s, id, %s, now(), 'deleted' from ratings where user_name=%s and name=%s and version=%s""", + select %s, %s, id, %s, current_timestamp, 'deleted' from ratings where user_name=%s and name=%s and version=%s""", (name, version, self.username, self.username, name, version)) safe_execute(cursor, "delete from ratings where user_name=%s and name=%s and version=%s", (self.username, name, version)) @@ -1099,9 +1117,9 @@ safe_execute(cursor, "select c.rating, r.name, r.version from comments c, ratings r where c.id=%s and c.rating=r.id", (msg,)) rating, name, version = cursor.fetchone() safe_execute(cursor, '''insert into comments(rating, user_name, date, message, in_reply_to) - values(%s,%s,now(),%s,%s)''', (rating, self.username, comment, msg)) + values(%s,%s,current_timestamp,%s,%s)''', (rating, self.username, comment, msg)) safe_execute(cursor, '''insert into comments_journal(name, version, id, submitted_by, date, action) - values(%s,%s,currval('comments_id_seq'),%s,now(),%s)''', (name, version, self.username, + values(%s,%s,currval('comments_id_seq'),%s,current_timestamp,%s)''', (name, version, self.username, 'add %s %r' % (msg, comment))) return name, version @@ -1112,7 +1130,7 @@ name, version = cursor.fetchone() safe_execute(cursor, "delete from comments where id=%s", (msg,)) safe_execute(cursor, '''insert into comments_journal(name, version, id, submitted_by, date, action) - values(%s, %s, %s, %s, now(), 'delete')''', (name, version, msg, self.username)) + values(%s, %s, %s, %s, current_timestamp, 'delete')''', (name, version, msg, self.username)) def has_package_comments(self, name): "Return true if the package has any comments" @@ -1283,7 +1301,7 @@ (name, )) return int(cursor.fetchone()[0]) - def store_user(self, name, password, email, gpg_keyid, otk=True): + def store_user(self, name, password, email, gpg_keyid="", otk=True): ''' Store info about the user to the database. The "password" argument is passed in cleartext and sha-ed @@ -1323,7 +1341,7 @@ if not otk: return None otk = ''.join([random.choice(chars) for x in range(32)]) - safe_execute(cursor, 'insert into rego_otk (name, otk, date) values (%s, %s, now())', + safe_execute(cursor, 'insert into rego_otk (name, otk, date) values (%s, %s, current_timestamp)', (name, otk)) return otk @@ -1508,7 +1526,8 @@ # Regenerate tally. First, release locks we hold on the timestamps self._conn.commit() # Clear old tally - cursor.execute("lock table browse_tally") + if self.can_lock: + cursor.execute("lock table browse_tally") cursor.execute("delete from browse_tally") # Regenerate tally; see browse() below cursor.execute("""insert into browse_tally @@ -1516,7 +1535,7 @@ from trove_classifiers t, release_classifiers rc, releases r where rc.name=r.name and rc.version=r.version and not r._pypi_hidden and rc.trove_id=t.id group by t.l2, rc.name, rc.version) res group by res.l2""") - cursor.execute("update timestamps set value=now() where name='browse_tally'") + cursor.execute("update timestamps set value=current_timestamp where name='browse_tally'") self._conn.commit() cursor.execute("select trove_id, tally from browse_tally") return [], cursor.fetchall() @@ -1534,7 +1553,7 @@ return [], cursor.fetchall() # First compute statement to produce all packages still selected - pkgs = "select name, version, summary from releases where _pypi_hidden=FALSE" + pkgs = "select name, version, summary from releases where _pypi_hidden="+self.false for c in selected_classifiers: level = t.trove[c].level pkgs = """select distinct a.name, a.version, summary from (%s) a, release_classifiers rc, trove_classifiers t @@ -1586,7 +1605,7 @@ cursor = self.get_cursor() sql = '''insert into release_files (name, version, python_version, packagetype, comment_text, filename, md5_digest, upload_time) values - (%s, %s, %s, %s, %s, %s, %s, now())''' + (%s, %s, %s, %s, %s, %s, %s, current_timestamp)''' safe_execute(cursor, sql, (name, version, pyversion, filetype, comment, filename, md5_digest)) @@ -1764,9 +1783,9 @@ name, last_seen = users[0] if datetime.datetime.now()-datetime.timedelta(0,60) > last_seen: # refresh cookie and login time every minute - sql = 'update cookies set last_seen=now() where cookie=%s' + sql = 'update cookies set last_seen=current_timestamp where cookie=%s' safe_execute(cursor, sql, (cookie,)) - sql ='update users set last_login=now() where name=%s' + sql ='update users set last_login=current_timestamp where name=%s' safe_execute(cursor, sql, (name,)) return name return None @@ -1776,7 +1795,7 @@ cursor = self.get_cursor() cookie = binascii.hexlify(os.urandom(16)) sql = '''insert into cookies(cookie, name, last_seen) - values(%s, %s, now())''' + values(%s, %s, current_timestamp)''' safe_execute(cursor, sql, (cookie, username)) return cookie @@ -1790,7 +1809,7 @@ cursor = self.get_cursor() # Check for existing session sql = '''select id,url, assoc_handle from openid_sessions - where provider=%s and expires>now()''' + where provider=%s and expires>current_timestamp''' safe_execute(cursor, sql, (provider[0],)) sessions = cursor.fetchall() if sessions: @@ -1827,7 +1846,7 @@ cursor = self.get_cursor() # Check for existing session sql = '''select assoc_handle from openid_sessions - where provider=%s and url=%s and expires>now()''' + where provider=%s and url=%s and expires>current_timestamp''' safe_execute(cursor, sql, (claimed, endpoint,)) sessions = cursor.fetchall() if sessions: @@ -1915,6 +1934,9 @@ # already closed connection = None return self.open() + elif self.config.database_driver == 'sqlite3': + self._conn = connection = sqlite3.connect(self.config.database_name, + detect_types=sqlite3.PARSE_DECLTYPES) else: self._conn = connection = psycopg2.connect(**cd) @@ -1938,8 +1960,8 @@ if self.has_user(username): self.username = username if update_last_login: - self.get_cursor().execute(''' - update users set last_login=now() where name=%s''', (username,)) + safe_execute(self.get_cursor(), ''' + update users set last_login=current_timestamp where name=%s''', (username,)) self.userip = userip def setpasswd(self, username, password): Added: trunk/pypi/tools/demodata ============================================================================== --- (empty file) +++ trunk/pypi/tools/demodata Thu Jul 22 23:47:37 2010 @@ -0,0 +1,54 @@ +#!/usr/bin/python +import sys, os, urllib + +root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.append(root) +import admin, store, config + +cfg = config.Config(root+'/config.ini') +st = store.Store(cfg) + +# classifiers +for c in urllib.urlopen("http://pypi.python.org/pypi?%3Aaction=list_classifiers").read().splitlines(): + admin.add_classifier(st, c) + +# Demo data starts here + +# an admin +otk = st.store_user('fred', 'fredpw', 'fred at python.test') +st.delete_otk(otk) +st.add_role('fred', 'Admin', None) +# an owner +otk = st.store_user('barney', 'barneypw', 'barney at python.test') +st.delete_otk(otk) + +# package spam +st.set_user('barney', '127.0.0.1', True) +for version in ('0.8', '0.9', '1.0'): + st.store_package('spam', version, { + 'author':'Barney Geroellheimer', + 'author_email':'barney at python.test', + 'homepage':'http://spam.python.test/', + 'license':'GPL', + 'summary':'The spam package', + 'description':'Does anybody want to provide real data here?', + 'classifiers':["Development Status :: 6 - Mature", + "Programming Language :: Python :: 2"], + '_pypi_hidden':False + }) + +# package eggs +for version in ('0.1', '0.2', '0.3', '0.4'): + st.store_package('eggs', version, { + 'author':'Barney Geroellheimer', + 'author_email':'barney at python.test', + 'homepage':'http://eggs.python.test/', + 'license':'GPL', + 'summary':'The eggs package', + 'description':'Does anybody want to provide real data here?', + 'classifiers':["Development Status :: 3 - Alpha", + "Programming Language :: Python :: 3"], + '_pypi_hidden':version!='0.4' + }) + +st.commit() Added: trunk/pypi/tools/mksqlite ============================================================================== --- (empty file) +++ trunk/pypi/tools/mksqlite Thu Jul 22 23:47:37 2010 @@ -0,0 +1,26 @@ +#!/usr/bin/python +import os +dbpath = "packages.db" + +if os.path.exists(dbpath): + print "Remove",dbpath,"first" + raise SystemExit + +print "Creating database", dbpath +sqlite = os.popen('sqlite3 '+dbpath, "w") +passthrough = True +for line in open('pkgbase_schema.sql'): + if 'nosqlite-end' in line: + # end of disabled block + passthrough = True + print >>sqlite + continue + if 'nosqlite' in line: + passthrough = False + print >>sqlite + continue + if not passthrough: + print >> sqlite + continue + sqlite.write(line) +sqlite.close() Modified: trunk/pypi/webui.py ============================================================================== --- trunk/pypi/webui.py (original) +++ trunk/pypi/webui.py Thu Jul 22 23:47:37 2010 @@ -6,7 +6,12 @@ from distutils.util import rfc822_escape from distutils2.metadata import DistributionMetadata -import psycopg2 +try: + import psycopg2 + OperationalError = psycopg2.OperationalError +except ImportError: + class OperationalError(Exception): + pass try: import cElementTree @@ -259,6 +264,7 @@ self.store = store.Store(self.config) try: try: + self.store.get_cursor() # make sure we can connect self.inner_run() except NotFound: self.fail('Not Found', code=404) @@ -288,7 +294,7 @@ except IOError, error: # ignore broken pipe errors (client vanished on us) if error.errno != 32: raise - except psycopg2.OperationalError, message: + except OperationalError, message: # clean things up self.store.force_close() message = str(message) From python-checkins at python.org Fri Jul 23 00:15:18 2010 From: python-checkins at python.org (martin.von.loewis) Date: Fri, 23 Jul 2010 00:15:18 +0200 (CEST) Subject: [Pypi-checkins] r778 - trunk/pypi Message-ID: <20100722221518.916EBF4CE@mail.python.org> Author: martin.von.loewis Date: Fri Jul 23 00:15:18 2010 New Revision: 778 Modified: trunk/pypi/openid2rp.py Log: Incorporate bug fixes from upstream. Modified: trunk/pypi/openid2rp.py ============================================================================== --- trunk/pypi/openid2rp.py (original) +++ trunk/pypi/openid2rp.py Fri Jul 23 00:15:18 2010 @@ -151,7 +151,7 @@ def do_meta(self, attrs): attrs = dict(attrs) # Yadis 6.2.5 option 1: meta tag - if attrs['http-equiv'].lower() == 'x-xrds-location': + if attrs.get('http-equiv','').lower() == 'x-xrds-location': self.xrds_location = attrs['content'] def discover(url): @@ -340,6 +340,9 @@ if 'error' in data: raise ValueError, "associate failed: "+data['error'] if url.startswith('http:'): + enc_mac_key = data.get('enc_mac_key') + if not enc_mac_key: + raise ValueError, "Provider protocol error: not using DH-SHA1" enc_mac_key = base64.b64decode(data['enc_mac_key']) dh_server_public = unbtwoc(base64.b64decode(data['dh_server_public'])) # shared secret: sha1(2^(server_priv*priv) mod prime) xor enc_mac_key From python-checkins at python.org Fri Jul 23 17:49:40 2010 From: python-checkins at python.org (georg.brandl) Date: Fri, 23 Jul 2010 17:49:40 +0200 (CEST) Subject: [Pypi-checkins] r779 - trunk/pypi/tools Message-ID: <20100723154940.84A5EEE9C4@mail.python.org> Author: georg.brandl Date: Fri Jul 23 17:49:40 2010 New Revision: 779 Modified: trunk/pypi/tools/demodata (props changed) Log: Make script executable. From python-checkins at python.org Fri Jul 23 17:59:56 2010 From: python-checkins at python.org (georg.brandl) Date: Fri, 23 Jul 2010 17:59:56 +0200 (CEST) Subject: [Pypi-checkins] r780 - trunk/pypi Message-ID: <20100723155956.75988EE9C4@mail.python.org> Author: georg.brandl Date: Fri Jul 23 17:59:56 2010 New Revision: 780 Modified: trunk/pypi/pkgbase_schema.sql Log: Remove unused sequences; use SERIAL PRIMARY KEY consistently to make life easier for SQLite. Modified: trunk/pypi/pkgbase_schema.sql ============================================================================== --- trunk/pypi/pkgbase_schema.sql (original) +++ trunk/pypi/pkgbase_schema.sql Fri Jul 23 17:59:56 2010 @@ -90,10 +90,9 @@ ); CREATE TABLE cheesecake_main_indices ( - id SERIAL, + id SERIAL PRIMARY KEY, absolute INTEGER NOT NULL, - relative INTEGER NOT NULL, - PRIMARY KEY (id) + relative INTEGER NOT NULL ); CREATE TABLE cheesecake_subindices ( @@ -146,15 +145,6 @@ CREATE INDEX trove_class_id_idx ON trove_classifiers(id); --- trove ids sequence --- nosqlite -CREATE TABLE dual (dummy INTEGER); -INSERT INTO dual VALUES (1); -CREATE SEQUENCE trove_ids; -SELECT setval('trove_ids', 1000) FROM dual; --- nosqlite-end - - -- Table structure for table: release_classifiers CREATE TABLE release_classifiers ( name TEXT, @@ -335,13 +325,12 @@ -- ratings CREATE TABLE ratings( - id SERIAL UNIQUE, + id SERIAL PRIMARY KEY, name TEXT, version TEXT, user_name TEXT REFERENCES users ON DELETE CASCADE, date TIMESTAMP, rating INTEGER, - PRIMARY KEY (name, version, user_name), FOREIGN KEY (name, version) REFERENCES releases ON UPDATE CASCADE ON DELETE CASCADE ); CREATE INDEX rating_name_version ON ratings(name, version); From python-checkins at python.org Fri Jul 23 18:10:51 2010 From: python-checkins at python.org (georg.brandl) Date: Fri, 23 Jul 2010 18:10:51 +0200 (CEST) Subject: [Pypi-checkins] r781 - trunk/pypi Message-ID: <20100723161051.4B44FEEA25@mail.python.org> Author: georg.brandl Date: Fri Jul 23 18:10:51 2010 New Revision: 781 Modified: trunk/pypi/store.py Log: Introduce a database-specific way to get the last inserted rowid. Modified: trunk/pypi/store.py ============================================================================== --- trunk/pypi/store.py (original) +++ trunk/pypi/store.py Fri Jul 23 18:10:51 2010 @@ -200,6 +200,15 @@ self.true, self.false = 'TRUE', 'FALSE' self.can_lock = True + def last_id(self, tablename): + ''' Return an SQL expression that returns the last inserted row, + where the row is in the given table. + ''' + if self.config.database_driver == 'sqlite3': + return 'last_insert_rowid()' + else: + return "currval('%s_id_seq')" % tablename + def trove(self): if not self._trove: self._trove = trove.Trove(self.get_cursor()) @@ -1047,10 +1056,12 @@ values(%s, %s, %s, current_timestamp, %s)''', (name, version, self.username, rating)) if message: safe_execute(cursor, '''insert into comments(rating, user_name, date, message, in_reply_to) - values(currval('ratings_id_seq'), %s, current_timestamp, %s, NULL)''', + values(%s, %%s, current_timestamp, %%s, NULL)''' + % self.last_id('ratings'), (self.username, message)) safe_execute(cursor, '''insert into comments_journal(name, version, id, submitted_by, date, action) - values(%s,%s,currval('comments_id_seq'),%s,current_timestamp,%s)''', + values(%%s, %%s, %s, %%s, current_timestamp, %%s)''' + % self.last_id('comments'), (name, version, self.username, 'add_rating %r' % message)) def copy_rating(self, name, fromversion, toversion): @@ -1067,11 +1078,11 @@ if cid: cid = cid[0] safe_execute(cursor, '''insert into comments(rating, user_name, date, message, in_reply_to) - select currval('ratings_id_seq'), user_name, date, message, in_reply_to - from comments where id=%s''', (cid,)) + select %s, user_name, date, message, in_reply_to + from comments where id=%%s''' % self.last_id('ratings'), (cid,)) safe_execute(cursor, '''insert into comments_journal(name, version, id, submitted_by, date, action) - values(%s, %s, currval('comments_id_seq'), %s, current_timestamp, %s)''', (name, toversion, - self.username, 'copied %s' % cid)) + values(%%s, %%s, %s, %%s, current_timestamp, %%s)''' % self.last_id('comments'), + (name, toversion, self.username, 'copied %s' % cid)) safe_execute(cursor, '''select message from comments where id=%s''', (cid,)) @@ -1119,8 +1130,8 @@ safe_execute(cursor, '''insert into comments(rating, user_name, date, message, in_reply_to) values(%s,%s,current_timestamp,%s,%s)''', (rating, self.username, comment, msg)) safe_execute(cursor, '''insert into comments_journal(name, version, id, submitted_by, date, action) - values(%s,%s,currval('comments_id_seq'),%s,current_timestamp,%s)''', (name, version, self.username, - 'add %s %r' % (msg, comment))) + values(%%s, %%s, %s, %%s, current_timestamp, %%s)''' % self.last_id('comments'), + (name, version, self.username, 'add %s %r' % (msg, comment))) return name, version def remove_comment(self, msg): @@ -1833,10 +1844,11 @@ session['assoc_handle'], now+datetime.timedelta(0,int(session['expires_in'])), session['mac_key'])) + safe_execute(cursor, 'select %s' % self.last_id('openid_sessions')) + session_id = cursor.fetchone()[0] for t in stypes: safe_execute(cursor, '''insert into openid_stypes(id, stype) - values(currval('openid_sessions_id_seq'),%s)''', - (t,)) + values(%s, %s)''', (session_id, t)) return stypes, url, session['assoc_handle'] def get_session_for_endpoint(self, claimed, stypes, endpoint): @@ -1863,11 +1875,12 @@ session['assoc_handle'], now+datetime.timedelta(0,int(session['expires_in'])), session['mac_key'])) + safe_execute(cursor, 'select %s' % self.last_id('openid_sessions')) + session_id = cursor.fetchone()[0] # store stypes as well, so we can remember whether claimed is an OP ID or a user ID for t in stypes: safe_execute(cursor, '''insert into openid_stypes(id, stype) - values(currval('openid_sessions_id_seq'),%s)''', - (t,)) + values(%s, %s)''', (session_id, t)) return session['assoc_handle'] def get_session_by_handle(self, assoc_handle): From python-checkins at python.org Fri Jul 23 18:11:10 2010 From: python-checkins at python.org (georg.brandl) Date: Fri, 23 Jul 2010 18:11:10 +0200 (CEST) Subject: [Pypi-checkins] r782 - trunk/pypi/tools Message-ID: <20100723161110.969A3EEA08@mail.python.org> Author: georg.brandl Date: Fri Jul 23 18:11:10 2010 New Revision: 782 Modified: trunk/pypi/tools/mksqlite Log: SQLite unfortunately does not know about SERIAL. Modified: trunk/pypi/tools/mksqlite ============================================================================== --- trunk/pypi/tools/mksqlite (original) +++ trunk/pypi/tools/mksqlite Fri Jul 23 18:11:10 2010 @@ -22,5 +22,6 @@ if not passthrough: print >> sqlite continue - sqlite.write(line) + # make sqlite happy: SERIAL is not a valid type + sqlite.write(line.replace('SERIAL PRIMARY KEY', 'INTEGER PRIMARY KEY')) sqlite.close() From python-checkins at python.org Fri Jul 23 18:12:01 2010 From: python-checkins at python.org (georg.brandl) Date: Fri, 23 Jul 2010 18:12:01 +0200 (CEST) Subject: [Pypi-checkins] r783 - trunk/pypi/templates Message-ID: <20100723161201.A5076EE9F8@mail.python.org> Author: georg.brandl Date: Fri Jul 23 18:12:01 2010 New Revision: 783 Modified: trunk/pypi/templates/display.pt Log: Vote -> rating for consistency. Modified: trunk/pypi/templates/display.pt ============================================================================== --- trunk/pypi/templates/display.pt (original) +++ trunk/pypi/templates/display.pt Fri Jul 23 18:12:01 2010 @@ -238,7 +238,7 @@ - +
Modified: trunk/pypi/templates/standard_template.pt ============================================================================== --- trunk/pypi/templates/standard_template.pt (original) +++ trunk/pypi/templates/standard_template.pt Sat Jul 24 16:37:09 2010 @@ -28,7 +28,14 @@
+ Downloads ↓ +
+
Latest Version:
@@ -32,8 +36,8 @@