From python-checkins at python.org Sun Jul 4 20:36:29 2010 From: python-checkins at python.org (martin.von.loewis) Date: Sun, 4 Jul 2010 20:36:29 +0200 (CEST) Subject: [Pypi-checkins] r765 - trunk/pypi/tools Message-ID: <20100704183629.2679BEE988@mail.python.org> Author: martin.von.loewis Date: Sun Jul 4 20:36:28 2010 New Revision: 765 Modified: trunk/pypi/tools/apache_stats.py Log: Integrate change from pep381client: use two-digits months and days. Modified: trunk/pypi/tools/apache_stats.py ============================================================================== --- trunk/pypi/tools/apache_stats.py (original) +++ trunk/pypi/tools/apache_stats.py Sun Jul 4 20:36:28 2010 @@ -112,7 +112,7 @@ def build_local_stats(self, year, month, day, logfile, directory=None): """builds local stats with default values""" - filename = '%d-%d-%d.bz2' % (year, month, day) + filename = '%d-%.2d-%.2d.bz2' % (year, month, day) if directory is not None: filename = os.path.join(directory, filename) From python-checkins at python.org Sun Jul 4 20:45:13 2010 From: python-checkins at python.org (martin.von.loewis) Date: Sun, 4 Jul 2010 20:45:13 +0200 (CEST) Subject: [Pypi-checkins] r766 - trunk/pypi/tools Message-ID: <20100704184513.AF20FEE988@mail.python.org> Author: martin.von.loewis Date: Sun Jul 4 20:45:13 2010 New Revision: 766 Added: trunk/pypi/tools/downloadstats (contents, props changed) Log: Adapt script from pep381client. Added: trunk/pypi/tools/downloadstats ============================================================================== --- (empty file) +++ trunk/pypi/tools/downloadstats Sun Jul 4 20:45:13 2010 @@ -0,0 +1,24 @@ +#!/usr/bin/python +# Generates download stats for all days in the given log files, +# except for the oldest and the newest day. +import sys, os, csv +import apache_reader, apache_stats + +statsdir = '/data/pypi/local-stats/' + +days = set() +records = [] +for fn in sys.argv[1:]: + for record in apache_reader.ApacheLogReader(fn, '/packages'): + days.add((record['year'], record['month'], record['day'])) + records.append(record) + +days = sorted(days)[1:-1] + +class Stats(apache_stats.LocalStats): + def _get_logs(self, logfile, files_url): + return records +stats = Stats() +for year,month,day in days: + stats.build_local_stats(year, month, day, None, statsdir+'days') + From python-checkins at python.org Fri Jul 9 01:24:58 2010 From: python-checkins at python.org (martin.von.loewis) Date: Fri, 9 Jul 2010 01:24:58 +0200 (CEST) Subject: [Pypi-checkins] r767 - trunk/pypi/tools Message-ID: <20100708232458.30296EE991@mail.python.org> Author: martin.von.loewis Date: Fri Jul 9 01:24:57 2010 New Revision: 767 Added: trunk/pypi/tools/integratestats (contents, props changed) Modified: trunk/pypi/tools/apache_stats.py Log: Add integratestats tool. Modified: trunk/pypi/tools/apache_stats.py ============================================================================== --- trunk/pypi/tools/apache_stats.py (original) +++ trunk/pypi/tools/apache_stats.py Fri Jul 9 01:24:57 2010 @@ -37,13 +37,6 @@ current line. if the callable returns True, the line is not included """ - if isinstance(fileobj, str): - fileobj = self._get_file_obj(fileobj, 'w', compression) - file_created = True - else: - file_created = False - - writer = csv.writer(fileobj) downloads = {} for log in self._get_logs(logfile, files_url): if filter is not None: @@ -58,6 +51,16 @@ downloads[key] += count else: downloads[key] = count + self._write_stats(fileobj, downloads) + + def _write_stats(self, fileobj, downloads, compression=None): + if isinstance(fileobj, str): + fileobj = self._get_file_obj(fileobj, 'w', compression) + file_created = True + else: + file_created = False + + writer = csv.writer(fileobj) filenames = downloads.keys() filenames.sort() for key in filenames: @@ -107,9 +110,17 @@ yield {'packagename': line[0], 'filename': line[1], 'useragent': line[2], - 'count': line[3]} + 'count': int(line[3])} #reader.close() + def read_stats_dict(self, stats_file): + res = {} + for r in self.read_stats(stats_file): + key = (r['packagename'], r['filename'], r['useragent']) + value = r['count'] + res[key] = value + return res + def build_local_stats(self, year, month, day, logfile, directory=None): """builds local stats with default values""" filename = '%d-%.2d-%.2d.bz2' % (year, month, day) @@ -119,6 +130,27 @@ self.build_daily_stats(year, month, day, logfile, filename, compression='bz2') + def integrate_stats(self, targetdir, year, month, day, fd): + new = self.read_stats_dict(fd) + oldpath = "%s/days/%s-%.2s-%.2s.bz2" % (targetdir, year, month, day) + if os.path.exists(oldpath): + old = self.read_stats_dict(oldpath) + for k, v in new.items(): + old[k] = old.get(k, 0) + v + else: + old = new + self._write_stats(oldpath, old, 'bz2') + monthpath = "%s/months/%s-%.2s.bz2" % (targetdir, year, month) + if os.path.exists(monthpath): + old = self.read_stats_dict(monthpath) + for k, v in new.items(): + old[k] = old.get(k, 0) + v + else: + old = new + self._write_stats(monthpath, old, 'bz2') + return new + + class ApacheLocalStats(LocalStats): """concrete class that uses the ApacheLogReader""" def _get_logs(self, logfile, files_url): Added: trunk/pypi/tools/integratestats ============================================================================== --- (empty file) +++ trunk/pypi/tools/integratestats Fri Jul 9 01:24:57 2010 @@ -0,0 +1,65 @@ +#!/usr/bin/python +import sys, os, socket, psycopg2, urllib, re, bz2, cStringIO, ConfigParser +sys.path.append(os.path.dirname(__file__)+"/..") +import apache_stats + +statsdir = '/data/pypi/stats/' + +def integrate(config, data): + # Setup database connection + c = ConfigParser.ConfigParser({'user':'', 'password':''}) + c.read(config) + dbname = c.get('database', 'name') + dbuser = c.get('database', 'user') + dbpass = c.get('database', 'password') + dbconn = psycopg2.connect(database=dbname, user=dbuser, password=dbpass) + cursor = dbconn.cursor() + for (package, filename, browser), count in data.items(): + cursor.execute('update release_files set downloads=downloads+%s where filename=%s', + (count, filename)) + dbconn.commit() + dbconn.close() + +def integrate_remote(config, host, dbupdate=True): + index = urllib.urlopen('http://%s.pypi.python.org/local-stats/days' % host).read() + files = set(re.findall('href=.(20..-..-..).bz2', index)) + try: + integrated = open('/data/pypi/stats/integrated/'+host).readlines() + integrated = set([x.strip() for x in integrated]) + except IOError: + integrated = set() + missing = files-integrated + stats = apache_stats.LocalStats() + for m in missing: + data = urllib.urlopen('http://%s.pypi.python.org/local-stats/days/%s.bz2' % (host, m)).read() + data = bz2.decompress(data) + data = cStringIO.StringIO(data) + year, month, day = m.split('-') + # index integration + delta = stats.integrate_stats(statsdir, year, month, day, data) + if dbupdate: + # database integration + integrate(config, delta) + integrated.add(m) + open('/data/pypi/stats/integrated/'+host, 'w').write('\n'.join(sorted(integrated))) + +def main(): + lasts = socket.gethostbyname_ex('last.pypi.python.org') + # look for name X.pypi.python.org + lasts = [lasts[0]] + lasts[1] + for last in lasts: + if last[1:] == '.pypi.python.org': + break + else: + raise ValueError, "Could not properly resolve last mirror name" + last = last.split('.')[0] + integrate_remote(None, 'a', False) + host = 'b' + while True: + integrate_remote(sys.argv[1], host) + host = chr(ord(host)+1) + if host == last: + break + +main() + From python-checkins at python.org Fri Jul 9 01:28:37 2010 From: python-checkins at python.org (martin.von.loewis) Date: Fri, 9 Jul 2010 01:28:37 +0200 (CEST) Subject: [Pypi-checkins] r768 - trunk/pypi/tools Message-ID: <20100708232837.1185EEE995@mail.python.org> Author: martin.von.loewis Date: Fri Jul 9 01:28:36 2010 New Revision: 768 Modified: trunk/pypi/tools/integratestats Log: Fix stats path. Modified: trunk/pypi/tools/integratestats ============================================================================== --- trunk/pypi/tools/integratestats (original) +++ trunk/pypi/tools/integratestats Fri Jul 9 01:28:36 2010 @@ -3,7 +3,7 @@ sys.path.append(os.path.dirname(__file__)+"/..") import apache_stats -statsdir = '/data/pypi/stats/' +statsdir = '/data/www/pypi/stats/' def integrate(config, data): # Setup database connection From python-checkins at python.org Fri Jul 9 01:34:19 2010 From: python-checkins at python.org (martin.von.loewis) Date: Fri, 9 Jul 2010 01:34:19 +0200 (CEST) Subject: [Pypi-checkins] r769 - trunk/pypi/tools Message-ID: <20100708233419.3217DEEAF6@mail.python.org> Author: martin.von.loewis Date: Fri Jul 9 01:34:19 2010 New Revision: 769 Modified: trunk/pypi/tools/integratestats Log: Fix integrated path. Modified: trunk/pypi/tools/integratestats ============================================================================== --- trunk/pypi/tools/integratestats (original) +++ trunk/pypi/tools/integratestats Fri Jul 9 01:34:19 2010 @@ -3,7 +3,7 @@ sys.path.append(os.path.dirname(__file__)+"/..") import apache_stats -statsdir = '/data/www/pypi/stats/' +statsdir = '/data/www/pypi/stats' def integrate(config, data): # Setup database connection @@ -24,7 +24,7 @@ index = urllib.urlopen('http://%s.pypi.python.org/local-stats/days' % host).read() files = set(re.findall('href=.(20..-..-..).bz2', index)) try: - integrated = open('/data/pypi/stats/integrated/'+host).readlines() + integrated = open(statsdir+'/integrated/'+host).readlines() integrated = set([x.strip() for x in integrated]) except IOError: integrated = set() @@ -41,7 +41,7 @@ # database integration integrate(config, delta) integrated.add(m) - open('/data/pypi/stats/integrated/'+host, 'w').write('\n'.join(sorted(integrated))) + open(statsdir+'/integrated/'+host, 'w').write('\n'.join(sorted(integrated))) def main(): lasts = socket.gethostbyname_ex('last.pypi.python.org') From python-checkins at python.org Fri Jul 9 09:09:00 2010 From: python-checkins at python.org (martin.von.loewis) Date: Fri, 9 Jul 2010 09:09:00 +0200 (CEST) Subject: [Pypi-checkins] r770 - trunk/pypi/tools Message-ID: <20100709070900.477DCEE990@mail.python.org> Author: martin.von.loewis Date: Fri Jul 9 09:09:00 2010 New Revision: 770 Modified: trunk/pypi/tools/apache_stats.py Log: Restore missing parameter. Modified: trunk/pypi/tools/apache_stats.py ============================================================================== --- trunk/pypi/tools/apache_stats.py (original) +++ trunk/pypi/tools/apache_stats.py Fri Jul 9 09:09:00 2010 @@ -51,7 +51,7 @@ downloads[key] += count else: downloads[key] = count - self._write_stats(fileobj, downloads) + self._write_stats(fileobj, downloads, compression=compression) def _write_stats(self, fileobj, downloads, compression=None): if isinstance(fileobj, str): From python-checkins at python.org Sat Jul 10 12:51:13 2010 From: python-checkins at python.org (martin.von.loewis) Date: Sat, 10 Jul 2010 12:51:13 +0200 (CEST) Subject: [Pypi-checkins] r771 - trunk/pypi Message-ID: <20100710105113.60ECBEEA5A@mail.python.org> Author: martin.von.loewis Date: Sat Jul 10 12:51:13 2010 New Revision: 771 Added: trunk/pypi/mirrors.txt (contents, props changed) Log: Document mirrors in this text file for now. Added: trunk/pypi/mirrors.txt ============================================================================== --- (empty file) +++ trunk/pypi/mirrors.txt Sat Jul 10 12:51:13 2010 @@ -0,0 +1,4 @@ +b: 141.89.226.2, martin at v.loewis.de +c: h1606280.stratoserver.net, pypi at zopyx.com +d: pypi.websushi.org, jezdez+pypi at enn.io + From python-checkins at python.org Wed Jul 21 00:09:03 2010 From: python-checkins at python.org (martin.von.loewis) Date: Wed, 21 Jul 2010 00:09:03 +0200 (CEST) Subject: [Pypi-checkins] r773 - trunk/pypi Message-ID: <20100720220903.66128EE995@mail.python.org> Author: martin.von.loewis Date: Wed Jul 21 00:09:03 2010 New Revision: 773 Modified: trunk/pypi/store.py trunk/pypi/webui.py Log: Cleanup imports. Modified: trunk/pypi/store.py ============================================================================== --- trunk/pypi/store.py (original) +++ trunk/pypi/store.py Wed Jul 21 00:09:03 2010 @@ -1,6 +1,6 @@ ''' Implements a store of disutils PKG-INFO entries, keyed off name, version. ''' -import sys, os, re, psycopg2, time, sha, random, types, math, stat, errno +import sys, os, re, psycopg2, time, hashlib, random, types, math, stat, errno import logging, cStringIO, string, datetime, calendar, binascii, urllib2, cgi from xml.parsers import expat from distutils.version import LooseVersion @@ -1295,7 +1295,7 @@ if self.has_user(name): if password: # update existing user, including password - password = sha.sha(password).hexdigest() + password = hashlib.sha1(password).hexdigest() safe_execute(cursor, 'update users set password=%s, email=%s where name=%s', (password, email, name)) @@ -1314,7 +1314,7 @@ if cursor.fetchone()[0] > 0: raise ValueError, "Email address already belongs to a different user" - password = sha.sha(password).hexdigest() + password = hashlib.sha1(password).hexdigest() # new user safe_execute(cursor, @@ -1943,7 +1943,7 @@ self.userip = userip def setpasswd(self, username, password): - password = sha.sha(password).hexdigest() + password = hashlib.sha1(password).hexdigest() self.get_cursor().execute(''' update users set password=%s where name=%s ''', (password, username)) Modified: trunk/pypi/webui.py ============================================================================== --- trunk/pypi/webui.py (original) +++ trunk/pypi/webui.py Wed Jul 21 00:09:03 2010 @@ -1,7 +1,7 @@ # system imports -import sys, os, urllib, cStringIO, traceback, cgi, binascii, getopt, md5 -import time, random, smtplib, base64, sha, email, types, stat, urlparse -import re, zipfile, logging, pprint, sets, shutil, Cookie, subprocess +import sys, os, urllib, cStringIO, traceback, cgi, binascii +import time, random, smtplib, base64, email, types, urlparse +import re, zipfile, logging, shutil, Cookie, subprocess, hashlib from zope.pagetemplate.pagetemplatefile import PageTemplateFile from distutils.util import rfc822_escape from distutils2.metadata import DistributionMetadata @@ -14,7 +14,7 @@ from xml.etree import cElementTree # local imports -import store, config, trove, versionpredicate, verify_filetype, rpc +import store, config, versionpredicate, verify_filetype, rpc import MailingLogger, openid2rp from mini_pkg_resources import safe_name @@ -433,7 +433,7 @@ # Invalid base64, or not exactly one colon un = pw = '' if self.store.has_user(un): - pw = sha.sha(pw).hexdigest() + pw = hashlib.sha1(pw).hexdigest() user = self.store.get_user(un) if pw != user['password']: raise Unauthorised, 'Incorrect password' @@ -989,7 +989,7 @@ write_element(pelem, person, 'foaf:name') email = info[person+'_email'] if email and email != 'UNKNOWN': - obj = sha.new(email) + obj = hashlib.sha1(email) email = binascii.b2a_hex(obj.digest()) elem = SE(pelem, 'foaf:mbox_sha1sum') elem.text = email @@ -1326,7 +1326,7 @@ self.write_template('index.pt', title="Index of Packages", matches=l) - STOPWORDS = sets.Set([ + STOPWORDS = set([ "a", "and", "are", "as", "at", "be", "but", "by", "for", "if", "in", "into", "is", "it", "no", "not", "of", "on", "or", "such", @@ -2218,7 +2218,7 @@ raise FormError, "signature is not ASCII-armored" # digest content - m = md5.new() + m = hashlib.md5() m.update(content) calc_digest = m.hexdigest() @@ -2228,7 +2228,7 @@ self.fail(heading='MD5 digest mismatch', message='''The MD5 digest supplied does not match a digest calculated from the uploaded file (m = - md5.new(); m.update(content); digest = + hashlib.md5(); m.update(content); digest = m.hexdigest())''') return From python-checkins at python.org Wed Jul 21 14:48:11 2010 From: python-checkins at python.org (martin.von.loewis) Date: Wed, 21 Jul 2010 14:48:11 +0200 (CEST) Subject: [Pypi-checkins] r774 - trunk/pypi Message-ID: <20100721124811.C9016E9E6@mail.python.org> Author: martin.von.loewis Date: Wed Jul 21 14:48:11 2010 New Revision: 774 Modified: trunk/pypi/pkgbase_schema.sql Log: Fix schema so that postgres can import it. Modified: trunk/pypi/pkgbase_schema.sql ============================================================================== --- trunk/pypi/pkgbase_schema.sql (original) +++ trunk/pypi/pkgbase_schema.sql Wed Jul 21 14:48:11 2010 @@ -1,3 +1,4 @@ +begin; -- Table structure for table: users CREATE TABLE users ( name TEXT PRIMARY KEY, @@ -35,7 +36,7 @@ created TIMESTAMP, nonce TEXT ); -CREATE INDEX openid_nonces_nonce ON openid_nonces(created); +CREATE INDEX openid_nonces_created ON openid_nonces(created); CREATE INDEX openid_nonces_nonce ON openid_nonces(nonce); CREATE TABLE cookies ( @@ -51,7 +52,6 @@ key TEXT ); CREATE INDEX sshkeys_name ON sshkeys(name); -CREATE INDEX rego_otk_otk_idx ON rego_otk(otk); -- Table structure for table: rego_otk CREATE TABLE rego_otk ( @@ -59,7 +59,7 @@ otk TEXT, date TIMESTAMP ); CREATE INDEX rego_otk_name_idx ON rego_otk(name); - +CREATE INDEX rego_otk_otk_idx ON rego_otk(otk); -- Table structure for table: journals CREATE TABLE journals ( @@ -102,25 +102,6 @@ PRIMARY KEY (main_index_id, name) ); --- Table structure for table: cheesecake_main_indices -CREATE TABLE cheesecake_main_indices ( - id SERIAL, - absolute INTEGER NOT NULL, - relative INTEGER NOT NULL, - PRIMARY KEY (id) -); - - --- Table structure for table: cheesecake_subindices -CREATE TABLE cheesecake_subindices ( - main_index_id INTEGER REFERENCES cheesecake_main_indices, - name TEXT, - value INTEGER NOT NULL, - details TEXT NOT NULL, - PRIMARY KEY (main_index_id, name) -); - - -- Table structure for table: releases CREATE TABLE releases ( name TEXT REFERENCES packages ON UPDATE CASCADE, @@ -203,7 +184,7 @@ ); CREATE INDEX rel_req_name_idx ON release_requires(name); CREATE INDEX rel_req_version_id_idx ON release_requires(version); -CREATE INDEX rel_req_name_version_idx ON release_obsoletes (name,version); +CREATE INDEX rel_req_name_version_idx ON release_requires(name,version); -- Table structure for table: release_obsoletes CREATE TABLE release_obsoletes ( @@ -349,14 +330,6 @@ ); -- ratings -CREATE TABLE comments( - id SERIAL PRIMARY KEY, - rating INTEGER REFERENCES ratings(id) ON DELETE CASCADE, - user_name TEXT REFERENCES users ON DELETE CASCADE, - date TIMESTAMP, - message TEXT, - in_reply_to INTEGER REFERENCES comments ON DELETE CASCADE -); CREATE TABLE ratings( id SERIAL UNIQUE, name TEXT, @@ -368,6 +341,14 @@ FOREIGN KEY (name, version) REFERENCES releases ON UPDATE CASCADE ON DELETE CASCADE ); CREATE INDEX rating_name_version ON ratings(name, version); +CREATE TABLE comments( + id SERIAL PRIMARY KEY, + rating INTEGER REFERENCES ratings(id) ON DELETE CASCADE, + user_name TEXT REFERENCES users ON DELETE CASCADE, + date TIMESTAMP, + message TEXT, + in_reply_to INTEGER REFERENCES comments ON DELETE CASCADE +); CREATE TABLE comments_journal( name text, version text, @@ -378,3 +359,4 @@ FOREIGN KEY (name, version) REFERENCES releases (name, version) ON DELETE CASCADE ); +commit; \ No newline at end of file From python-checkins at python.org Thu Jul 22 18:55:41 2010 From: python-checkins at python.org (martin.von.loewis) Date: Thu, 22 Jul 2010 18:55:41 +0200 (CEST) Subject: [Pypi-checkins] r775 - trunk/pypi Message-ID: <20100722165541.29D4EEE98E@mail.python.org> Author: martin.von.loewis Date: Thu Jul 22 18:55:40 2010 New Revision: 775 Added: trunk/pypi/config.ini.template - copied unchanged from r774, trunk/pypi/config.ini Removed: trunk/pypi/config.ini Log: Make config.ini a template file only. Deleted: /trunk/pypi/config.ini ============================================================================== --- /trunk/pypi/config.ini Thu Jul 22 18:55:40 2010 +++ (empty file) @@ -1,32 +0,0 @@ -[database] -name = packages -user = pypi -files_dir = /MacDev/svn.python.org/pypi-pep345/files -docs_dir = /MacDev/svn.python.org/pypi-pep345/docs - -[webui] -mailhost = mail.commonground.com.au -adminemail = richard at commonground.com.au -replyto = richard at commonground.com.au -url = http://localhost/cgi-bin/pypi.cgi -pydotorg = http://www.python.org/ - -simple_script = /simple -files_url = http://localhost/pypi_files -rss_file = /tmp/pypi_rss.xml -debug_mode = yes -cheesecake_password = secret -privkey = privkey -simple_sign_script = /serversig - -[logging] -file = -mailhost = -fromaddr = -toaddrs = - -[mirrors] -folder = mirrors -local-stats = local-stats -global-stats = global-stats - From python-checkins at python.org Thu Jul 22 18:57:40 2010 From: python-checkins at python.org (martin.von.loewis) Date: Thu, 22 Jul 2010 18:57:40 +0200 (CEST) Subject: [Pypi-checkins] r776 - trunk/pypi Message-ID: <20100722165740.66A25EE98E@mail.python.org> Author: martin.von.loewis Date: Thu Jul 22 18:57:40 2010 New Revision: 776 Modified: trunk/pypi/README Log: Mention config.ini template. Modified: trunk/pypi/README ============================================================================== --- trunk/pypi/README (original) +++ trunk/pypi/README Thu Jul 22 18:57:40 2010 @@ -20,8 +20,9 @@ Make sure you read http://wiki.python.org/moin/CheeseShopDev#DevelopmentEnvironmentHints and you have a working PostgreSQL DB. -Make sure your config.ini is up-to-date. Change CONFIG_FILE at the begining -of pypi.wsgi, so it looks like this:: +Make sure your config.ini is up-to-date, initially copying from +config.ini.template. Change CONFIG_FILE at the begining of pypi.wsgi, +so it looks like this:: CONFIG_FILE = 'config.ini' From python-checkins at python.org Thu Jul 22 23:47:37 2010 From: python-checkins at python.org (martin.von.loewis) Date: Thu, 22 Jul 2010 23:47:37 +0200 (CEST) Subject: [Pypi-checkins] r777 - in trunk/pypi: . tools Message-ID: <20100722214737.C7476F591@mail.python.org> Author: martin.von.loewis Date: Thu Jul 22 23:47:37 2010 New Revision: 777 Added: trunk/pypi/tools/demodata trunk/pypi/tools/mksqlite (contents, props changed) Modified: trunk/pypi/README trunk/pypi/admin.py trunk/pypi/config.ini.template trunk/pypi/config.py trunk/pypi/pkgbase_schema.sql trunk/pypi/store.py trunk/pypi/webui.py Log: Add support for running PyPI on sqlite3. Modified: trunk/pypi/README ============================================================================== --- trunk/pypi/README (original) +++ trunk/pypi/README Thu Jul 22 23:47:37 2010 @@ -10,7 +10,7 @@ - zope.tal - zope.tales - zope.i18nmessageid -- psycopg2 +- psycopg2 (for testing, sqlite3 might be sufficient) - docutils - distutils2 @@ -32,7 +32,7 @@ $ virtualenv --no-site-packages --distribute . $ bin/easy_install cElementTree zope.interface zope.pagetemplate $ bin/easy_install zope.tal zope.tales zope.i18nmessageid psycopg2 - $ bin/easy_install M2Crypto BeautifulSoup docutils + $ bin/easy_install docutils Then you can launch the server using the pypi.wsgi script:: @@ -41,3 +41,12 @@ PyPI will be available in your browser at http://localhost:8000 +Database Setup +-------------- + +To fill a database, run pkgbase_schema.sql on an empty Postgres database. +Then run tools/demodata to populate the database with dummy data. + +For testing purposes, run tools/mksqlite to create packages.db. Set +[database]driver to sqlite3, and [database]name to packages.db, then +run tools/demodata to populate the database. \ No newline at end of file Modified: trunk/pypi/admin.py ============================================================================== --- trunk/pypi/admin.py (original) +++ trunk/pypi/admin.py Thu Jul 22 23:47:37 2010 @@ -41,27 +41,30 @@ raise ValueError, "user is not currently owner" store.delete_role(owner, 'Owner', package) -def add_classifier(store, classifier): +def add_classifier(st, classifier): ''' Add a classifier to the trove_classifiers list ''' - cursor = store.get_cursor() + cursor = st.get_cursor() cursor.execute("select max(id) from trove_classifiers") - id = int(cursor.fetchone()[0]) + 1 + id = cursor.fetchone()[0] + if id: + id = int(id) + 1 + else: + id = 1 fields = [f.strip() for f in classifier.split('::')] for f in fields: assert ':' not in f levels = [] for l in range(2, len(fields)): c2 = ' :: '.join(fields[:l]) - cursor.execute('select id from trove_classifiers where classifier=%s', (c2,)) + store.safe_execute(cursor, 'select id from trove_classifiers where classifier=%s', (c2,)) l = cursor.fetchone() if not l: raise ValueError, c2 + " is not a known classifier" levels.append(l[0]) levels += [id] + [0]*(3-len(levels)) - cursor.execute('insert into trove_classifiers (id, classifier, l2, l3, l4, l5) ' + store.safe_execute(cursor, 'insert into trove_classifiers (id, classifier, l2, l3, l4, l5) ' 'values (%s,%s,%s,%s,%s,%s)', [id, classifier]+levels) - print 'done' def rename_package(store, old, new): ''' Rename a package. ''' @@ -135,6 +138,7 @@ remove_package(*args) elif command == 'addclass': add_classifier(*args) + print 'done' elif command == 'addowner': add_owner(*args) elif command == 'delowner': Modified: trunk/pypi/config.ini.template ============================================================================== --- trunk/pypi/config.ini.template (original) +++ trunk/pypi/config.ini.template Thu Jul 22 23:47:37 2010 @@ -1,4 +1,5 @@ [database] +driver = postgresql2 name = packages user = pypi files_dir = /MacDev/svn.python.org/pypi-pep345/files @@ -8,7 +9,7 @@ mailhost = mail.commonground.com.au adminemail = richard at commonground.com.au replyto = richard at commonground.com.au -url = http://localhost/cgi-bin/pypi.cgi +url = http://localhost:8000/pypi pydotorg = http://www.python.org/ simple_script = /simple Modified: trunk/pypi/config.py ============================================================================== --- trunk/pypi/config.py (original) +++ trunk/pypi/config.py Thu Jul 22 23:47:37 2010 @@ -8,6 +8,10 @@ c.read(configfile) self.database_name = c.get('database', 'name') self.database_user = c.get('database', 'user') + if c.has_option('database', 'driver'): + self.database_driver = c.get('database', 'driver') + else: + self.database_driver = 'psycopg2' if c.has_option('database', 'password'): self.database_pw = c.get('database', 'password') else: Modified: trunk/pypi/pkgbase_schema.sql ============================================================================== --- trunk/pypi/pkgbase_schema.sql (original) +++ trunk/pypi/pkgbase_schema.sql Thu Jul 22 23:47:37 2010 @@ -72,9 +72,11 @@ ); CREATE INDEX journals_name_idx ON journals(name); CREATE INDEX journals_version_idx ON journals(version); +-- nosqlite CREATE INDEX journals_latest_releases ON journals(submitted_date, name, version) WHERE version IS NOT NULL AND action='new release'; +-- nosqlite-end CREATE INDEX journals_changelog ON journals(submitted_date, name, version, action); @@ -145,10 +147,12 @@ -- trove ids sequence +-- nosqlite CREATE TABLE dual (dummy INTEGER); INSERT INTO dual VALUES (1); CREATE SEQUENCE trove_ids; SELECT setval('trove_ids', 1000) FROM dual; +-- nosqlite-end -- Table structure for table: release_classifiers @@ -312,9 +316,9 @@ name TEXT PRIMARY KEY, value TIMESTAMP ); -INSERT INTO timestamps(name, value) VALUES('http','1970-01-01'); -INSERT INTO timestamps(name, value) VALUES('ftp','1970-01-01'); -INSERT INTO timestamps(name, value) VALUES('browse_tally','1970-01-01'); +INSERT INTO timestamps(name, value) VALUES('http','1970-01-01 00:00:00'); +INSERT INTO timestamps(name, value) VALUES('ftp','1970-01-01 00:00:00'); +INSERT INTO timestamps(name, value) VALUES('browse_tally','1970-01-01 00:00:00'); -- Table structure for table: timestamps -- Note: stamp_name is ftp, http Modified: trunk/pypi/store.py ============================================================================== --- trunk/pypi/store.py (original) +++ trunk/pypi/store.py Thu Jul 22 23:47:37 2010 @@ -1,7 +1,16 @@ ''' Implements a store of disutils PKG-INFO entries, keyed off name, version. ''' -import sys, os, re, psycopg2, time, hashlib, random, types, math, stat, errno +import sys, os, re, time, hashlib, random, types, math, stat, errno import logging, cStringIO, string, datetime, calendar, binascii, urllib2, cgi +try: + import psycopg2 +except ImportError: + pass +try: + import sqlite3 + sqlite3_cursor = sqlite3.Cursor +except ImportError: + sqlite3_cursor = type(None) from xml.parsers import expat from distutils.version import LooseVersion import trove, openid2rp @@ -156,6 +165,9 @@ if params is None: return cursor.execute(sql) + if isinstance(cursor, sqlite3_cursor): + sql = sql.replace('%s', "?") + # Encode every incoming param to UTF-8 if it's a string safe_params = [] for param in params: @@ -181,6 +193,12 @@ self._conn = None self._cursor = None self._trove = None + if self.config.database_driver == 'sqlite3': + self.true, self.false = '1', '0' + self.can_lock = False + else: + self.true, self.false = 'TRUE', 'FALSE' + self.can_lock = True def trove(self): if not self._trove: @@ -339,7 +357,7 @@ # hide all other releases of this package if thus configured if self.get_package_autohide(name): safe_execute(cursor, 'update releases set _pypi_hidden=%s where ' - 'name=%s and version <> %s', ('TRUE', name, version)) + 'name=%s and version <> %s', (self.true, name, version)) # add description urls if html: @@ -594,8 +612,8 @@ where = ' %s '%operator.join(where) if '_pypi_hidden' in spec: - if spec['_pypi_hidden'] in ('1', 1): v = 'TRUE' - else: v = 'FALSE' + if spec['_pypi_hidden'] in ('1', 1): v = self.true + else: v = self.false if where: where += ' AND _pypi_hidden = %s'%v else: @@ -762,7 +780,7 @@ where j.version is not NULL and j.action = 'new release' and j.name = r.name and j.version = r.version - and r._pypi_hidden = FALSE + and r._pypi_hidden = '''+self.false+''' and j.submitted_date > %s order by submitted_date desc ''', (time.strftime('%Y-%m-%d %H:%M:%S +0000', time.gmtime(since)),)) @@ -833,7 +851,7 @@ from journals j, releases r where j.version is not NULL and j.name = r.name and j.version = r.version - and r._pypi_hidden = FALSE + and r._pypi_hidden = '''+self.false+''' order by submitted_date desc ''') @@ -1026,13 +1044,13 @@ '''Add a user rating of a release; message is optional''' cursor = self.get_cursor() safe_execute(cursor, '''insert into ratings (name, version, user_name, date, rating) - values(%s, %s, %s, now(), %s)''', (name, version, self.username, rating)) + values(%s, %s, %s, current_timestamp, %s)''', (name, version, self.username, rating)) if message: safe_execute(cursor, '''insert into comments(rating, user_name, date, message, in_reply_to) - values(currval('ratings_id_seq'), %s, now(), %s, NULL)''', + values(currval('ratings_id_seq'), %s, current_timestamp, %s, NULL)''', (self.username, message)) safe_execute(cursor, '''insert into comments_journal(name, version, id, submitted_by, date, action) - values(%s,%s,currval('comments_id_seq'),%s,now(),%s)''', + values(%s,%s,currval('comments_id_seq'),%s,current_timestamp,%s)''', (name, version, self.username, 'add_rating %r' % message)) def copy_rating(self, name, fromversion, toversion): @@ -1040,7 +1058,7 @@ return the comment if any''' cursor = self.get_cursor() safe_execute(cursor, '''insert into ratings(name,version,user_name,date,rating) - select name,%s,user_name,now(),rating from ratings + select name,%s,user_name,current_timestamp,rating from ratings where name=%s and version=%s and user_name=%s''', (toversion, name, fromversion, self.username)) # only copy comment, not follow-ups @@ -1052,7 +1070,7 @@ select currval('ratings_id_seq'), user_name, date, message, in_reply_to from comments where id=%s''', (cid,)) safe_execute(cursor, '''insert into comments_journal(name, version, id, submitted_by, date, action) - values(%s, %s, currval('comments_id_seq'), %s, now(), %s)''', (name, toversion, + values(%s, %s, currval('comments_id_seq'), %s, current_timestamp, %s)''', (name, toversion, self.username, 'copied %s' % cid)) safe_execute(cursor, '''select message from comments @@ -1064,7 +1082,7 @@ '''Remove a rating for the current user''' cursor = self.get_cursor() safe_execute(cursor, """insert into comments_journal(name, version, id, submitted_by, date, action) - select %s, %s, id, %s, now(), 'deleted' from ratings where user_name=%s and name=%s and version=%s""", + select %s, %s, id, %s, current_timestamp, 'deleted' from ratings where user_name=%s and name=%s and version=%s""", (name, version, self.username, self.username, name, version)) safe_execute(cursor, "delete from ratings where user_name=%s and name=%s and version=%s", (self.username, name, version)) @@ -1099,9 +1117,9 @@ safe_execute(cursor, "select c.rating, r.name, r.version from comments c, ratings r where c.id=%s and c.rating=r.id", (msg,)) rating, name, version = cursor.fetchone() safe_execute(cursor, '''insert into comments(rating, user_name, date, message, in_reply_to) - values(%s,%s,now(),%s,%s)''', (rating, self.username, comment, msg)) + values(%s,%s,current_timestamp,%s,%s)''', (rating, self.username, comment, msg)) safe_execute(cursor, '''insert into comments_journal(name, version, id, submitted_by, date, action) - values(%s,%s,currval('comments_id_seq'),%s,now(),%s)''', (name, version, self.username, + values(%s,%s,currval('comments_id_seq'),%s,current_timestamp,%s)''', (name, version, self.username, 'add %s %r' % (msg, comment))) return name, version @@ -1112,7 +1130,7 @@ name, version = cursor.fetchone() safe_execute(cursor, "delete from comments where id=%s", (msg,)) safe_execute(cursor, '''insert into comments_journal(name, version, id, submitted_by, date, action) - values(%s, %s, %s, %s, now(), 'delete')''', (name, version, msg, self.username)) + values(%s, %s, %s, %s, current_timestamp, 'delete')''', (name, version, msg, self.username)) def has_package_comments(self, name): "Return true if the package has any comments" @@ -1283,7 +1301,7 @@ (name, )) return int(cursor.fetchone()[0]) - def store_user(self, name, password, email, gpg_keyid, otk=True): + def store_user(self, name, password, email, gpg_keyid="", otk=True): ''' Store info about the user to the database. The "password" argument is passed in cleartext and sha-ed @@ -1323,7 +1341,7 @@ if not otk: return None otk = ''.join([random.choice(chars) for x in range(32)]) - safe_execute(cursor, 'insert into rego_otk (name, otk, date) values (%s, %s, now())', + safe_execute(cursor, 'insert into rego_otk (name, otk, date) values (%s, %s, current_timestamp)', (name, otk)) return otk @@ -1508,7 +1526,8 @@ # Regenerate tally. First, release locks we hold on the timestamps self._conn.commit() # Clear old tally - cursor.execute("lock table browse_tally") + if self.can_lock: + cursor.execute("lock table browse_tally") cursor.execute("delete from browse_tally") # Regenerate tally; see browse() below cursor.execute("""insert into browse_tally @@ -1516,7 +1535,7 @@ from trove_classifiers t, release_classifiers rc, releases r where rc.name=r.name and rc.version=r.version and not r._pypi_hidden and rc.trove_id=t.id group by t.l2, rc.name, rc.version) res group by res.l2""") - cursor.execute("update timestamps set value=now() where name='browse_tally'") + cursor.execute("update timestamps set value=current_timestamp where name='browse_tally'") self._conn.commit() cursor.execute("select trove_id, tally from browse_tally") return [], cursor.fetchall() @@ -1534,7 +1553,7 @@ return [], cursor.fetchall() # First compute statement to produce all packages still selected - pkgs = "select name, version, summary from releases where _pypi_hidden=FALSE" + pkgs = "select name, version, summary from releases where _pypi_hidden="+self.false for c in selected_classifiers: level = t.trove[c].level pkgs = """select distinct a.name, a.version, summary from (%s) a, release_classifiers rc, trove_classifiers t @@ -1586,7 +1605,7 @@ cursor = self.get_cursor() sql = '''insert into release_files (name, version, python_version, packagetype, comment_text, filename, md5_digest, upload_time) values - (%s, %s, %s, %s, %s, %s, %s, now())''' + (%s, %s, %s, %s, %s, %s, %s, current_timestamp)''' safe_execute(cursor, sql, (name, version, pyversion, filetype, comment, filename, md5_digest)) @@ -1764,9 +1783,9 @@ name, last_seen = users[0] if datetime.datetime.now()-datetime.timedelta(0,60) > last_seen: # refresh cookie and login time every minute - sql = 'update cookies set last_seen=now() where cookie=%s' + sql = 'update cookies set last_seen=current_timestamp where cookie=%s' safe_execute(cursor, sql, (cookie,)) - sql ='update users set last_login=now() where name=%s' + sql ='update users set last_login=current_timestamp where name=%s' safe_execute(cursor, sql, (name,)) return name return None @@ -1776,7 +1795,7 @@ cursor = self.get_cursor() cookie = binascii.hexlify(os.urandom(16)) sql = '''insert into cookies(cookie, name, last_seen) - values(%s, %s, now())''' + values(%s, %s, current_timestamp)''' safe_execute(cursor, sql, (cookie, username)) return cookie @@ -1790,7 +1809,7 @@ cursor = self.get_cursor() # Check for existing session sql = '''select id,url, assoc_handle from openid_sessions - where provider=%s and expires>now()''' + where provider=%s and expires>current_timestamp''' safe_execute(cursor, sql, (provider[0],)) sessions = cursor.fetchall() if sessions: @@ -1827,7 +1846,7 @@ cursor = self.get_cursor() # Check for existing session sql = '''select assoc_handle from openid_sessions - where provider=%s and url=%s and expires>now()''' + where provider=%s and url=%s and expires>current_timestamp''' safe_execute(cursor, sql, (claimed, endpoint,)) sessions = cursor.fetchall() if sessions: @@ -1915,6 +1934,9 @@ # already closed connection = None return self.open() + elif self.config.database_driver == 'sqlite3': + self._conn = connection = sqlite3.connect(self.config.database_name, + detect_types=sqlite3.PARSE_DECLTYPES) else: self._conn = connection = psycopg2.connect(**cd) @@ -1938,8 +1960,8 @@ if self.has_user(username): self.username = username if update_last_login: - self.get_cursor().execute(''' - update users set last_login=now() where name=%s''', (username,)) + safe_execute(self.get_cursor(), ''' + update users set last_login=current_timestamp where name=%s''', (username,)) self.userip = userip def setpasswd(self, username, password): Added: trunk/pypi/tools/demodata ============================================================================== --- (empty file) +++ trunk/pypi/tools/demodata Thu Jul 22 23:47:37 2010 @@ -0,0 +1,54 @@ +#!/usr/bin/python +import sys, os, urllib + +root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.append(root) +import admin, store, config + +cfg = config.Config(root+'/config.ini') +st = store.Store(cfg) + +# classifiers +for c in urllib.urlopen("http://pypi.python.org/pypi?%3Aaction=list_classifiers").read().splitlines(): + admin.add_classifier(st, c) + +# Demo data starts here + +# an admin +otk = st.store_user('fred', 'fredpw', 'fred at python.test') +st.delete_otk(otk) +st.add_role('fred', 'Admin', None) +# an owner +otk = st.store_user('barney', 'barneypw', 'barney at python.test') +st.delete_otk(otk) + +# package spam +st.set_user('barney', '127.0.0.1', True) +for version in ('0.8', '0.9', '1.0'): + st.store_package('spam', version, { + 'author':'Barney Geroellheimer', + 'author_email':'barney at python.test', + 'homepage':'http://spam.python.test/', + 'license':'GPL', + 'summary':'The spam package', + 'description':'Does anybody want to provide real data here?', + 'classifiers':["Development Status :: 6 - Mature", + "Programming Language :: Python :: 2"], + '_pypi_hidden':False + }) + +# package eggs +for version in ('0.1', '0.2', '0.3', '0.4'): + st.store_package('eggs', version, { + 'author':'Barney Geroellheimer', + 'author_email':'barney at python.test', + 'homepage':'http://eggs.python.test/', + 'license':'GPL', + 'summary':'The eggs package', + 'description':'Does anybody want to provide real data here?', + 'classifiers':["Development Status :: 3 - Alpha", + "Programming Language :: Python :: 3"], + '_pypi_hidden':version!='0.4' + }) + +st.commit() Added: trunk/pypi/tools/mksqlite ============================================================================== --- (empty file) +++ trunk/pypi/tools/mksqlite Thu Jul 22 23:47:37 2010 @@ -0,0 +1,26 @@ +#!/usr/bin/python +import os +dbpath = "packages.db" + +if os.path.exists(dbpath): + print "Remove",dbpath,"first" + raise SystemExit + +print "Creating database", dbpath +sqlite = os.popen('sqlite3 '+dbpath, "w") +passthrough = True +for line in open('pkgbase_schema.sql'): + if 'nosqlite-end' in line: + # end of disabled block + passthrough = True + print >>sqlite + continue + if 'nosqlite' in line: + passthrough = False + print >>sqlite + continue + if not passthrough: + print >> sqlite + continue + sqlite.write(line) +sqlite.close() Modified: trunk/pypi/webui.py ============================================================================== --- trunk/pypi/webui.py (original) +++ trunk/pypi/webui.py Thu Jul 22 23:47:37 2010 @@ -6,7 +6,12 @@ from distutils.util import rfc822_escape from distutils2.metadata import DistributionMetadata -import psycopg2 +try: + import psycopg2 + OperationalError = psycopg2.OperationalError +except ImportError: + class OperationalError(Exception): + pass try: import cElementTree @@ -259,6 +264,7 @@ self.store = store.Store(self.config) try: try: + self.store.get_cursor() # make sure we can connect self.inner_run() except NotFound: self.fail('Not Found', code=404) @@ -288,7 +294,7 @@ except IOError, error: # ignore broken pipe errors (client vanished on us) if error.errno != 32: raise - except psycopg2.OperationalError, message: + except OperationalError, message: # clean things up self.store.force_close() message = str(message) From python-checkins at python.org Fri Jul 23 00:15:18 2010 From: python-checkins at python.org (martin.von.loewis) Date: Fri, 23 Jul 2010 00:15:18 +0200 (CEST) Subject: [Pypi-checkins] r778 - trunk/pypi Message-ID: <20100722221518.916EBF4CE@mail.python.org> Author: martin.von.loewis Date: Fri Jul 23 00:15:18 2010 New Revision: 778 Modified: trunk/pypi/openid2rp.py Log: Incorporate bug fixes from upstream. Modified: trunk/pypi/openid2rp.py ============================================================================== --- trunk/pypi/openid2rp.py (original) +++ trunk/pypi/openid2rp.py Fri Jul 23 00:15:18 2010 @@ -151,7 +151,7 @@ def do_meta(self, attrs): attrs = dict(attrs) # Yadis 6.2.5 option 1: meta tag - if attrs['http-equiv'].lower() == 'x-xrds-location': + if attrs.get('http-equiv','').lower() == 'x-xrds-location': self.xrds_location = attrs['content'] def discover(url): @@ -340,6 +340,9 @@ if 'error' in data: raise ValueError, "associate failed: "+data['error'] if url.startswith('http:'): + enc_mac_key = data.get('enc_mac_key') + if not enc_mac_key: + raise ValueError, "Provider protocol error: not using DH-SHA1" enc_mac_key = base64.b64decode(data['enc_mac_key']) dh_server_public = unbtwoc(base64.b64decode(data['dh_server_public'])) # shared secret: sha1(2^(server_priv*priv) mod prime) xor enc_mac_key From python-checkins at python.org Fri Jul 23 17:49:40 2010 From: python-checkins at python.org (georg.brandl) Date: Fri, 23 Jul 2010 17:49:40 +0200 (CEST) Subject: [Pypi-checkins] r779 - trunk/pypi/tools Message-ID: <20100723154940.84A5EEE9C4@mail.python.org> Author: georg.brandl Date: Fri Jul 23 17:49:40 2010 New Revision: 779 Modified: trunk/pypi/tools/demodata (props changed) Log: Make script executable. From python-checkins at python.org Fri Jul 23 17:59:56 2010 From: python-checkins at python.org (georg.brandl) Date: Fri, 23 Jul 2010 17:59:56 +0200 (CEST) Subject: [Pypi-checkins] r780 - trunk/pypi Message-ID: <20100723155956.75988EE9C4@mail.python.org> Author: georg.brandl Date: Fri Jul 23 17:59:56 2010 New Revision: 780 Modified: trunk/pypi/pkgbase_schema.sql Log: Remove unused sequences; use SERIAL PRIMARY KEY consistently to make life easier for SQLite. Modified: trunk/pypi/pkgbase_schema.sql ============================================================================== --- trunk/pypi/pkgbase_schema.sql (original) +++ trunk/pypi/pkgbase_schema.sql Fri Jul 23 17:59:56 2010 @@ -90,10 +90,9 @@ ); CREATE TABLE cheesecake_main_indices ( - id SERIAL, + id SERIAL PRIMARY KEY, absolute INTEGER NOT NULL, - relative INTEGER NOT NULL, - PRIMARY KEY (id) + relative INTEGER NOT NULL ); CREATE TABLE cheesecake_subindices ( @@ -146,15 +145,6 @@ CREATE INDEX trove_class_id_idx ON trove_classifiers(id); --- trove ids sequence --- nosqlite -CREATE TABLE dual (dummy INTEGER); -INSERT INTO dual VALUES (1); -CREATE SEQUENCE trove_ids; -SELECT setval('trove_ids', 1000) FROM dual; --- nosqlite-end - - -- Table structure for table: release_classifiers CREATE TABLE release_classifiers ( name TEXT, @@ -335,13 +325,12 @@ -- ratings CREATE TABLE ratings( - id SERIAL UNIQUE, + id SERIAL PRIMARY KEY, name TEXT, version TEXT, user_name TEXT REFERENCES users ON DELETE CASCADE, date TIMESTAMP, rating INTEGER, - PRIMARY KEY (name, version, user_name), FOREIGN KEY (name, version) REFERENCES releases ON UPDATE CASCADE ON DELETE CASCADE ); CREATE INDEX rating_name_version ON ratings(name, version); From python-checkins at python.org Fri Jul 23 18:10:51 2010 From: python-checkins at python.org (georg.brandl) Date: Fri, 23 Jul 2010 18:10:51 +0200 (CEST) Subject: [Pypi-checkins] r781 - trunk/pypi Message-ID: <20100723161051.4B44FEEA25@mail.python.org> Author: georg.brandl Date: Fri Jul 23 18:10:51 2010 New Revision: 781 Modified: trunk/pypi/store.py Log: Introduce a database-specific way to get the last inserted rowid. Modified: trunk/pypi/store.py ============================================================================== --- trunk/pypi/store.py (original) +++ trunk/pypi/store.py Fri Jul 23 18:10:51 2010 @@ -200,6 +200,15 @@ self.true, self.false = 'TRUE', 'FALSE' self.can_lock = True + def last_id(self, tablename): + ''' Return an SQL expression that returns the last inserted row, + where the row is in the given table. + ''' + if self.config.database_driver == 'sqlite3': + return 'last_insert_rowid()' + else: + return "currval('%s_id_seq')" % tablename + def trove(self): if not self._trove: self._trove = trove.Trove(self.get_cursor()) @@ -1047,10 +1056,12 @@ values(%s, %s, %s, current_timestamp, %s)''', (name, version, self.username, rating)) if message: safe_execute(cursor, '''insert into comments(rating, user_name, date, message, in_reply_to) - values(currval('ratings_id_seq'), %s, current_timestamp, %s, NULL)''', + values(%s, %%s, current_timestamp, %%s, NULL)''' + % self.last_id('ratings'), (self.username, message)) safe_execute(cursor, '''insert into comments_journal(name, version, id, submitted_by, date, action) - values(%s,%s,currval('comments_id_seq'),%s,current_timestamp,%s)''', + values(%%s, %%s, %s, %%s, current_timestamp, %%s)''' + % self.last_id('comments'), (name, version, self.username, 'add_rating %r' % message)) def copy_rating(self, name, fromversion, toversion): @@ -1067,11 +1078,11 @@ if cid: cid = cid[0] safe_execute(cursor, '''insert into comments(rating, user_name, date, message, in_reply_to) - select currval('ratings_id_seq'), user_name, date, message, in_reply_to - from comments where id=%s''', (cid,)) + select %s, user_name, date, message, in_reply_to + from comments where id=%%s''' % self.last_id('ratings'), (cid,)) safe_execute(cursor, '''insert into comments_journal(name, version, id, submitted_by, date, action) - values(%s, %s, currval('comments_id_seq'), %s, current_timestamp, %s)''', (name, toversion, - self.username, 'copied %s' % cid)) + values(%%s, %%s, %s, %%s, current_timestamp, %%s)''' % self.last_id('comments'), + (name, toversion, self.username, 'copied %s' % cid)) safe_execute(cursor, '''select message from comments where id=%s''', (cid,)) @@ -1119,8 +1130,8 @@ safe_execute(cursor, '''insert into comments(rating, user_name, date, message, in_reply_to) values(%s,%s,current_timestamp,%s,%s)''', (rating, self.username, comment, msg)) safe_execute(cursor, '''insert into comments_journal(name, version, id, submitted_by, date, action) - values(%s,%s,currval('comments_id_seq'),%s,current_timestamp,%s)''', (name, version, self.username, - 'add %s %r' % (msg, comment))) + values(%%s, %%s, %s, %%s, current_timestamp, %%s)''' % self.last_id('comments'), + (name, version, self.username, 'add %s %r' % (msg, comment))) return name, version def remove_comment(self, msg): @@ -1833,10 +1844,11 @@ session['assoc_handle'], now+datetime.timedelta(0,int(session['expires_in'])), session['mac_key'])) + safe_execute(cursor, 'select %s' % self.last_id('openid_sessions')) + session_id = cursor.fetchone()[0] for t in stypes: safe_execute(cursor, '''insert into openid_stypes(id, stype) - values(currval('openid_sessions_id_seq'),%s)''', - (t,)) + values(%s, %s)''', (session_id, t)) return stypes, url, session['assoc_handle'] def get_session_for_endpoint(self, claimed, stypes, endpoint): @@ -1863,11 +1875,12 @@ session['assoc_handle'], now+datetime.timedelta(0,int(session['expires_in'])), session['mac_key'])) + safe_execute(cursor, 'select %s' % self.last_id('openid_sessions')) + session_id = cursor.fetchone()[0] # store stypes as well, so we can remember whether claimed is an OP ID or a user ID for t in stypes: safe_execute(cursor, '''insert into openid_stypes(id, stype) - values(currval('openid_sessions_id_seq'),%s)''', - (t,)) + values(%s, %s)''', (session_id, t)) return session['assoc_handle'] def get_session_by_handle(self, assoc_handle): From python-checkins at python.org Fri Jul 23 18:11:10 2010 From: python-checkins at python.org (georg.brandl) Date: Fri, 23 Jul 2010 18:11:10 +0200 (CEST) Subject: [Pypi-checkins] r782 - trunk/pypi/tools Message-ID: <20100723161110.969A3EEA08@mail.python.org> Author: georg.brandl Date: Fri Jul 23 18:11:10 2010 New Revision: 782 Modified: trunk/pypi/tools/mksqlite Log: SQLite unfortunately does not know about SERIAL. Modified: trunk/pypi/tools/mksqlite ============================================================================== --- trunk/pypi/tools/mksqlite (original) +++ trunk/pypi/tools/mksqlite Fri Jul 23 18:11:10 2010 @@ -22,5 +22,6 @@ if not passthrough: print >> sqlite continue - sqlite.write(line) + # make sqlite happy: SERIAL is not a valid type + sqlite.write(line.replace('SERIAL PRIMARY KEY', 'INTEGER PRIMARY KEY')) sqlite.close() From python-checkins at python.org Fri Jul 23 18:12:01 2010 From: python-checkins at python.org (georg.brandl) Date: Fri, 23 Jul 2010 18:12:01 +0200 (CEST) Subject: [Pypi-checkins] r783 - trunk/pypi/templates Message-ID: <20100723161201.A5076EE9F8@mail.python.org> Author: georg.brandl Date: Fri Jul 23 18:12:01 2010 New Revision: 783 Modified: trunk/pypi/templates/display.pt Log: Vote -> rating for consistency. Modified: trunk/pypi/templates/display.pt ============================================================================== --- trunk/pypi/templates/display.pt (original) +++ trunk/pypi/templates/display.pt Fri Jul 23 18:12:01 2010 @@ -238,7 +238,7 @@ - +
@@ -262,14 +262,14 @@ - +
- +
From python-checkins at python.org Fri Jul 23 18:12:28 2010 From: python-checkins at python.org (georg.brandl) Date: Fri, 23 Jul 2010 18:12:28 +0200 (CEST) Subject: [Pypi-checkins] r784 - trunk/pypi/templates Message-ID: <20100723161228.6852FEEA1D@mail.python.org> Author: georg.brandl Date: Fri Jul 23 18:12:28 2010 New Revision: 784 Modified: trunk/pypi/templates/register.pt Log: Give the user registering form an explicit action="/pypi". Modified: trunk/pypi/templates/register.pt ============================================================================== --- trunk/pypi/templates/register.pt (original) +++ trunk/pypi/templates/register.pt Fri Jul 23 18:12:28 2010 @@ -5,7 +5,7 @@ metal:use-macro="standard_template/macros/page"> -
+ From python-checkins at python.org Fri Jul 23 18:15:15 2010 From: python-checkins at python.org (georg.brandl) Date: Fri, 23 Jul 2010 18:15:15 +0200 (CEST) Subject: [Pypi-checkins] r785 - trunk/pypi Message-ID: <20100723161515.5D024EEA07@mail.python.org> Author: georg.brandl Date: Fri Jul 23 18:15:15 2010 New Revision: 785 Modified: trunk/pypi/webui.py Log: In the "registered ok" message, indicate that the user has to visit the confirm link. Modified: trunk/pypi/webui.py ============================================================================== --- trunk/pypi/webui.py (original) +++ trunk/pypi/webui.py Fri Jul 23 18:15:15 2010 @@ -2478,7 +2478,9 @@ info['admin'] = self.config.adminemail self.send_email(info['email'], rego_message%info) response = 'Registration OK' - message = 'You should receive a confirmation email shortly.' + message = ('You should receive a confirmation email to %s shortly. ' + 'To complete the registration process, visit the link ' + 'indicated in the email.') % info['email'] else: # update details From python-checkins at python.org Fri Jul 23 18:22:59 2010 From: python-checkins at python.org (martin.von.loewis) Date: Fri, 23 Jul 2010 18:22:59 +0200 (CEST) Subject: [Pypi-checkins] r786 - trunk/pypi/tools Message-ID: <20100723162259.47A14EE988@mail.python.org> Author: martin.von.loewis Date: Fri Jul 23 18:22:59 2010 New Revision: 786 Modified: trunk/pypi/tools/daily.sql Log: Delete openids of expired users. Modified: trunk/pypi/tools/daily.sql ============================================================================== --- trunk/pypi/tools/daily.sql (original) +++ trunk/pypi/tools/daily.sql Fri Jul 23 18:22:59 2010 @@ -2,4 +2,5 @@ delete from cookies where last_seen < now()-INTERVAL'1day'; delete from openid_sessions where expires < now(); delete from openid_nonces where created < now()-INTERVAL'1day'; -delete from users where name in (select name from rego_otk where date < now()-INTERVAL'7days'); \ No newline at end of file +delete from openids where name in (select name from rego_otk where date < now()-INTERVAL'7days'); +delete from users where name in (select name from rego_otk where date < now()-INTERVAL'7days'); From python-checkins at python.org Fri Jul 23 18:31:57 2010 From: python-checkins at python.org (georg.brandl) Date: Fri, 23 Jul 2010 18:31:57 +0200 (CEST) Subject: [Pypi-checkins] r787 - trunk/pypi Message-ID: <20100723163157.309EFF46E@mail.python.org> Author: georg.brandl Date: Fri Jul 23 18:31:56 2010 New Revision: 787 Modified: trunk/pypi/webui.py Log: Notify the user of the rating given together with the comment; also fix a traceback when the comment form is empty. Modified: trunk/pypi/webui.py ============================================================================== --- trunk/pypi/webui.py (original) +++ trunk/pypi/webui.py Fri Jul 23 18:31:56 2010 @@ -98,7 +98,20 @@ Reply-To: %(replyto)s [REPLIES TO THIS MESSAGE WILL NOT GO TO THE COMMENTER] -%(author)s has made the following comment on your package. +%(author)s has made the following comment on your package: + +%(comment)s + +You can read all comments on %(url)s. +''' + +rating_message = '''Subject: New rating on %(package)s +From: PyPI operators <%(admin)s> +To: %(email)s +Reply-To: %(replyto)s + +[REPLIES TO THIS MESSAGE WILL NOT GO TO THE COMMENTER] +%(author)s has rated your package as %(rating)s/5. Comment (optional): %(comment)s @@ -138,7 +151,7 @@ else: PyPiPageTemplate = _PyPiPageTemplate -def comment_email(store, package, version, author, comment, add_recipients): +def comment_email(store, package, version, author, comment, rating, add_recipients): emails = set() recipients = [r['user_name'] for r in store.get_package_roles(package)] + add_recipients for r in recipients: @@ -153,9 +166,14 @@ 'email': ','.join(emails), 'comment': comment, 'url': '%s/%s/%s' % (store.config.url, package, version), + 'rating': rating, } smtp = smtplib.SMTP(store.config.mailhost) - smtp.sendmail(store.config.adminemail, list(emails), comment_message % info) + if rating is None: + message = comment_message % info + else: + message = rating_message % info + smtp.sendmail(store.config.adminemail, list(emails), message) class FileUpload: @@ -266,8 +284,8 @@ try: self.store.get_cursor() # make sure we can connect self.inner_run() - except NotFound: - self.fail('Not Found', code=404) + except NotFound, err: + self.fail('Not Found (%s)' % err, code=404) except Unauthorised, message: message = str(message) if not message: @@ -516,10 +534,15 @@ raise Unauthorised, "Incomplete registration; check your email" # handle the action - if action in 'debug home browse rss index search submit doap display_pkginfo submit_pkg_info remove_pkg pkg_edit verify submit_form display register_form user_form forgotten_password_form user password_reset role role_form list_classifiers login logout files file_upload show_md5 doc_upload claim openid openid_return dropid rate comment addcomment delcomment clear_auth addkey delkey lasthour'.split(): + if action in '''debug home browse rss index search submit doap + display_pkginfo submit_pkg_info remove_pkg pkg_edit verify submit_form + display register_form user_form forgotten_password_form user + password_reset role role_form list_classifiers login logout files + file_upload show_md5 doc_upload claim openid openid_return dropid + rate comment addcomment delcomment clear_auth addkey delkey lasthour'''.split(): getattr(self, action)() else: - #raise NotFound, 'Unknown action' + #raise NotFound, 'Unknown action %s' % action raise NotFound if action in 'submit submit_pkg_info pkg_edit remove_pkg'.split(): @@ -1276,8 +1299,8 @@ rating = ', %s points' % rating else: rating = '' - result.append("
  • %s (%s%s):
    %s %s" % - (c['user'], date, rating, message, reply)) + result.append("
  • %s (%s%s):
    %s %s" % + (c['user'], date, rating, message, reply)) if children: result.extend(render_comments(children, False)) result.append("
    • \n") @@ -1901,18 +1924,21 @@ raise FormError, "fromversion missing" comment = self.store.copy_rating(name, self.form['fromversion'], version) if comment: - comment_email(self.store, name, version, self.username, comment, []) + comment_email(self.store, name, version, self.username, comment, + None, []) return self.display() if self.form.has_key('rate'): if self.store.has_rating(name, version): raise Forbidden, "You have already rated this release" if not self.form.has_key('rating'): raise FormError, "rating not provided" - message = self.form['comment'].strip() + message = self.form.get('comment', '').strip() if message and not self.store.get_package_comments(name): raise FormError, "package does not allow comments" - self.store.add_rating(name, version, self.form['rating'], message) - comment_email(self.store, name, version, self.username, message, []) + rating = self.form['rating'] + self.store.add_rating(name, version, rating, message) + comment_email(self.store, name, version, self.username, message, + rating, []) return self.display() raise FormError, "Bad button" @@ -1943,7 +1969,8 @@ raise FormError, "You must fill in a comment" name, version = self.store.add_comment(msg, comment) - comment_email(self.store, name, version, self.username, comment, [orig['user']]) + comment_email(self.store, name, version, self.username, comment, + None, [orig['user']]) return self.display(name=name, version=version) From python-checkins at python.org Fri Jul 23 18:50:07 2010 From: python-checkins at python.org (georg.brandl) Date: Fri, 23 Jul 2010 18:50:07 +0200 (CEST) Subject: [Pypi-checkins] r788 - trunk/pypi Message-ID: <20100723165007.B00AEEE9D8@mail.python.org> Author: georg.brandl Date: Fri Jul 23 18:50:07 2010 New Revision: 788 Modified: trunk/pypi/webui.py Log: Remove wrap="hard" from textareas to prevent the browser from destroying valid reStructuredText. Modified: trunk/pypi/webui.py ============================================================================== --- trunk/pypi/webui.py (original) +++ trunk/pypi/webui.py Fri Jul 23 18:50:07 2010 @@ -1505,12 +1505,12 @@ '''%(a,b) elif property in ('license', 'platform'): - field = '''
    You should enter a full description here only if appropriate classifiers aren\'t available (see below).'''%(property, cgi.escape(value)) elif property.endswith('description'): - field = '''
    You may use ReStructuredText formatting for this field.'''%(property, From python-checkins at python.org Sat Jul 24 10:15:02 2010 From: python-checkins at python.org (martin.von.loewis) Date: Sat, 24 Jul 2010 10:15:02 +0200 (CEST) Subject: [Pypi-checkins] r789 - trunk/pypi Message-ID: <20100724081502.53459EE990@mail.python.org> Author: martin.von.loewis Date: Sat Jul 24 10:15:02 2010 New Revision: 789 Modified: trunk/pypi/README trunk/pypi/webui.py Log: Restore dependency to M2Crypto. Modified: trunk/pypi/README ============================================================================== --- trunk/pypi/README (original) +++ trunk/pypi/README Sat Jul 24 10:15:02 2010 @@ -12,6 +12,7 @@ - zope.i18nmessageid - psycopg2 (for testing, sqlite3 might be sufficient) - docutils +- M2Crypto - distutils2 Quick development setup @@ -32,7 +33,7 @@ $ virtualenv --no-site-packages --distribute . $ bin/easy_install cElementTree zope.interface zope.pagetemplate $ bin/easy_install zope.tal zope.tales zope.i18nmessageid psycopg2 - $ bin/easy_install docutils + $ bin/easy_install docutils M2Crypto Then you can launch the server using the pypi.wsgi script:: Modified: trunk/pypi/webui.py ============================================================================== --- trunk/pypi/webui.py (original) +++ trunk/pypi/webui.py Sat Jul 24 10:15:02 2010 @@ -18,6 +18,11 @@ except ImportError: from xml.etree import cElementTree +# Importing M2Crypto patches urllib; don't let them do that +orig = urllib.URLopener.open_https.im_func +from M2Crypto import EVP, DSA +urllib.URLopener.open_https = orig + # local imports import store, config, versionpredicate, verify_filetype, rpc import MailingLogger, openid2rp From python-checkins at python.org Sat Jul 24 12:53:18 2010 From: python-checkins at python.org (richard) Date: Sat, 24 Jul 2010 12:53:18 +0200 (CEST) Subject: [Pypi-checkins] r790 - trunk/pypi/tools Message-ID: <20100724105318.7129CCA5E@mail.python.org> Author: richard Date: Sat Jul 24 12:53:18 2010 New Revision: 790 Added: trunk/pypi/tools/demodata.py - copied, changed from r789, trunk/pypi/tools/demodata Log: rename for convenience Copied: trunk/pypi/tools/demodata.py (from r789, trunk/pypi/tools/demodata) ============================================================================== --- trunk/pypi/tools/demodata (original) +++ trunk/pypi/tools/demodata.py Sat Jul 24 12:53:18 2010 @@ -50,5 +50,5 @@ "Programming Language :: Python :: 3"], '_pypi_hidden':version!='0.4' }) - + st.commit() From python-checkins at python.org Sat Jul 24 12:55:59 2010 From: python-checkins at python.org (richard) Date: Sat, 24 Jul 2010 12:55:59 +0200 (CEST) Subject: [Pypi-checkins] r791 - trunk/pypi/tools Message-ID: <20100724105559.92B91EE982@mail.python.org> Author: richard Date: Sat Jul 24 12:55:59 2010 New Revision: 791 Added: trunk/pypi/tools/mksqlite.py - copied unchanged from r789, trunk/pypi/tools/mksqlite Log: rename for convenience From python-checkins at python.org Sat Jul 24 12:58:20 2010 From: python-checkins at python.org (martin.von.loewis) Date: Sat, 24 Jul 2010 12:58:20 +0200 (CEST) Subject: [Pypi-checkins] r792 - in trunk/pypi: . tools Message-ID: <20100724105820.44FA5F303@mail.python.org> Author: martin.von.loewis Date: Sat Jul 24 12:58:20 2010 New Revision: 792 Added: trunk/pypi/tools/sql-migrate-20100724.sql Modified: trunk/pypi/pkgbase_schema.sql Log: Follow pkgbase change to change ratings primary key. Restore unique constraint. Modified: trunk/pypi/pkgbase_schema.sql ============================================================================== --- trunk/pypi/pkgbase_schema.sql (original) +++ trunk/pypi/pkgbase_schema.sql Sat Jul 24 12:58:20 2010 @@ -331,6 +331,7 @@ user_name TEXT REFERENCES users ON DELETE CASCADE, date TIMESTAMP, rating INTEGER, + UNIQUE(name,version,user_name), FOREIGN KEY (name, version) REFERENCES releases ON UPDATE CASCADE ON DELETE CASCADE ); CREATE INDEX rating_name_version ON ratings(name, version); Added: trunk/pypi/tools/sql-migrate-20100724.sql ============================================================================== --- (empty file) +++ trunk/pypi/tools/sql-migrate-20100724.sql Sat Jul 24 12:58:20 2010 @@ -0,0 +1,5 @@ +begin; +alter table ratings drop constraint ratings_pkey; +alter table ratings add primary key (id); +alter table ratings add unique(name,version,user_name); +end; \ No newline at end of file From python-checkins at python.org Sat Jul 24 13:00:37 2010 From: python-checkins at python.org (richard) Date: Sat, 24 Jul 2010 13:00:37 +0200 (CEST) Subject: [Pypi-checkins] r793 - trunk/pypi/tools Message-ID: <20100724110037.F1B16EE982@mail.python.org> Author: richard Date: Sat Jul 24 13:00:37 2010 New Revision: 793 Removed: trunk/pypi/tools/mksqlite Log: rename for convenience Deleted: /trunk/pypi/tools/mksqlite ============================================================================== --- /trunk/pypi/tools/mksqlite Sat Jul 24 13:00:37 2010 +++ (empty file) @@ -1,27 +0,0 @@ -#!/usr/bin/python -import os -dbpath = "packages.db" - -if os.path.exists(dbpath): - print "Remove",dbpath,"first" - raise SystemExit - -print "Creating database", dbpath -sqlite = os.popen('sqlite3 '+dbpath, "w") -passthrough = True -for line in open('pkgbase_schema.sql'): - if 'nosqlite-end' in line: - # end of disabled block - passthrough = True - print >>sqlite - continue - if 'nosqlite' in line: - passthrough = False - print >>sqlite - continue - if not passthrough: - print >> sqlite - continue - # make sqlite happy: SERIAL is not a valid type - sqlite.write(line.replace('SERIAL PRIMARY KEY', 'INTEGER PRIMARY KEY')) -sqlite.close() From python-checkins at python.org Sat Jul 24 13:00:44 2010 From: python-checkins at python.org (richard) Date: Sat, 24 Jul 2010 13:00:44 +0200 (CEST) Subject: [Pypi-checkins] r794 - trunk/pypi/tools Message-ID: <20100724110044.C94DDEE9BC@mail.python.org> Author: richard Date: Sat Jul 24 13:00:44 2010 New Revision: 794 Removed: trunk/pypi/tools/demodata Log: rename for convenience Deleted: /trunk/pypi/tools/demodata ============================================================================== --- /trunk/pypi/tools/demodata Sat Jul 24 13:00:44 2010 +++ (empty file) @@ -1,54 +0,0 @@ -#!/usr/bin/python -import sys, os, urllib - -root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.append(root) -import admin, store, config - -cfg = config.Config(root+'/config.ini') -st = store.Store(cfg) - -# classifiers -for c in urllib.urlopen("http://pypi.python.org/pypi?%3Aaction=list_classifiers").read().splitlines(): - admin.add_classifier(st, c) - -# Demo data starts here - -# an admin -otk = st.store_user('fred', 'fredpw', 'fred at python.test') -st.delete_otk(otk) -st.add_role('fred', 'Admin', None) -# an owner -otk = st.store_user('barney', 'barneypw', 'barney at python.test') -st.delete_otk(otk) - -# package spam -st.set_user('barney', '127.0.0.1', True) -for version in ('0.8', '0.9', '1.0'): - st.store_package('spam', version, { - 'author':'Barney Geroellheimer', - 'author_email':'barney at python.test', - 'homepage':'http://spam.python.test/', - 'license':'GPL', - 'summary':'The spam package', - 'description':'Does anybody want to provide real data here?', - 'classifiers':["Development Status :: 6 - Mature", - "Programming Language :: Python :: 2"], - '_pypi_hidden':False - }) - -# package eggs -for version in ('0.1', '0.2', '0.3', '0.4'): - st.store_package('eggs', version, { - 'author':'Barney Geroellheimer', - 'author_email':'barney at python.test', - 'homepage':'http://eggs.python.test/', - 'license':'GPL', - 'summary':'The eggs package', - 'description':'Does anybody want to provide real data here?', - 'classifiers':["Development Status :: 3 - Alpha", - "Programming Language :: Python :: 3"], - '_pypi_hidden':version!='0.4' - }) - -st.commit() From python-checkins at python.org Sat Jul 24 13:00:56 2010 From: python-checkins at python.org (richard) Date: Sat, 24 Jul 2010 13:00:56 +0200 (CEST) Subject: [Pypi-checkins] r795 - trunk/pypi Message-ID: <20100724110056.BB3F4EE99A@mail.python.org> Author: richard Date: Sat Jul 24 13:00:56 2010 New Revision: 795 Modified: trunk/pypi/config.ini.template Log: more sensible defaults Modified: trunk/pypi/config.ini.template ============================================================================== --- trunk/pypi/config.ini.template (original) +++ trunk/pypi/config.ini.template Sat Jul 24 13:00:56 2010 @@ -6,9 +6,9 @@ docs_dir = /MacDev/svn.python.org/pypi-pep345/docs [webui] -mailhost = mail.commonground.com.au -adminemail = richard at commonground.com.au -replyto = richard at commonground.com.au +mailhost = mail.python.org +adminemail = richard at python.org +replyto = richard at python.org url = http://localhost:8000/pypi pydotorg = http://www.python.org/ From python-checkins at python.org Sat Jul 24 13:01:07 2010 From: python-checkins at python.org (richard) Date: Sat, 24 Jul 2010 13:01:07 +0200 (CEST) Subject: [Pypi-checkins] r796 - trunk/pypi/tools Message-ID: <20100724110107.1075EEE99D@mail.python.org> Author: richard Date: Sat Jul 24 13:01:06 2010 New Revision: 796 Modified: trunk/pypi/tools/demodata.py Log: create a file Modified: trunk/pypi/tools/demodata.py ============================================================================== --- trunk/pypi/tools/demodata.py (original) +++ trunk/pypi/tools/demodata.py Sat Jul 24 13:01:06 2010 @@ -51,4 +51,7 @@ '_pypi_hidden':version!='0.4' }) +st.add_file('spam', '1.0', 'THIS IS SOME CONTENT', '1234', 'text/plain', + 'any', '', 'demo.txt', None) + st.commit() From python-checkins at python.org Sat Jul 24 13:01:40 2010 From: python-checkins at python.org (richard) Date: Sat, 24 Jul 2010 13:01:40 +0200 (CEST) Subject: [Pypi-checkins] r797 - trunk/pypi Message-ID: <20100724110140.BF97AEE9A9@mail.python.org> Author: richard Date: Sat Jul 24 13:01:40 2010 New Revision: 797 Modified: trunk/pypi/rpc.py Log: add some useful urls to the release data Modified: trunk/pypi/rpc.py ============================================================================== --- trunk/pypi/rpc.py (original) +++ trunk/pypi/rpc.py Sat Jul 24 13:01:40 2010 @@ -85,6 +85,9 @@ classifiers = [r[0] for r in store.get_release_classifiers(package_name, version)] info['classifiers' ] = classifiers + info['package_url'] = 'http://pypi.python.org/pypi/%s' % package_name + info['release_url'] = 'http://pypi.python.org/pypi/%s/%s' % (package_name, + version) return info package_data = release_data # "deprecated" From python-checkins at python.org Sat Jul 24 13:03:10 2010 From: python-checkins at python.org (richard) Date: Sat, 24 Jul 2010 13:03:10 +0200 (CEST) Subject: [Pypi-checkins] r798 - trunk/pypi Message-ID: <20100724110310.72CB4EE9C9@mail.python.org> Author: richard Date: Sat Jul 24 13:03:10 2010 New Revision: 798 Modified: trunk/pypi/webui.py Log: refactor to allow more methods to do the automatic last-version selection URL thing add a JSON data dump to get release info in that format Modified: trunk/pypi/webui.py ============================================================================== --- trunk/pypi/webui.py (original) +++ trunk/pypi/webui.py Sat Jul 24 13:03:10 2010 @@ -5,6 +5,7 @@ from zope.pagetemplate.pagetemplatefile import PageTemplateFile from distutils.util import rfc822_escape from distutils2.metadata import DistributionMetadata +import json try: import psycopg2 @@ -531,6 +532,9 @@ else: action = 'home' + if self.form.get('version') in ('doap', 'json'): + action, self.form['version'] = self.form['version'], None + # make sure the user has permission if action in ('submit', ): if not self.authenticated: @@ -544,7 +548,8 @@ display register_form user_form forgotten_password_form user password_reset role role_form list_classifiers login logout files file_upload show_md5 doc_upload claim openid openid_return dropid - rate comment addcomment delcomment clear_auth addkey delkey lasthour'''.split(): + rate comment addcomment delcomment clear_auth addkey delkey lasthour + json'''.split(): getattr(self, action)() else: #raise NotFound, 'Unknown action %s' % action @@ -958,32 +963,11 @@ self.role_form() - def _get_pkg_info(self, name, version): - # get the appropriate package info from the database - if name is None: - try: - name = self.form['name'] - except KeyError: - raise NotFound, 'no package name supplied' - if version is None: - if self.form.has_key('version'): - version = self.form['version'] - else: - l = self.store.get_latest_release(name, hidden=False) - try: - version = l[-1][1] - except IndexError: - raise NotFound, 'no releases' - return self.store.get_package(name, version), name, version - def doap(self, name=None, version=None): '''Return DOAP rendering of a package. ''' - info, name, version = self._get_pkg_info(name, version) - if not info: - return self.fail('No such package / version', - heading='%s %s'%(name, version), - content="I can't find the package / version you're requesting") + info, latest_version = self._load_release_info(name, version) + name = info['name'] root = cElementTree.Element('rdf:RDF', { 'xmlns:rdf': "http://www.w3.org/1999/02/22-rdf-syntax-ns#", @@ -1052,6 +1036,26 @@ self.handler.end_headers() self.wfile.write(s.getvalue()) + def json(self, name=None, version=None): + '''Return DOAP rendering of a package. + ''' + info, latest_version = self._load_release_info(name, version) + name, version = info['name'], info['version'] + d = { + 'info': rpc.release_data(self.store, name, version), + 'urls': rpc.release_urls(self.store, name, version), + } + for url in d['urls']: + url['upload_time'] = url['upload_time'].strftime('%Y-%m-%dT%H:%M:%S') + self.handler.send_response(200, "OK") + self.handler.set_content_type('application/json; charset="UTF-8"') + filename = '%s-%s.xml'%(name.encode('ascii', 'replace'), + version.encode('ascii', 'replace')) + self.handler.send_header('Content-Disposition', + 'attachment; filename=%s'%filename) + self.handler.end_headers() + self.wfile.write(json.dumps(d)) + def display_pkginfo(self, name=None, version=None): '''Reconstruct and send a PKG-INFO metadata file. ''' @@ -1145,9 +1149,13 @@ def quote_plus(self, data): return urllib.quote_plus(data) - def display(self, name=None, version=None, ok_message=None, - error_message=None): - ''' Print up an entry + def _load_release_info(self, name, version): + '''Determine the information about a release of the named package. + + If version is specified then we return that version and also determine + what the latest version of the package is. + + If the version is None then we return the latest version. ''' # get the appropriate package info from the database if name is None: @@ -1156,9 +1164,9 @@ if name is None or isinstance(name, list): self.fail("Which package do you want to display?") - using_latest=False + using_latest = False if version is None: - if self.form.has_key('version'): + if self.form.get('version'): version = self.form['version'] else: l = self.store.get_package_releases(name, hidden=False) @@ -1168,7 +1176,7 @@ try: version = l[-1][1] except IndexError: - using_latest=True + using_latest = True version = "(latest release)" if not using_latest: @@ -1184,15 +1192,19 @@ else: latest_version = None - if latest_version==version: - using_latest=True - info = self.store.get_package(name, version) if not info: raise NotFound -# return self.fail('No such package / version', -# heading='%s %s'%(name, version), -# content="I can't find the package / version you're requesting") + return info, latest_version + + def display(self, name=None, version=None, ok_message=None, + error_message=None): + ''' Print up an entry + ''' + info, latest_version = self._load_release_info(name, version) + name = info['name'] + version = info['version'] + using_latest = latest_version==version # RJ: disabled cheesecake because the (strange) errors were getting annoying # columns = 'name version author author_email maintainer maintainer_email home_page download_url summary license description description_html keywords platform cheesecake_installability_id cheesecake_documentation_id cheesecake_code_kwalitee_id'.split() @@ -1245,7 +1257,8 @@ url = url, id = c['trove_id'])) - latest_version_url = self.config.url+'/'+name+'/'+latest_version + latest_version_url = '%s/%s/%s' % (self.config.url, name, + latest_version) # Compute rating data has_rated = self.loggedin and self.store.has_rating(name, version) From python-checkins at python.org Sat Jul 24 14:37:53 2010 From: python-checkins at python.org (richard) Date: Sat, 24 Jul 2010 14:37:53 +0200 (CEST) Subject: [Pypi-checkins] r799 - trunk/pypi Message-ID: <20100724123753.62E71F3AD@mail.python.org> Author: richard Date: Sat Jul 24 14:37:53 2010 New Revision: 799 Modified: trunk/pypi/webui.py Log: handle multiple matches again Modified: trunk/pypi/webui.py ============================================================================== --- trunk/pypi/webui.py (original) +++ trunk/pypi/webui.py Sat Jul 24 14:37:53 2010 @@ -53,6 +53,10 @@ class FormError(Exception): pass +class MultipleReleases(object): + def __init__(self, releases): + self.releases = releases + __version__ = '1.1' # email sent to user indicating how they should complete their registration @@ -1171,7 +1175,7 @@ else: l = self.store.get_package_releases(name, hidden=False) if len(l) > 1: - return self.index(releases=l) + raise MultipleResults(releases=l) l = self.store.get_latest_release(name, hidden=False) try: version = l[-1][1] @@ -1201,7 +1205,11 @@ error_message=None): ''' Print up an entry ''' - info, latest_version = self._load_release_info(name, version) + try: + info, latest_version = self._load_release_info(name, version) + except MultipleReleases, e: + return self.index(releases=e.releases) + name = info['name'] version = info['version'] using_latest = latest_version==version From python-checkins at python.org Sat Jul 24 14:39:05 2010 From: python-checkins at python.org (richard) Date: Sat, 24 Jul 2010 14:39:05 +0200 (CEST) Subject: [Pypi-checkins] r800 - trunk/pypi Message-ID: <20100724123905.A942FF46E@mail.python.org> Author: richard Date: Sat Jul 24 14:39:05 2010 New Revision: 800 Modified: trunk/pypi/webui.py Log: handle multiple matches again Modified: trunk/pypi/webui.py ============================================================================== --- trunk/pypi/webui.py (original) +++ trunk/pypi/webui.py Sat Jul 24 14:39:05 2010 @@ -1175,7 +1175,7 @@ else: l = self.store.get_package_releases(name, hidden=False) if len(l) > 1: - raise MultipleResults(releases=l) + raise MultipleReleases(releases=l) l = self.store.get_latest_release(name, hidden=False) try: version = l[-1][1] From python-checkins at python.org Sat Jul 24 14:40:14 2010 From: python-checkins at python.org (richard) Date: Sat, 24 Jul 2010 14:40:14 +0200 (CEST) Subject: [Pypi-checkins] r801 - trunk/pypi Message-ID: <20100724124014.9829DF46E@mail.python.org> Author: richard Date: Sat Jul 24 14:40:14 2010 New Revision: 801 Modified: trunk/pypi/webui.py Log: fix Modified: trunk/pypi/webui.py ============================================================================== --- trunk/pypi/webui.py (original) +++ trunk/pypi/webui.py Sat Jul 24 14:40:14 2010 @@ -53,7 +53,7 @@ class FormError(Exception): pass -class MultipleReleases(object): +class MultipleReleases(Exception): def __init__(self, releases): self.releases = releases From python-checkins at python.org Sat Jul 24 15:02:42 2010 From: python-checkins at python.org (richard) Date: Sat, 24 Jul 2010 15:02:42 +0200 (CEST) Subject: [Pypi-checkins] r802 - trunk/pypi Message-ID: <20100724130242.0DD95EE982@mail.python.org> Author: richard Date: Sat Jul 24 15:02:41 2010 New Revision: 802 Modified: trunk/pypi/webui.py Log: fix Modified: trunk/pypi/webui.py ============================================================================== --- trunk/pypi/webui.py (original) +++ trunk/pypi/webui.py Sat Jul 24 15:02:41 2010 @@ -1060,6 +1060,24 @@ self.handler.end_headers() self.wfile.write(json.dumps(d)) + def _get_pkg_info(self, name, version): + # get the appropriate package info from the database + if name is None: + try: + name = self.form['name'] + except KeyError: + raise NotFound, 'no package name supplied' + if version is None: + if self.form.has_key('version'): + version = self.form['version'] + else: + l = self.store.get_latest_release(name, hidden=False) + try: + version = l[-1][1] + except IndexError: + raise NotFound, 'no releases' + return self.store.get_package(name, version), name, version + def display_pkginfo(self, name=None, version=None): '''Reconstruct and send a PKG-INFO metadata file. ''' From python-checkins at python.org Sat Jul 24 16:37:09 2010 From: python-checkins at python.org (richard) Date: Sat, 24 Jul 2010 16:37:09 +0200 (CEST) Subject: [Pypi-checkins] r803 - in trunk/pypi: . templates tools Message-ID: <20100724143709.B28D9EE9D5@mail.python.org> Author: richard Date: Sat Jul 24 16:37:09 2010 New Revision: 803 Modified: trunk/pypi/templates/display.pt trunk/pypi/templates/standard_template.pt trunk/pypi/tools/demodata.py trunk/pypi/webui.py Log: some py3k fun :-) Modified: trunk/pypi/templates/display.pt ============================================================================== --- trunk/pypi/templates/display.pt (original) +++ trunk/pypi/templates/display.pt Sat Jul 24 16:37:09 2010 @@ -19,7 +19,6 @@ -

    Modified: trunk/pypi/templates/standard_template.pt ============================================================================== --- trunk/pypi/templates/standard_template.pt (original) +++ trunk/pypi/templates/standard_template.pt Sat Jul 24 16:37:09 2010 @@ -28,7 +28,14 @@

    - + + + + + + + +

    skip to navigation
    @@ -63,7 +70,7 @@ -
  • Python 3 packages
    • +
  • Python 3 packages
  • Tutorial
    • @@ -177,8 +184,7 @@
    - -

    +

    Modified: trunk/pypi/tools/demodata.py ============================================================================== --- trunk/pypi/tools/demodata.py (original) +++ trunk/pypi/tools/demodata.py Sat Jul 24 16:37:09 2010 @@ -51,7 +51,7 @@ '_pypi_hidden':version!='0.4' }) -st.add_file('spam', '1.0', 'THIS IS SOME CONTENT', '1234', 'text/plain', +st.add_file('spam', '1.0', 'THIS IS SOME CONTENT', '1234', 'sdist', 'any', '', 'demo.txt', None) st.commit() Modified: trunk/pypi/webui.py ============================================================================== --- trunk/pypi/webui.py (original) +++ trunk/pypi/webui.py Sat Jul 24 16:37:09 2010 @@ -1273,13 +1273,17 @@ return [ cgi.escape(x['specifier']) for x in l] categories = [] + is_py3k = False for c in self.store.get_release_classifiers(name, version): path = str2path(c['classifier']) + pathstr = path2str(path) + if pathstr.startswith('Programming Language :: Python :: 3'): + is_py3k = True url = "%s?:action=browse&c=%s" % (self.url_path, c['trove_id']) categories.append(dict( name = c['classifier'], path = path, - pathstr = path2str(path), + pathstr = pathstr, url = url, id = c['trove_id'])) @@ -1367,6 +1371,7 @@ tally_ratings=tally, comments=comments, categories=categories, + is_py3k=is_py3k, roles=roles, newline_to_br=newline_to_br, usinglatest=using_latest, From python-checkins at python.org Sat Jul 24 16:41:50 2010 From: python-checkins at python.org (richard) Date: Sat, 24 Jul 2010 16:41:50 +0200 (CEST) Subject: [Pypi-checkins] r804 - trunk/pypi/templates Message-ID: <20100724144150.D78FFF651@mail.python.org> Author: richard Date: Sat Jul 24 16:41:50 2010 New Revision: 804 Modified: trunk/pypi/templates/standard_template.pt Log: undo link changes Modified: trunk/pypi/templates/standard_template.pt ============================================================================== --- trunk/pypi/templates/standard_template.pt (original) +++ trunk/pypi/templates/standard_template.pt Sat Jul 24 16:41:50 2010 @@ -70,7 +70,7 @@ -
  • Python 3 packages
    • +
  • Python 3 packages
  • Tutorial
    • From python-checkins at python.org Sat Jul 24 17:14:13 2010 From: python-checkins at python.org (richard) Date: Sat, 24 Jul 2010 17:14:13 +0200 (CEST) Subject: [Pypi-checkins] r805 - in trunk/pypi: . templates Message-ID: <20100724151413.0315CEE9B3@mail.python.org> Author: richard Date: Sat Jul 24 17:14:12 2010 New Revision: 805 Modified: trunk/pypi/templates/display.pt trunk/pypi/webui.py Log: link to docs Modified: trunk/pypi/templates/display.pt ============================================================================== --- trunk/pypi/templates/display.pt (original) +++ trunk/pypi/templates/display.pt Sat Jul 24 17:14:12 2010 @@ -96,6 +96,11 @@ +
  • + Documentation: + +
    • +
  • Home Page: Modified: trunk/pypi/webui.py ============================================================================== --- trunk/pypi/webui.py (original) +++ trunk/pypi/webui.py Sat Jul 24 17:14:12 2010 @@ -1356,6 +1356,12 @@ return result comments = "".join(render_comments(hcomments, True)) + docs = '' + for sub in [[], ['html']]: + path = [self.config.database_docs_dir, name] + sub + ['index.html'] + if os.path.exists(os.path.join(path)): + docs = '/'.join(['http://packages.python.org', name] + sub) + self.write_template('display.pt', name=name, version=version, release=release, description=release.get('summary') or name, @@ -1366,6 +1372,7 @@ obsoletes=values('obsoletes'), has_rated=has_rated, latest_rating=latest_rating, + docs=docs, sum_ratings=total, nr_ratings=len(ratings), tally_ratings=tally, From python-checkins at python.org Sat Jul 24 17:24:24 2010 From: python-checkins at python.org (richard) Date: Sat, 24 Jul 2010 17:24:24 +0200 (CEST) Subject: [Pypi-checkins] r806 - in trunk/pypi: . templates Message-ID: <20100724152424.AE110D6CF@mail.python.org> Author: richard Date: Sat Jul 24 17:24:24 2010 New Revision: 806 Modified: trunk/pypi/templates/display.pt trunk/pypi/webui.py Log: better Modified: trunk/pypi/templates/display.pt ============================================================================== --- trunk/pypi/templates/display.pt (original) +++ trunk/pypi/templates/display.pt Sat Jul 24 17:24:24 2010 @@ -98,7 +98,7 @@
  • Documentation: - +
    • Modified: trunk/pypi/webui.py ============================================================================== --- trunk/pypi/webui.py (original) +++ trunk/pypi/webui.py Sat Jul 24 17:24:24 2010 @@ -1359,7 +1359,7 @@ docs = '' for sub in [[], ['html']]: path = [self.config.database_docs_dir, name] + sub + ['index.html'] - if os.path.exists(os.path.join(path)): + if os.path.exists(os.path.join(*path)): docs = '/'.join(['http://packages.python.org', name] + sub) self.write_template('display.pt', From python-checkins at python.org Sat Jul 24 17:33:14 2010 From: python-checkins at python.org (richard) Date: Sat, 24 Jul 2010 17:33:14 +0200 (CEST) Subject: [Pypi-checkins] r807 - trunk/pypi/templates Message-ID: <20100724153314.473E4F503@mail.python.org> Author: richard Date: Sat Jul 24 17:33:14 2010 New Revision: 807 Modified: trunk/pypi/templates/display.pt Log: date does not need to be so precise Modified: trunk/pypi/templates/display.pt ============================================================================== --- trunk/pypi/templates/display.pt (original) +++ trunk/pypi/templates/display.pt Sat Jul 24 17:33:14 2010 @@ -51,7 +51,7 @@ comments file/comment_text; pkg_type python:app.dist_file_types_d[file['packagetype']]; py_version file/python_version; - uploaded file/upload_time; + uploaded python:file.upload_time.strftime('%Y-%m-%d'); size python:app.pretty_size(file['size']); downloads file/downloads; md5_url string:${app/url_path}?:action=show_md5&digest=${file/md5_digest}; From python-checkins at python.org Sat Jul 24 17:33:50 2010 From: python-checkins at python.org (richard) Date: Sat, 24 Jul 2010 17:33:50 +0200 (CEST) Subject: [Pypi-checkins] r808 - trunk/pypi/templates Message-ID: <20100724153350.05143CA51@mail.python.org> Author: richard Date: Sat Jul 24 17:33:49 2010 New Revision: 808 Modified: trunk/pypi/templates/display.pt Log: ergh Modified: trunk/pypi/templates/display.pt ============================================================================== --- trunk/pypi/templates/display.pt (original) +++ trunk/pypi/templates/display.pt Sat Jul 24 17:33:49 2010 @@ -51,7 +51,7 @@ comments file/comment_text; pkg_type python:app.dist_file_types_d[file['packagetype']]; py_version file/python_version; - uploaded python:file.upload_time.strftime('%Y-%m-%d'); + uploaded python:file['upload_time'].strftime('%Y-%m-%d'); size python:app.pretty_size(file['size']); downloads file/downloads; md5_url string:${app/url_path}?:action=show_md5&digest=${file/md5_digest}; From python-checkins at python.org Sat Jul 24 17:34:28 2010 From: python-checkins at python.org (richard) Date: Sat, 24 Jul 2010 17:34:28 +0200 (CEST) Subject: [Pypi-checkins] r809 - trunk/pypi/templates Message-ID: <20100724153428.8A5F3F591@mail.python.org> Author: richard Date: Sat Jul 24 17:34:28 2010 New Revision: 809 Modified: trunk/pypi/templates/display.pt Log: hmm Modified: trunk/pypi/templates/display.pt ============================================================================== --- trunk/pypi/templates/display.pt (original) +++ trunk/pypi/templates/display.pt Sat Jul 24 17:34:28 2010 @@ -82,7 +82,7 @@ - +