\n' + cgi.escape(source) + '') - else: - output = parts['body'] - - if output_encoding != 'unicode': - output = output.encode(output_encoding) - - return output - def xmlescape(url): '''Make sure a URL is valid XML''' p = expat.ParserCreate() Modified: trunk/pypi/webui.py ============================================================================== --- trunk/pypi/webui.py (original) +++ trunk/pypi/webui.py Tue Mar 13 00:10:07 2012 @@ -2196,15 +2196,19 @@ "You are not allowed to edit '%s' package information"%name # verify the release exists - if not self.store.has_release(name, version): + if self.store.has_release(name, version): + release_metadata = self.store.get_package(name, version) + has_description = release_metadata['description'] + else: # auto-register the release... - data = self.form_metadata() + release_metadata = self.form_metadata() + has_description = release_metadata.get('description') try: - self.validate_metadata(data) + self.validate_metadata(release_metadata) except ValueError, message: raise FormError, message - data['_pypi_hidden'] = False - self.store.store_package(name, version, data) + release_metadata['_pypi_hidden'] = False + self.store.store_package(name, version, release_metadata) self.store.changed() # verify we have enough information @@ -2286,6 +2290,15 @@ if signature and not signature.startswith("-----BEGIN PGP SIGNATURE-----"): raise FormError, "signature is not ASCII-armored" + # Determine whether we could use a README to fill out a missing + # description + if not has_description: + desc_text, desc_html = extractPackageReadme(content, + filename, filetype) + if desc_text: + self.store.set_description(name, version, desc_text, desc_html, + from_readme=True) + # digest content m = hashlib.md5() m.update(content) From python-checkins at python.org Tue Mar 13 00:10:18 2012 From: python-checkins at python.org (richard) Date: Tue, 13 Mar 2012 00:10:18 +0100 (CET) Subject: [Pypi-checkins] r1012 - trunk/pypi Message-ID: <3V6FKQ10FBzNWT@mail.python.org> Author: richard Date: Tue Mar 13 00:10:17 2012 New Revision: 1012 Added: trunk/pypi/description_utils.py Log: add reading of long description from README files Added: trunk/pypi/description_utils.py ============================================================================== --- (empty file) +++ trunk/pypi/description_utils.py Tue Mar 13 00:10:17 2012 @@ -0,0 +1,168 @@ +import sys +import zipfile +import tarfile +import gzip +import bz2 +import StringIO + +from docutils.core import publish_parts + + +def trim_docstring(text): + """ + Trim indentation and blank lines from docstring text & return it. + + See PEP 257. + """ + if not text: + return text + # Convert tabs to spaces (following the normal Python rules) + # and split into a list of lines: + lines = text.expandtabs().splitlines() + # Determine minimum indentation (first line doesn't count): + indent = sys.maxint + for line in lines[1:]: + stripped = line.lstrip() + if stripped: + indent = min(indent, len(line) - len(stripped)) + # Remove indentation (first line is special): + trimmed = [lines[0].strip()] + if indent < sys.maxint: + for line in lines[1:]: + trimmed.append(line[indent:].rstrip()) + # Strip off trailing and leading blank lines: + while trimmed and not trimmed[-1]: + trimmed.pop() + while trimmed and not trimmed[0]: + trimmed.pop(0) + # Return a single string: + return '\n'.join(trimmed) + + +def processDescription(source, output_encoding='unicode'): + """Given an source string, returns an HTML fragment as a string. + + The return value is the contents of the tag. + + Parameters: + + - `source`: A multi-line text string; required. + - `output_encoding`: The desired encoding of the output. If a Unicode + string is desired, use the default value of "unicode" . + """ + # Dedent all lines of `source`. + source = trim_docstring(source) + + settings_overrides={ + 'raw_enabled': 0, # no raw HTML code + 'file_insertion_enabled': 0, # no file/URL access + 'halt_level': 2, # at warnings or errors, raise an exception + 'report_level': 5, # never report problems with the reST code + } + + # capture publishing errors, they go to stderr + old_stderr = sys.stderr + sys.stderr = s = StringIO.StringIO() + parts = None + try: + # Convert reStructuredText to HTML using Docutils. + parts = publish_parts(source=source, writer_name='html', + settings_overrides=settings_overrides) + except: + pass + + sys.stderr = old_stderr + + # original text if publishing errors occur + if parts is None or len(s.getvalue()) > 0: + output = "".join('
\n' + cgi.escape(source) + '') + else: + output = parts['body'] + + if output_encoding != 'unicode': + output = output.encode(output_encoding) + + return output + +def extractPackageReadme(content, filename, filetype): + '''Extract the README from a file and attempt to turn it into HTML. + + Return the source text and html version or emty strings in either case if + extraction fails. + ''' + text = html = '' + if filename.endswith('.zip') or filename.endswith('.egg'): + try: + t = StringIO.StringIO(content) + t.filename = filename + zip = zipfile.ZipFile(t) + l = zip.namelist() + except zipfile.error: + return '', '' + for entry in l: + parts = entry.split('/') + if len(parts) != 2: + continue + filename = parts[-1] + if filename.count('.') > 1: + continue + if filename.count('.') == 1: + name, ext = filename.split('.') + else: + # just use the filename and assume a readme is plain text + name = filename + ext = 'txt' + if name.upper() != 'README': + continue + # grab the content and parse if it's something we might understand, + # based on the file extension + text = zip.open(entry).read() + if ext in ('txt', 'rst', 'md'): + html = processDescription(text) + return text, html + + elif (filename.endswith('.tar.gz') or filename.endswith('.tgz') or + filename.endswith('.tar.bz2') or filename.endswith('.tbz2')): + # open the tar file with the appropriate compression + ext = filename.split('.')[-1] + if ext[-2:] == 'gz': + file = StringIO.StringIO(content) + file = gzip.GzipFile(filename, fileobj=file) + else: + file = StringIO.StringIO(bz2.decompress(content)) + try: + tar = tarfile.TarFile(filename, 'r', file) + l = tar.getmembers() + except tarfile.TarError: + return '', '' + for entry in l: + parts = entry.name.split('/') + if len(parts) != 2: + continue + filename = parts[-1] + if filename.count('.') > 1: + continue + if filename.count('.') == 1: + name, ext = filename.split('.') + else: + # just use the filename and assume a readme is plain text + name = filename + ext = 'txt' + if name.upper() != 'README': + continue + print 'FOUND', filename + # grab the content and parse if it's something we might understand, + # based on the file extension + text = tar.extractfile(entry).read() + if ext in ('txt', 'rst', 'md'): + html = processDescription(text) + return text, html + + return text, html + +if __name__ == '__main__': + fname ='../parse/dist/parse-1.4.1.tar.gz' + fname ='../parse/dist/parse-1.4.1.zip' + fname ='../parse/dist/parse-1.4.1.tar.bz2' + text, html = extractPackageReadme(open(fname).read(), fname, 'sdist') + From python-checkins at python.org Tue Mar 13 00:24:37 2012 From: python-checkins at python.org (richard) Date: Tue, 13 Mar 2012 00:24:37 +0100 (CET) Subject: [Pypi-checkins] r1013 - trunk/pypi Message-ID: <3V6Fdx41xSzNbr@mail.python.org> Author: richard Date: Tue Mar 13 00:24:37 2012 New Revision: 1013 Modified: trunk/pypi/webui.py Log: handle distutils "UNKNOWN" Modified: trunk/pypi/webui.py ============================================================================== --- trunk/pypi/webui.py (original) +++ trunk/pypi/webui.py Tue Mar 13 00:24:37 2012 @@ -2198,11 +2198,11 @@ # verify the release exists if self.store.has_release(name, version): release_metadata = self.store.get_package(name, version) - has_description = release_metadata['description'] + description = release_metadata['description'] else: # auto-register the release... release_metadata = self.form_metadata() - has_description = release_metadata.get('description') + description = release_metadata.get('description') try: self.validate_metadata(release_metadata) except ValueError, message: @@ -2211,6 +2211,10 @@ self.store.store_package(name, version, release_metadata) self.store.changed() + # distutils handily substitutes blank descriptions with "UNKNOWN" + if description == 'UNKNOWN': + description = '' + # verify we have enough information pyversion = 'source' content = filetype = md5_digest = comment = None @@ -2292,11 +2296,11 @@ # Determine whether we could use a README to fill out a missing # description - if not has_description: - desc_text, desc_html = extractPackageReadme(content, + if not description: + description, desc_html = extractPackageReadme(content, filename, filetype) - if desc_text: - self.store.set_description(name, version, desc_text, desc_html, + if description: + self.store.set_description(name, version, description, desc_html, from_readme=True) # digest content From python-checkins at python.org Tue Mar 13 00:32:17 2012 From: python-checkins at python.org (richard) Date: Tue, 13 Mar 2012 00:32:17 +0100 (CET) Subject: [Pypi-checkins] r1014 - trunk/pypi Message-ID: <3V6Fpn56dBzPFk@mail.python.org> Author: richard Date: Tue Mar 13 00:32:17 2012 New Revision: 1014 Modified: trunk/pypi/webui.py Log: add missing import Modified: trunk/pypi/webui.py ============================================================================== --- trunk/pypi/webui.py (original) +++ trunk/pypi/webui.py Tue Mar 13 00:32:17 2012 @@ -35,6 +35,7 @@ import store, config, versionpredicate, verify_filetype, rpc import MailingLogger, openid2rp, gae from mini_pkg_resources import safe_name +from description_utils import extractPackageReadme esc = cgi.escape esq = lambda x: cgi.escape(x, True) From python-checkins at python.org Tue Mar 13 01:17:05 2012 From: python-checkins at python.org (richard) Date: Tue, 13 Mar 2012 01:17:05 +0100 (CET) Subject: [Pypi-checkins] r1015 - trunk/pypi Message-ID: <3V6GpT558MzNcn@mail.python.org> Author: richard Date: Tue Mar 13 01:17:05 2012 New Revision: 1015 Modified: trunk/pypi/config.py trunk/pypi/webui.py Log: add sentry support Modified: trunk/pypi/config.py ============================================================================== --- trunk/pypi/config.py (original) +++ trunk/pypi/config.py Tue Mar 13 01:17:05 2012 @@ -47,6 +47,8 @@ self.fromaddr = c.get('logging', 'fromaddr') self.toaddrs = c.get('logging', 'toaddrs').split(',') + self.sentry_dsn = c.get('sentry', 'dsn') + def make_https(self): if self.url.startswith("http:"): self.url = "https"+self.url[4:] Modified: trunk/pypi/webui.py ============================================================================== --- trunk/pypi/webui.py (original) +++ trunk/pypi/webui.py Tue Mar 13 01:17:05 2012 @@ -31,6 +31,9 @@ from openid.server import server as OpenIDServer +# Raven for error reporting +import raven + # local imports import store, config, versionpredicate, verify_filetype, rpc import MailingLogger, openid2rp, gae @@ -210,6 +213,7 @@ self.handler = handler self.config = handler.config self.wfile = handler.wfile + self.sentry_client = raven.Client(self.config.sentry_dsn) self.env = env self.nav_current = None self.privkey = None @@ -313,10 +317,17 @@ code=500, heading='Database connection failed') except: exc, value, tb = sys.exc_info() - if ('connection limit exceeded for non-superusers' - not in str(value)): - logging.exception('Internal Error\n----\n%s\n----\n'%( - '\n'.join(['%s: %s'%x for x in self.env.items()]))) + + # attempt to send all the exceptions to Raven + try: + self.sentry_client.captureException() + except Exception: + # sentry broke so just email the exception like old times + if ('connection limit exceeded for non-superusers' + not in str(value)): + logging.exception('Internal Error\n----\n%s\n----\n'%( + '\n'.join(['%s: %s'%x for x in self.env.items()]))) + if self.config.debug_mode == 'yes': s = cStringIO.StringIO() traceback.print_exc(None, s) @@ -570,7 +581,8 @@ password_reset role role_form list_classifiers login logout files file_upload show_md5 doc_upload claim openid openid_return dropid clear_auth addkey delkey lasthour json gae_file about delete_user - rss_regen openid_endpoint openid_decide_post packages_rss'''.split(): + rss_regen openid_endpoint openid_decide_post packages_rss + exception'''.split(): getattr(self, action)() else: #raise NotFound, 'Unknown action %s' % action @@ -585,6 +597,9 @@ def debug(self): self.fail('Debug info', code=200, content=str(self.env)) + def exception(self): + FAIL + def xmlrpc(self): rpc.handle_request(self) From python-checkins at python.org Tue Mar 13 02:35:12 2012 From: python-checkins at python.org (richard) Date: Tue, 13 Mar 2012 02:35:12 +0100 (CET) Subject: [Pypi-checkins] r1016 - trunk/pypi Message-ID: <3V6JXc4508zNbr@mail.python.org> Author: richard Date: Tue Mar 13 02:35:12 2012 New Revision: 1016 Modified: trunk/pypi/description_utils.py Log: whoops Modified: trunk/pypi/description_utils.py ============================================================================== --- trunk/pypi/description_utils.py (original) +++ trunk/pypi/description_utils.py Tue Mar 13 02:35:12 2012 @@ -4,6 +4,7 @@ import gzip import bz2 import StringIO +import cgi from docutils.core import publish_parts From python-checkins at python.org Tue Mar 13 22:17:00 2012 From: python-checkins at python.org (richard) Date: Tue, 13 Mar 2012 22:17:00 +0100 (CET) Subject: [Pypi-checkins] r1017 - trunk/pypi/templates Message-ID: <3V6pmD1mSTzM6x@mail.python.org> Author: richard Date: Tue Mar 13 22:17:00 2012 New Revision: 1017 Modified: trunk/pypi/templates/home.pt Log: roll in change from production Modified: trunk/pypi/templates/home.pt ============================================================================== --- trunk/pypi/templates/home.pt (original) +++ trunk/pypi/templates/home.pt Tue Mar 13 22:17:00 2012 @@ -23,7 +23,7 @@ Get Packages
To use a package from this index either -"pip install package" +"pip install package" (get pip) or download, unpack and "python setup.py install" it. Browse all packages From python-checkins at python.org Tue Mar 13 22:17:54 2012 From: python-checkins at python.org (richard) Date: Tue, 13 Mar 2012 22:17:54 +0100 (CET) Subject: [Pypi-checkins] r1018 - trunk/pypi Message-ID: <3V6pnG1MztzMXH@mail.python.org> Author: richard Date: Tue Mar 13 22:17:54 2012 New Revision: 1018 Modified: trunk/pypi/pypi.wsgi Log: roll in change from production Modified: trunk/pypi/pypi.wsgi ============================================================================== --- trunk/pypi/pypi.wsgi (original) +++ trunk/pypi/pypi.wsgi Tue Mar 13 22:17:54 2012 @@ -6,7 +6,7 @@ store.keep_conn = True -CONFIG_FILE = 'config.ini' +CONFIG_FILE = os.path.join(prefix, 'config.ini') class Request: From python-checkins at python.org Tue Mar 13 22:18:37 2012 From: python-checkins at python.org (richard) Date: Tue, 13 Mar 2012 22:18:37 +0100 (CET) Subject: [Pypi-checkins] r1019 - trunk/pypi Message-ID: <3V6pp52XkGzP83@mail.python.org> Author: richard Date: Tue Mar 13 22:18:37 2012 New Revision: 1019 Modified: trunk/pypi/pypi.wsgi Log: roll in change from production Modified: trunk/pypi/pypi.wsgi ============================================================================== --- trunk/pypi/pypi.wsgi (original) +++ trunk/pypi/pypi.wsgi Tue Mar 13 22:18:37 2012 @@ -53,6 +53,7 @@ r = Request(environ, start_response) webui.WebUI(r, environ).run() return [r.wfile.getvalue()] +#application=debug if __name__ == '__main__': # very simple wsgi server so we can play locally From python-checkins at python.org Wed Mar 14 05:49:42 2012 From: python-checkins at python.org (richard) Date: Wed, 14 Mar 2012 05:49:42 +0100 (CET) Subject: [Pypi-checkins] r1020 - trunk/pypi Message-ID: <3V70pZ6VbBzPJS@mail.python.org> Author: richard Date: Wed Mar 14 05:49:42 2012 New Revision: 1020 Added: trunk/pypi/wsgi_app.py Modified: trunk/pypi/pypi.wsgi Log: refactor to allow multiple wsgi application configs more easily Modified: trunk/pypi/pypi.wsgi ============================================================================== --- trunk/pypi/pypi.wsgi (original) +++ trunk/pypi/pypi.wsgi Wed Mar 14 05:49:42 2012 @@ -1,64 +1,12 @@ #!/usr/bin/python -import sys,os +import sys, os prefix = os.path.dirname(__file__) sys.path.insert(0, prefix) -import cStringIO, webui, store, config -store.keep_conn = True - -CONFIG_FILE = os.path.join(prefix, 'config.ini') - -class Request: - - def __init__(self, environ, start_response): - self.start_response = start_response - try: - length = int(environ['CONTENT_LENGTH']) - except ValueError: - length = 0 - self.rfile = cStringIO.StringIO(environ['wsgi.input'].read(length)) - self.wfile = cStringIO.StringIO() - self.config = config.Config(CONFIG_FILE ) - - def send_response(self, code, message='no details available'): - self.status = '%s %s' % (code, message) - self.headers = [] - - def send_header(self, keyword, value): - self.headers.append((keyword, value)) - - def set_content_type(self, content_type): - self.send_header('Content-Type', content_type) - - def end_headers(self): - self.start_response(self.status, self.headers) - -def debug(environ, start_response): - if environ['PATH_INFO'].startswith("/auth") and \ - "HTTP_AUTHORIZATION" not in environ: - start_response("401 login", - [('WWW-Authenticate', 'Basic realm="foo"')]) - return - start_response("200 ok", [('Content-type', 'text/plain')]) - environ = environ.items() - environ.sort() - for k,v in environ: - yield "%s=%s\n" % (k, v) - return - - -def application(environ, start_response): - if "HTTP_AUTHORIZATION" in environ: - environ["HTTP_CGI_AUTHORIZATION"] = environ["HTTP_AUTHORIZATION"] - r = Request(environ, start_response) - webui.WebUI(r, environ).run() - return [r.wfile.getvalue()] -#application=debug +import wsgi_app +config_path = os.path.join(prefix, 'config.ini') +application = wsgi_app.Application(config_path, debug=False) if __name__ == '__main__': - # very simple wsgi server so we can play locally - from wsgiref.simple_server import make_server - httpd = make_server('', 8000, application) - print "Serving on port 8000..." - httpd.serve_forever() + application.test(8000) Added: trunk/pypi/wsgi_app.py ============================================================================== --- (empty file) +++ trunk/pypi/wsgi_app.py Wed Mar 14 05:49:42 2012 @@ -0,0 +1,65 @@ +#!/usr/bin/python +import sys,os +import cStringIO, webui, store, config +from wsgiref.simple_server import make_server + +class Request: + def __init__(self, environ, start_response, config): + self.start_response = start_response + try: + length = int(environ['CONTENT_LENGTH']) + except ValueError: + length = 0 + self.rfile = cStringIO.StringIO(environ['wsgi.input'].read(length)) + self.wfile = cStringIO.StringIO() + self.config = config.Config(config) + + def send_response(self, code, message='no details available'): + self.status = '%s %s' % (code, message) + self.headers = [] + + def send_header(self, keyword, value): + self.headers.append((keyword, value)) + + def set_content_type(self, content_type): + self.send_header('Content-Type', content_type) + + def end_headers(self): + self.start_response(self.status, self.headers) + +class Application: + def __init__(self, config, debug=False): + self.config = config + + if debug: + self.__call__ = self.debug + else: + self.__call__ = self.application + store.keep_conn = True + + def application(self, environ, start_response): + if "HTTP_AUTHORIZATION" in environ: + environ["HTTP_CGI_AUTHORIZATION"] = environ["HTTP_AUTHORIZATION"] + r = Request(environ, start_response) + webui.WebUI(r, environ).run() + return [r.wfile.getvalue()] + + def debug(self, environ, start_response): + if environ['PATH_INFO'].startswith("/auth") and \ + "HTTP_AUTHORIZATION" not in environ: + start_response("401 login", + [('WWW-Authenticate', 'Basic realm="foo"')]) + return + start_response("200 ok", [('Content-type', 'text/plain')]) + environ = environ.items() + environ.sort() + for k,v in environ: + yield "%s=%s\n" % (k, v) + return + + def test(self, port): + # very simple wsgi server so we can play locally + httpd = make_server('', port, self) + print "Serving on port %d..." % port + httpd.serve_forever() + From python-checkins at python.org Wed Mar 14 05:50:37 2012 From: python-checkins at python.org (richard) Date: Wed, 14 Mar 2012 05:50:37 +0100 (CET) Subject: [Pypi-checkins] r1021 - trunk/pypi Message-ID: <3V70qd0rlWzPJS@mail.python.org> Author: richard Date: Wed Mar 14 05:50:36 2012 New Revision: 1021 Added: trunk/pypi/testpypi.wsgi Log: add test pypi wsgi config Added: trunk/pypi/testpypi.wsgi ============================================================================== --- (empty file) +++ trunk/pypi/testpypi.wsgi Wed Mar 14 05:50:36 2012 @@ -0,0 +1,12 @@ +#!/usr/bin/python +import sys, os +prefix = os.path.dirname(__file__) +sys.path.insert(0, prefix) + +import wsgi_app +config_path = os.path.join(prefix, 'testpypi-config.ini') +application = wsgi_app.Application(config_path, debug=True) + +if __name__ == '__main__': + application.test(8000) + From python-checkins at python.org Wed Mar 14 18:50:05 2012 From: python-checkins at python.org (richard) Date: Wed, 14 Mar 2012 18:50:05 +0100 (CET) Subject: [Pypi-checkins] r1022 - trunk/pypi Message-ID: <3V7L7128t8zNFT@mail.python.org> Author: richard Date: Wed Mar 14 18:50:04 2012 New Revision: 1022 Removed: trunk/pypi/testpypi.wsgi Log: remove unnecessary file Deleted: /trunk/pypi/testpypi.wsgi ============================================================================== --- /trunk/pypi/testpypi.wsgi Wed Mar 14 18:50:04 2012 +++ (empty file) @@ -1,12 +0,0 @@ -#!/usr/bin/python -import sys, os -prefix = os.path.dirname(__file__) -sys.path.insert(0, prefix) - -import wsgi_app -config_path = os.path.join(prefix, 'testpypi-config.ini') -application = wsgi_app.Application(config_path, debug=True) - -if __name__ == '__main__': - application.test(8000) - From python-checkins at python.org Wed Mar 14 20:54:59 2012 From: python-checkins at python.org (richard) Date: Wed, 14 Mar 2012 20:54:59 +0100 (CET) Subject: [Pypi-checkins] r1023 - trunk/pypi Message-ID: <3V7Nv73sR2zMP9@mail.python.org> Author: richard Date: Wed Mar 14 20:54:59 2012 New Revision: 1023 Modified: trunk/pypi/description_utils.py Log: prevent javscript (well, non link-ish per the URL spec) links from propogating through from rest to html Modified: trunk/pypi/description_utils.py ============================================================================== --- trunk/pypi/description_utils.py (original) +++ trunk/pypi/description_utils.py Wed Mar 14 20:54:59 2012 @@ -5,8 +5,12 @@ import bz2 import StringIO import cgi +import urlparse -from docutils.core import publish_parts +from docutils import io, readers +from docutils.core import publish_doctree, Publisher +from docutils.writers import get_writer_class +from docutils.transforms import TransformError, Transform def trim_docstring(text): @@ -39,6 +43,9 @@ # Return a single string: return '\n'.join(trimmed) +ALLOWED_SCHEMES = '''file ftp gopher hdl http https imap mailto mms news nntp +prospero rsync rtsp rtspu sftp shttp sip sips snews svn svn+ssh telnet +wais'''.split() def processDescription(source, output_encoding='unicode'): """Given an source string, returns an HTML fragment as a string. @@ -65,10 +72,35 @@ old_stderr = sys.stderr sys.stderr = s = StringIO.StringIO() parts = None + try: # Convert reStructuredText to HTML using Docutils. - parts = publish_parts(source=source, writer_name='html', - settings_overrides=settings_overrides) + document = publish_doctree(source=source, + settings_overrides=settings_overrides) + + for node in document.traverse(): + if node.tagname == '#text': + continue + if node.hasattr('refuri'): + uri = node['refuri'] + elif node.hasattr('uri'): + uri = node['uri'] + else: + continue + o = urlparse.urlparse(uri) + if o.scheme not in ALLOWED_SCHEMES: + raise TransformError('link scheme not allowed') + + # now turn the transformed document into HTML + reader = readers.doctree.Reader(parser_name='null') + pub = Publisher(reader, source=io.DocTreeInput(document), + destination_class=io.StringOutput) + pub.set_writer('html') + pub.process_programmatic_settings(None, settings_overrides, None) + pub.set_destination(None, None) + pub.publish() + parts = pub.writer.parts + except: pass @@ -151,7 +183,6 @@ ext = 'txt' if name.upper() != 'README': continue - print 'FOUND', filename # grab the content and parse if it's something we might understand, # based on the file extension text = tar.extractfile(entry).read() @@ -163,7 +194,7 @@ if __name__ == '__main__': fname ='../parse/dist/parse-1.4.1.tar.gz' - fname ='../parse/dist/parse-1.4.1.zip' - fname ='../parse/dist/parse-1.4.1.tar.bz2' +# fname ='../parse/dist/parse-1.4.1.zip' +# fname ='../parse/dist/parse-1.4.1.tar.bz2' text, html = extractPackageReadme(open(fname).read(), fname, 'sdist') From python-checkins at python.org Wed Mar 14 21:07:31 2012 From: python-checkins at python.org (richard) Date: Wed, 14 Mar 2012 21:07:31 +0100 (CET) Subject: [Pypi-checkins] r1024 - trunk/pypi Message-ID: <3V7P9b1lWHzNLs@mail.python.org> Author: richard Date: Wed Mar 14 21:07:30 2012 New Revision: 1024 Removed: trunk/pypi/wsgi_app.py Modified: trunk/pypi/pypi.wsgi Log: revert that crap Modified: trunk/pypi/pypi.wsgi ============================================================================== --- trunk/pypi/pypi.wsgi (original) +++ trunk/pypi/pypi.wsgi Wed Mar 14 21:07:30 2012 @@ -1,12 +1,64 @@ #!/usr/bin/python -import sys, os +import sys,os prefix = os.path.dirname(__file__) sys.path.insert(0, prefix) +import cStringIO, webui, store, config -import wsgi_app -config_path = os.path.join(prefix, 'config.ini') -application = wsgi_app.Application(config_path, debug=False) +store.keep_conn = True + +CONFIG_FILE = os.path.join(prefix, 'config.ini') + +class Request: + + def __init__(self, environ, start_response): + self.start_response = start_response + try: + length = int(environ['CONTENT_LENGTH']) + except ValueError: + length = 0 + self.rfile = cStringIO.StringIO(environ['wsgi.input'].read(length)) + self.wfile = cStringIO.StringIO() + self.config = config.Config(CONFIG_FILE ) + + def send_response(self, code, message='no details available'): + self.status = '%s %s' % (code, message) + self.headers = [] + + def send_header(self, keyword, value): + self.headers.append((keyword, value)) + + def set_content_type(self, content_type): + self.send_header('Content-Type', content_type) + + def end_headers(self): + self.start_response(self.status, self.headers) + +def debug(environ, start_response): + if environ['PATH_INFO'].startswith("/auth") and \ + "HTTP_AUTHORIZATION" not in environ: + start_response("401 login", + [('WWW-Authenticate', 'Basic realm="foo"')]) + return + start_response("200 ok", [('Content-type', 'text/plain')]) + environ = environ.items() + environ.sort() + for k,v in environ: + yield "%s=%s\n" % (k, v) + return + + +def application(environ, start_response): + if "HTTP_AUTHORIZATION" in environ: + environ["HTTP_CGI_AUTHORIZATION"] = environ["HTTP_AUTHORIZATION"] + r = Request(environ, start_response) + webui.WebUI(r, environ).run() + return [r.wfile.getvalue()] +#application=debug if __name__ == '__main__': - application.test(8000) + # very simple wsgi server so we can play locally + from wsgiref.simple_server import make_server + httpd = make_server('', 8000, application) + print "Serving on port 8000..." + httpd.serve_forever() Deleted: /trunk/pypi/wsgi_app.py ============================================================================== --- /trunk/pypi/wsgi_app.py Wed Mar 14 21:07:30 2012 +++ (empty file) @@ -1,65 +0,0 @@ -#!/usr/bin/python -import sys,os -import cStringIO, webui, store, config -from wsgiref.simple_server import make_server - -class Request: - def __init__(self, environ, start_response, config): - self.start_response = start_response - try: - length = int(environ['CONTENT_LENGTH']) - except ValueError: - length = 0 - self.rfile = cStringIO.StringIO(environ['wsgi.input'].read(length)) - self.wfile = cStringIO.StringIO() - self.config = config.Config(config) - - def send_response(self, code, message='no details available'): - self.status = '%s %s' % (code, message) - self.headers = [] - - def send_header(self, keyword, value): - self.headers.append((keyword, value)) - - def set_content_type(self, content_type): - self.send_header('Content-Type', content_type) - - def end_headers(self): - self.start_response(self.status, self.headers) - -class Application: - def __init__(self, config, debug=False): - self.config = config - - if debug: - self.__call__ = self.debug - else: - self.__call__ = self.application - store.keep_conn = True - - def application(self, environ, start_response): - if "HTTP_AUTHORIZATION" in environ: - environ["HTTP_CGI_AUTHORIZATION"] = environ["HTTP_AUTHORIZATION"] - r = Request(environ, start_response) - webui.WebUI(r, environ).run() - return [r.wfile.getvalue()] - - def debug(self, environ, start_response): - if environ['PATH_INFO'].startswith("/auth") and \ - "HTTP_AUTHORIZATION" not in environ: - start_response("401 login", - [('WWW-Authenticate', 'Basic realm="foo"')]) - return - start_response("200 ok", [('Content-type', 'text/plain')]) - environ = environ.items() - environ.sort() - for k,v in environ: - yield "%s=%s\n" % (k, v) - return - - def test(self, port): - # very simple wsgi server so we can play locally - httpd = make_server('', port, self) - print "Serving on port %d..." % port - httpd.serve_forever() - From python-checkins at python.org Thu Mar 15 17:22:15 2012 From: python-checkins at python.org (martin.von.loewis) Date: Thu, 15 Mar 2012 17:22:15 +0100 (CET) Subject: [Pypi-checkins] r1025 - trunk/pypi Message-ID: <3V7w7C3hqBzQJj@mail.python.org> Author: martin.von.loewis Date: Thu Mar 15 17:22:15 2012 New Revision: 1025 Removed: trunk/pypi/ Log: Code is now in Mercurial. From python-checkins at python.org Thu Mar 15 17:32:15 2012 From: python-checkins at python.org (martin.von.loewis) Date: Thu, 15 Mar 2012 17:32:15 +0100 (CET) Subject: [Pypi-checkins] r1026 - trunk/appengine Message-ID: <3V7wLl1BrHzQRl@mail.python.org> Author: martin.von.loewis Date: Thu Mar 15 17:32:14 2012 New Revision: 1026 Removed: trunk/appengine/ Log: Code is now in https://bitbucket.org/loewis/pypi-appengine