syntax error line 72?
Nigel Hewett
nigel.hewett at kuhuna.com
Sun Jan 6 16:07:11 EST 2002
can anyone assist running 1.5.2 python on raq4 with this script and
getting -
ears the script it syntax errors on line 72
while (t < cur_time()):
t += 86400
return t
the script >>>>>
#!/usr/bin/python
#
# Copyright (C) 2000-2001 Jasper Jongmans. All rights reserved.
#
# $Id: frontend.py,v 1.32 2001/05/16 13:43:38 aprogas Exp $
#
### IMPORTS ### {{{1
# official modules
import cgi
import ftplib
import httplib
import robotparser
import shelve
import string
import sys
import time
import urlparse
import whrandom
# project modules
#sys.path.append('/home/webget/www/webget/') # uncomment if core.py is not
in the same dir as frontend.py/backend.py
from core import *
# }}}1
### GLOBALS ### {{{1
db_file = webget_path+'webget.db' # use an absolute path
# }}}1
### FUNCTIONS ### {{{1
def add(uri, fetch_time): # {{{2
'Puts the specified URI in the queue to be fetched at the specified time'
id = give_id()
while db.has_key(id): id = give_id() # FIXME - will loop forever if all IDs
are taken
status, filesize = prefetch(uri)
add_time = cur_time()
last_change_time = cur_time()
filerrors = []
db[id] = uri, status, filesize, fetch_time, add_time, last_change_time,
filerrors
# }}}2
def delete(id): # {{{2
'Removes the specified ID from the database file'
# For this functions it is useless to update the last_change_time.
del db[id]
# }}}2
def epoch2human_time(t): # {{{2
'Converts a epoch-time (in whole seconds) to a human-readable time'
t = time.localtime(t)
t = time.strftime('%Y/%m/%d %H:%M', t)
return t
# }}}2
def give_id(): # {{{2
'Returns a random integer'
return whrandom.randint(1,65535)
# }}}2
def input2epoch_time(t): # {{{2
'Converts a shutdown(8)-like time to an epoch-time (in whole seconds)'
Y, m, d, H, M, s, w, j, D = time.localtime(cur_time())
try:
if len(t) == 12: Y, m, d, H, M = time.strptime(t, '%Y%m%d%H%M')[:5]
elif len(t) == 10: Y, m, d, H, M = time.strptime(t, '%y%m%d%H%M')[:5]
elif len(t) == 8: m, d, H, M = time.strptime(t, '%m%d%H%M')[1:5]
elif len(t) == 6: d, H, M = time.strptime(t, '%d%H%M')[2:5]
elif len(t) == 4: H, M = time.strptime(t, '%H%M')[3:5]
elif len(t) == 2: M = time.strptime(t, '%M')[4]
else: warnings.append('Invalid fetch time, current time implied.')
except ValueError:
warnings.append('Invalid fetch time, current time implied.')
t = Y, m, d, H, M, s, w, j, D
t = time.mktime(t)
t = int(t)
while (t < cur_time()):
t += 86400
return t
# }}}2
def page_details(id): # {{{2
'Prints out detailed information about the entry specified with id'
global contents
uri, status, filesize, fetch_time, add_time, last_change_time, filerrors =
db[id]
contents += '<pre>\n'
contents += 'ID: %s\n' % id
contents += 'URI: %s\n' % uri
contents += 'Status: %s\n' % status
contents += 'Filesize: %s\n' % filesize
contents += 'Fetch time: %s\n' % epoch2human_time(fetch_time)
contents += 'Time added: %s\n' % epoch2human_time(add_time)
contents += 'Last change: %s\n' % epoch2human_time(last_change_time)
contents += 'Errors:\n'
for filerror in filerrors:
contents += ' %s\n' % filerror
contents += '</pre>\n'
contents += '<hr />\n'
# }}}2
def page_errors(errors): # {{{2
'Prints out all errors'
print '<h3>Errors</h3>'
print '<div class="errors">'
for error in errors:
print '%s<br />' % error
print '</div>'
# }}}2
def page_footer(): # {{{2
'Prints a generic page footer'
print '''\
</body>
</html>\
'''
# }}}2
def page_header(): # {{{2
'Prints a generic page header'
print '''\
Cache-Control: no-cache
Pragma: no-cache
Content-Type: text/html
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
"DTD/xhtml1-strict.dtd">
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<title>Webget %s</title>
<link rel="stylesheet" type="text/css" href="default.css" />
</head>
<body>
<h2>Webget %s</h2>
<hr />\
''' % (version, version)
# }}}2
def page_list(): # {{{2
'Prints the list of URIs'
global contents
if mass_submission:
submission = '<textarea name="uri" cols="48" rows="6" />'
toggle_submission = 'No'
else:
submission = '<input type="text" name="uri" size="36" />'
toggle_submission = 'Yes'
contents += '''\
<form action="frontend.py" method="post">
URI: %s<br />
Fetch time: <input type="text" name="fetch_time" value="%s" size="12"
maxlength="12" /> (yyyymmddhhmm)<br />
<input type="submit" name="action" value="Add" />
[<a href="?no-cache=%s">Reload</a>] [<a href="?action=setup">Setup</a>] [<a
href="?action=save&mass_submission=%s">Toggle MSM</a>]<br /><br />
<table width="100%%" cellspacing="0">
<tr class="header">
<th></th>
<th><a href="?action=save&sortby=0">URI</a></th>
<th><a href="?action=save&sortby=1">Status</a></th>
<th><a href="?action=save&sortby=2">Filesize</a></th>
<th><a href="?action=save&sortby=3">Fetch time</a></th>
<th><a href="?action=save&sortby=5">Last change</a></th>
</tr>\
''' % (submission, default_fetch_time, give_id(), toggle_submission)
even = 1
z = db.items()
z.sort(lambda x, y: cmp(x[1][sortby], y[1][sortby]))
for key in z:
id = key[0]
even = not even
if even: oddeven = 'even'
else: oddeven = 'odd'
contents += '<tr class="%s">\n' % oddeven
contents += '<td><input type="checkbox" name="id" value="%s"/></td>\n' %
id
contents += '<td><a href="frontend.py?action=details&id=%s">%s</a></td>\n'
% (id, db[id][0]) # uri
contents += '<td>%s</td>\n' % db[id][1] # status
contents += '<td>%s</td>\n'% db[id][2] # filesize
contents += '<td>%s</td>\n' % epoch2human_time(db[id][3]) # fetch_time
contents += '<td>%s</td>\n' % epoch2human_time(db[id][5]) #
last_changed_time
contents += '</tr>\n'
contents += '''\
</table><br />
<input type="submit" name="action" value="Retry">
<input type="submit" name="action" value="Delete">
</form>
<hr />\
'''
# }}}2
def page_setup(): # {{{2
'Prints out the setup page'
global contents
opt_sortby = '''\
<option value="0">0: URI</option>
<option value="1">1: Status</option>
<option value="2">2: Filesize</option>
<option value="3">3: Fetch time</option>
<option value="4">4: Time added</option>
<option value="5">5: Last change</option>
<option value="6">6: Errors</option>\
'''
opt_sortby = string.replace(opt_sortby, '<option value="%s">' % sortby,
'<option value="%s" selected="selected">' % sortby)
opt_mass_submission = '<input type="radio" name="mass_submission"
value="Yes">Yes</input><input type="radio" name="mass_submission"
value="No">No</input>'
if mass_submission: opt_mass_submission =
string.replace(opt_mass_submission, 'value="Yes"', 'value="Yes"
checked="checked"')
else: opt_mass_submission = string.replace(opt_mass_submission,
'value="No"', 'value="No" checked="checked"')
contents += '''\
<form action="frontend.py" method="post">
Default fetch time: <input type="text" name="default_fetch_time" value="%s"
size="12" maxlength="12" /> (yyyymmddhhmm)<br />
Sort URI list by: <select name="sortby">%s</select> (only some of these
values are shown in the list)<br />
Mass submission: %s<br />
<input type="submit" name="action" value="Save" />
</form>
<hr />\
''' % (default_fetch_time, opt_sortby, opt_mass_submission)
# }}}2
def page_warnings(warnings): # {{{2
'Prints out all warnings'
print '<h3>Warnings</h3>'
print '<div class="warnings">'
for warning in warnings:
print '%s<br />' % warning
print '</div>\n'
# }}}2
def prefetch(uri): # {{{2
'Prefetches the specified URI to get certain information about it'
scheme, host, file = urlparse.urlparse(uri)[:3]
if scheme == '': scheme = 'http'
if scheme == 'http':
r = robotparser.RobotFileParser()
r.set_url(urlparse.urlunparse((scheme, host, '/robots.txt', '', '', '')))
r.read()
if not r.can_fetch("Webget", urlparse.urlunparse((scheme, host, file, '',
'', ''))):
errors.append('Invalid URI, Robot Exclusion Standard.')
status = 'ERROR'
filesize = None
return status, filesize
h = httplib.HTTP(host)
h.putrequest('HEAD', file)
h.putheader('Host', host)
h.putheader('User-Agent', 'Webget/%s' % version)
h.endheaders()
code, msg, headers = h.getreply()
filesize = int(headers.getheader('Content-Length') or -1)
elif scheme == 'ftp':
f = ftplib.FTP(host)
f.login()
filesize = f.size(file)
f.close()
else:
errors.append('Unknown URI scheme.')
status = 'ERROR'
filesize = None
return status, filesize
status = 'PENDING'
return status, filesize
# }}}2
def retry(id, fetch_time): # {{{2
'Requeue the specified ID for fetching'
uri, status, filesize, blackhole, add_time, last_change_time, filerrors =
db[id]
status = 'PENDING'
last_change_time = cur_time()
db[id] = uri, status, filesize, fetch_time, add_time, last_change_time,
filerrors
# }}}2
# }}}1
### MAIN PROGRAM ### {{{1
realdb = shelve.open(db_file)
db = shelve2dict(realdb) # mirror into a dict primarly for sorting and speed
if not (db.has_key(0) and type(db[0]) == type(()) and len(db[0]) == 3): #
sanity check
db[0] = '0400', 3, 0 # set default settings
default_fetch_time, sortby, mass_submission = db[0] # import saved settings
del db[0] # the settings field is abnormal, so avoid problems
contents = ''
warnings = []
errors = []
form = cgi.FieldStorage()
form_ok = 1
if string.lower(form.getvalue('action', '')) == 'add': # {{{2
if form.has_key('fetch_time'):
fetch_time = input2epoch_time(form.getvalue('fetch_time'))
elif default_fetch_time:
warnings.append('Unknown fetch time, default time implied.')
fetch_time = input2epoch_time(default_fetch_time)
else:
warnings.append('Unknown fetch time, current time implied.')
fetch_time = cur_time()
if form.has_key('uri'):
for uri in string.split(form.getvalue('uri')):
add(uri, fetch_time)
else:
errors.append('Unknown URI.')
page_list()
# }}}2
elif string.lower(form.getvalue('action', '')) == 'details': # {{{2
if form.has_key('id'):
id = int(form.getvalue('id'))
if db.has_key(id):
page_details(id)
else:
errors.append('Nonexistent ID.')
else:
errors.append('Unknown ID.')
# }}}2
elif string.lower(form.getvalue('action', '')) == 'delete': # {{{2
if form.has_key('id'):
ids = []
value = form.getvalue('id')
if type(value) == type([]):
ids = value
else:
ids.append(value)
for id in ids:
id = int(id)
if db.has_key(id):
delete(id)
else:
errors.append('Nonexistent ID.')
else:
errors.append('Unknown ID.')
page_list()
# }}}2
elif string.lower(form.getvalue('action', '')) == 'retry': # {{{2
if form.has_key('fetch_time'):
fetch_time = input2epoch_time(form.getvalue('fetch_time'))
elif default_fetch_time:
warnings.append('Unknown fetch time, default time implied.')
fetch_time = input2epoch_time(default_fetch_time)
else:
warnings.append('Unknown fetch time, current time implied.')
fetch_time = cur_time()
if form.has_key('id'):
ids = []
value = form.getvalue('id')
if type(value) == type([]):
ids = value
else:
ids.append(value)
for id in ids:
id = int(id)
if db.has_key(id):
retry(id, fetch_time)
else:
errors.append('Nonexistent ID.')
else:
errors.append('Unknown ID.')
page_list()
# }}}2
elif string.lower(form.getvalue('action', '')) == 'save': # {{{2
if form.has_key('default_fetch_time'):
default_fetch_time = form.getvalue('default_fetch_time')
if form.has_key('sortby'):
sortby = int(form.getvalue('sortby'))
if form.has_key('mass_submission'):
if form.getvalue('mass_submission') == 'Yes':
mass_submission = 1
else:
mass_submission = 0
page_list()
# }}}2
elif string.lower(form.getvalue('action', '')) == 'setup': # {{{2
page_setup()
# }}}2
else: # {{{2
page_list()
# }}}2
page_header()
print contents
page_warnings(warnings)
page_errors(errors)
page_footer()
dict2shelve(db, realdb)
realdb['0'] = default_fetch_time, sortby, mass_submission
realdb.close()
# }}}1
More information about the Python-list
mailing list