[Baypiggies] More dramatic material?

[Bill Janssen]

Just for fun, I grepped the UpLib Python sources for re.compile, and
here's what I found.  I usually use them to pull a field or fields out
of a complicated expression.  I'm particularly fond of the twisted and
almost certainly broken-in-some-way URLPATTERN in plibUtil.py.

Those interested in more aggressive use of regular expressions,
particularly for natural language work, might want to look at this book:

  http://www.stanford.edu/~laurik/fsmbook/home.html

or this paper

  http://www.cs.cmu.edu/afs/cs/user/alavie/11-711/711-cmt/Class-notes/lauri-et-al.pdf

This software is available as a Python module at

  http://www.stanford.edu/~laurik/.book2software

Bill
------------------------------------------------------------------------
addDocument.py:newline_expression = re.compile(r'(\s*\n)+')
addDocument.py:whitespace_expression = re.compile(r'\s+')
addDocument.py:charset_pattern = re.compile(r"^Content-Type:\s*text/plain;\s*charset=([^)]*)\n", re.IGNORECASE)
addDocument.py:IMAGE_EXTENSIONS = re.compile(r'((\.gif)|(\.png)|(\.jpeg)|(\.jpg)|(\.tif)|(\.tiff)|(\.pbm)|(\.pnm)|(\.pgm)|(\.ppm)|(\.bmp)|(\.eps)|(\.rast)|(\.xbm)|(\.xpm))$', re.IGNORECASE)
addDocument.py:TIFF_EXTENSION = re.compile(r'((\.tiff)|(\.tif))$', re.IGNORECASE)
addDocument.py:PDF_EXTENSION = re.compile(r'\.pdf', re.IGNORECASE)
addDocument.py:POSTSCRIPT_EXTENSION = re.compile(r'\.ps', re.IGNORECASE)
addDocument.py:HTML_EXTENSION = re.compile(r'((\.html)|(\.htm))$', re.IGNORECASE)
addDocument.py:MS_EXTENSIONS = re.compile(r'((\.ppt)|(\.pps)|(\.doc)|(\.dot)|(\.xls)|(\.xlt)|(\.docx)|(\.pptx)|(\.ppsx)|(\.xlsx))$', re.IGNORECASE)
addDocument.py:FILE_TEXT_ENDING = re.compile(r'\stext$', re.IGNORECASE)
addDocument.py:URL_PREFIX = re.compile(r'^((http:)|(https:)|(ftp:))')
addDocument.py:    SEPARATOR_PAGE_PATTERN = re.compile('(?:^|\f)([^\f]*UpLib Scan Separator Sheet){4}[^\f]*', re.MULTILINE)
addDocument.py:    FORMFEED_PATTERN = re.compile('\f')
addDocument.py:    TEXT_EXTENSION = re.compile(r'((\.txt)|(\.text))$', re.IGNORECASE)
addDocument.py:    CARD_SIZE_ENDING = re.compile(r'.*\.([0-9]+)x([0-9]+)$')
angelHandler.py:CONTENT_LENGTH = re.compile ('Content-Length: ([0-9]+)', re.IGNORECASE)
angelHandler.py:CONTENT_TYPE = re.compile ('Content-Type: (.+)', re.IGNORECASE)
angelHandler.py:USER_AGENT = re.compile ('User-Agent: (.+)', re.IGNORECASE)
angelHandler.py:HOST_HEADER = re.compile ('Host: (.+)', re.IGNORECASE)
angelHandler.py:ACCEPTS_HEADER = re.compile ('Accept: (.+)', re.IGNORECASE)
angelHandler.py:DOC_CHANGES_SINCE_HEADER = re.compile ('X-UpLib-Docs-Modified-Since: (.+)', re.IGNORECASE)
angelHandler.py:TOUCHPATH = re.compile('/docs/(?P<docid>[^/]+)/(.*\.(html|pdf|tiff)|contents.txt)')
angelHandler.py:DOCPATH = re.compile('/docs/(?P<docid>[^/]+)/.*\.(html|pdf|tiff|txt|png|bboxes)')
angelHandler.py:COOKIE = re.compile('Cookie: (.+)', re.IGNORECASE)
angelHandler.py:PASSWORD = re.compile('Password: (.+)', re.IGNORECASE)
basicPlugins.py:SENSIBLE_BROWSERS = re.compile("Safari|Camino|Firefox|Konqueror|Gecko/|Opera")
basicPlugins.py:SIMPLE_PAGE_NUMBER = re.compile(r"^-?[0-9]+$")
basicPlugins.py:PAGE_RANGE = re.compile(r"^([0-9]+)(-|--)([0-9]+)$")
basicPlugins.py:PAGE_NUMBERS = re.compile(r"^((b,[0-9]*,[0-9]+(-[0-9]+)*)|(d,[0-9]+,[0-9]+(-[0-9]+)*)|(r,[0-9]+,[0-9]+(-[0-9]+)*))(;"
categories.py:            re.compile(x.strip()) for x in configurator.default_configurator().get("categorize-excluded-categories", "").split(",") if x.strip()]}
createHotSpots.py:URLPATTERN = re.compile(r'(^|\W)(?P<scheme>http|ftp|https)://(?P<host>[^/\s:]+)(:(?P<port>[0-9]+)){0,1}(?P<path>(/[-+~&$/\w,.?=%#]*[-+~$/\w?=%#])|\s*)')
createHotSpots.py:CHARSET_PATTERN = re.compile(r"^Content-Type:\s*text/plain;\s*charset=([^)]*)\n", re.IGNORECASE)
createPageBboxes.py:CHARSETPATTERN = re.compile(r"^Content-Type:\s*text/plain;\s*charset=([^)]*)\n", re.IGNORECASE)
createPageBboxes.py:    FRAGMENT_PATTERN = re.compile(r"^\|([0-9]+)\|(0|1)\|(0|1)\|[01]\|(0|1)\|([-A-Za-z0-9]+)\|([0-9]+)\|(.+)$")
createThumbnails.py:PAGE_RANGE = re.compile(r"^([0-9]+)(-|--)([0-9]+)$")
createThumbnails.py:PAGE_NUMBERS = re.compile(r"^((b,[0-9]*,[0-9]+(-[0-9]+)*)|(d,[0-9]+,[0-9]+(-[0-9]+)*)|(r,[0-9]+,[0-9]+(-[0-9]+)*))(;"
document.py:CHARSET_PATTERN = re.compile(r"^Content-Type:\s*text/plain;\s*charset=([^)]*)\n", re.IGNORECASE)
document.py:LANGUAGE_PATTERN = re.compile(r"^Content-Language:\s*(.*)\n", re.IGNORECASE)
document.py:NEWLINE_PATTERN = re.compile(r"([^\r])\n", re.MULTILINE)
emailParser.py:SUBJ_CRUFT_PATTERN = re.compile(r"^((?P<re>[rR][eE]:)\s+)?\[[^]]+\]\s(?P<subj>.*)$")
emailParser.py:HEADER_NEWLINE_PATTERN = re.compile("\n\s+", (re.MULTILINE|re.DOTALL))
extensions.py:CODE_EXCLUDES = re.compile(r"(.*~$)|(.*\.class$)|(.*\.pyc$)")
indexing.py:PAGEBREAK = re.compile('\n\f\\r?\n')
indexing.py:                splitter = re.compile(name[p+1:])
pageview.py:CONTENT_TYPE = re.compile ('Content-Type: (.+)', re.IGNORECASE)
paragraphs.py:REFERENCE_INDICATOR = re.compile('(\[[0-9]+\])|([0-9]+\.)|(\[[A-Za-z0-9]+\])')
paragraphs.py:BULLET_INDICATOR = re.compile(u'\u2022\s+')
paragraphs.py:IMAGE_TYPES = re.compile('(image/.*)|(application/vnd.ms-powerpoint)')
plibUtil.py:URLPATTERN = re.compile(r'(^|\W)(?P<url>(?P<scheme>http|ftp|https)://(?P<host>[-a-z0-9A-Z_\.]+[a-zA-Z0-9])(:(?P<port>[0-9]+)){0,1}(?P<path>(/[-+~&$/\w,.?=%#]*[-+~$/\w?=%#])|))')
plibUtil.py:EMAILPATTERN = re.compile(r'(?P<email>(?P<mailbox>[-a-zA-Z0-9._]+)@(?P<domain>[-a-zA-Z0-9_$.]+\.((com)|(edu)|(net)|(info)|(org)|(name)|([A-Za-z]{2})){1}))')
plibUtil.py:GEMAILPATTERN = re.compile(r'({(?P<group>([-a-zA-Z0-9._]+(\s*,\s*[-a-zA-Z0-9._]+)+))}@(?P<domain>[-a-zA-Z0-9_$.]+\.(com|edu|net|info|org|name|[A-Za-z]{2}){1}))')
plibUtil.py:NOTELINE_BEGINNING = re.compile(r'(?P<linestart>\A|\n)(?P<firstchar>.)', re.MULTILINE)
plibUtil.py:DOC_ID_RE = re.compile('[0-9]{5}-[0-9]{2}-[0-9]{4}-[0-9]{3}')
plibUtil.py:COLL_ID_RE = re.compile('[0-9]{5}-[0-9]{2}-[0-9]{4}-[0-9]{3}-[0-9]+')
plibUtil.py:CHARSET_PATTERN = re.compile(r"^Content-Type:\s*text/plain;\s*charset=([^)]*)\n", re.IGNORECASE)
plibUtil.py:LANGUAGE_PATTERN = re.compile(r"^Content-Language:\s*(.*)\n", re.IGNORECASE)
plibUtil.py:PAGENUMBER = re.compile(r'page(\d+)\.png')
plibUtil.py:UTF_8_EXPR = re.compile(r'^=\?UTF-8\?([BQbq])\?(.*)\?=$')
plibUtil.py:CHARSETPATTERN = re.compile(r"^Content-Type:\s*text/plain;\s*charset=([^)]*)\n", re.IGNORECASE)
ripper.py:newline_expression = re.compile(r'(\s*\n)+')
ripper.py:whitespace_expression = re.compile(r'\s+')
ripper.py:charset_pattern = re.compile(r"^Content-Type:\s*text/plain;\s*charset=([^)]*)\n", re.IGNORECASE)
roman.py:romanNumeralPattern = re.compile('''
startAngel.py:    update_directory_tree(dist_docfiles_dir, docroot, re.compile(r'.*\.pdf$|.*\.html$|.*\.css$|.*\.png$|.*\.js$|.*\.txt$'))
webutils.py:_HTMLCHARREFPATTERN = re.compile("&#?[A-Z0-9a-z]+;")
webutils.py:LOCATION_RE = re.compile('((?P<userinfo>[^@]+)@)?(?P<host>[-A-Z._a-z0-9]+)(:(?P<port>[0-9]+))?')
webutils.py:    COOKIE_LINE = re.compile("([^\s]+)\s+(TRUE|FALSE)\s+([^\s]+)\s+(TRUE|FALSE)\s+([0-9]+)\s+([^\s]+)\s+(.*)$")
webutils.py:    DATA_URI_PATTERN = re.compile(r'data:(?P<maintype>[^/;]+?)/(?P<subtype>[^/;]+?)(?P<params>;[^;]+?)*?(;(?P<encoding>base64))?,(?P<data>.*)$', re.MULTILINE | re.DOTALL)
webutils.py:    CDATA_EXPR = re.compile(r'(/\*\<\!\[CDATA\[\*/)|(/\*\]\]\>\*/)|(//\<\!\[CDATA\[)|(//\]\]\>)')
webutils.py:    CSS_URL = re.compile(r"\s+url\(['\"]?([^)'\"]+)['\"]?\)")
webutils.py:    CSS_IMPORT = re.compile(r'(@import\s+"(?P<url1>[^"]+)"\s*;)|(@import\s+url\((?P<url2>[^)]+)\)\s*;)', re.MULTILINE)
webutils.py:    CDATA_EXPR = re.compile(r'(/\*\<\!\[CDATA\[\*/)|(/\*\]\]\>\*/)|(//\<\!\[CDATA\[)|(//\]\]\>)')
xdocParser.py:MARKUP = re.compile(r"\[(([XYRa-z];[^\[\]]+\])|([A-Z]))", re.MULTILINE)
xdocParser.py:FONT = re.compile(r"\[f;([0-9]+);\"([^\"]+)\";([A-Z]);([a-z]);[0-9]+;(F|V);[0-9]+;[0-9]+;[0-9]+;([0-9]+);[0-9]+\]")
xdocParser.py:PAGE = re.compile(r"\[p;([0-9]+);(P|L);[0-9]+;[A-Z]+;([-0-9\.]+);([-0-9\.]+);([-0-9]+);([-0-9]+);([-0-9]+);([-0-9]+);[-0-9]+;[-0-9]+\]")
xdocParser.py:WBOX = re.compile(r"\[b;([0-9]+);([0-9]+);([0-9]+);([0-9]+);([0-9]+);([0-9]+)\]")
xdocParser.py:REGION = re.compile(r"\[t;([0-9]+);[0-9]+;([0-9]+);([0-9]+);[A-Z];[\"A-Za-z0-9]+;[\"A-Za-z0-9]+;[\"A-Za-z0-9]+;([0-9]+);([0-9]+);([0-9]+);([0-9]+);[0-9]+;[0-9]+\]")
xdocParser.py:LANGUAGE = re.compile(r"\[O;([0-9]+);([0-9]+)\]")