X-Git-Url: https://git.ozlabs.org/?a=blobdiff_plain;f=apps%2Fpatchwork%2Fbin%2Fparsemail.py;h=e2beeae1b752d8ce03d64ee77d839caf294cd09d;hb=07ded8bef209073969ca66049c91783ce59fc0f7;hp=13c6d783d5a5b7329ef8e144de21f9166c9d60aa;hpb=04767c65175f2a76127cd78649c5d92a82a39bb6;p=patchwork diff --git a/apps/patchwork/bin/parsemail.py b/apps/patchwork/bin/parsemail.py index 13c6d78..e2beeae 100755 --- a/apps/patchwork/bin/parsemail.py +++ b/apps/patchwork/bin/parsemail.py @@ -26,18 +26,35 @@ import time import operator from email import message_from_file try: - from email.header import Header + from email.header import Header, decode_header from email.utils import parsedate_tz, mktime_tz except ImportError: # Python 2.4 compatibility - from email.Header import Header + from email.Header import Header, decode_header from email.Utils import parsedate_tz, mktime_tz -from patchparser import parse_patch +from patchwork.parser import parse_patch from patchwork.models import Patch, Project, Person, Comment list_id_headers = ['List-ID', 'X-Mailing-List'] +whitespace_re = re.compile('\s+') +def normalise_space(str): + return whitespace_re.sub(' ', str).strip() + +def clean_header(header): + """ Decode (possibly non-ascii) headers """ + + def decode(fragment): + (frag_str, frag_encoding) = fragment + if frag_encoding: + return frag_str.decode(frag_encoding) + return frag_str.decode() + + fragments = map(decode, decode_header(header)) + + return normalise_space(u' '.join(fragments)) + def find_project(mail): project = None listid_re = re.compile('.*<([^>]+)>.*', re.S) @@ -60,7 +77,7 @@ def find_project(mail): def find_author(mail): - from_header = mail.get('From').strip() + from_header = clean_header(mail.get('From')) (name, email) = (None, None) # tuple of (regex, fn) @@ -93,7 +110,7 @@ def find_author(mail): new_person = False try: - person = Person.objects.get(email = email) + person = Person.objects.get(email__iexact = email) except Person.DoesNotExist: person = Person(name = name, email = email) new_person = True @@ -103,7 +120,6 @@ def find_author(mail): def mail_date(mail): t = parsedate_tz(mail.get('Date', '')) if not t: - print "using now()" return datetime.datetime.utcnow() return datetime.datetime.utcfromtimestamp(mktime_tz(t)) @@ -149,14 +165,14 @@ def find_content(project, mail): if patchbuf: mail_headers(mail) - patch = Patch(name = clean_subject(mail.get('Subject')), - content = patchbuf, date = mail_date(mail), - headers = mail_headers(mail)) + name = clean_subject(mail.get('Subject'), [project.linkname]) + patch = Patch(name = name, content = patchbuf, + date = mail_date(mail), headers = mail_headers(mail)) if commentbuf: if patch: - cpatch = patch - else: + cpatch = patch + else: cpatch = find_patch_for_comment(mail) if not cpatch: return (None, None) @@ -199,18 +215,107 @@ def find_patch_for_comment(mail): return None -re_re = re.compile('^(re|fwd?)[:\s]\s*', re.I) -prefix_re = re.compile('^\[.*\]\s*') -whitespace_re = re.compile('\s+') +split_re = re.compile('[,\s]+') + +def split_prefixes(prefix): + """ Turn a prefix string into a list of prefix tokens + + >>> split_prefixes('PATCH') + ['PATCH'] + >>> split_prefixes('PATCH,RFC') + ['PATCH', 'RFC'] + >>> split_prefixes('') + [] + >>> split_prefixes('PATCH,') + ['PATCH'] + >>> split_prefixes('PATCH ') + ['PATCH'] + >>> split_prefixes('PATCH,RFC') + ['PATCH', 'RFC'] + >>> split_prefixes('PATCH 1/2') + ['PATCH', '1/2'] + """ + matches = split_re.split(prefix) + return [ s for s in matches if s != '' ] -def clean_subject(subject): +re_re = re.compile('^(re|fwd?)[:\s]\s*', re.I) +prefix_re = re.compile('^\[([^\]]*)\]\s*(.*)$') + +def clean_subject(subject, drop_prefixes = None): + """ Clean a Subject: header from an incoming patch. + + Removes Re: and Fwd: strings, as well as [PATCH]-style prefixes. By + default, only [PATCH] is removed, and we keep any other bracketed data + in the subject. If drop_prefixes is provided, remove those too, + comparing case-insensitively. + + >>> clean_subject('meep') + 'meep' + >>> clean_subject('Re: meep') + 'meep' + >>> clean_subject('[PATCH] meep') + 'meep' + >>> clean_subject('[PATCH] meep \\n meep') + 'meep meep' + >>> clean_subject('[PATCH RFC] meep') + '[RFC] meep' + >>> clean_subject('[PATCH,RFC] meep') + '[RFC] meep' + >>> clean_subject('[PATCH,1/2] meep') + '[1/2] meep' + >>> clean_subject('[PATCH RFC 1/2] meep') + '[RFC,1/2] meep' + >>> clean_subject('[PATCH] [RFC] meep') + '[RFC] meep' + >>> clean_subject('[PATCH] [RFC,1/2] meep') + '[RFC,1/2] meep' + >>> clean_subject('[PATCH] [RFC] [1/2] meep') + '[RFC,1/2] meep' + >>> clean_subject('[PATCH] rewrite [a-z] regexes') + 'rewrite [a-z] regexes' + >>> clean_subject('[PATCH] [RFC] rewrite [a-z] regexes') + '[RFC] rewrite [a-z] regexes' + >>> clean_subject('[foo] [bar] meep', ['foo']) + '[bar] meep' + >>> clean_subject('[FOO] [bar] meep', ['foo']) + '[bar] meep' + """ + + if drop_prefixes is None: + drop_prefixes = [] + else: + drop_prefixes = [ s.lower() for s in drop_prefixes ] + + drop_prefixes.append('patch') + + # remove Re:, Fwd:, etc subject = re_re.sub(' ', subject) - subject = prefix_re.sub('', subject) - subject = whitespace_re.sub(' ', subject) - return subject.strip() -sig_re = re.compile('^(-{2,3} ?|_+)\n.*', re.S | re.M) + subject = normalise_space(subject) + + prefixes = [] + + match = prefix_re.match(subject) + + while match: + prefix_str = match.group(1) + prefixes += [ p for p in split_prefixes(prefix_str) \ + if p.lower() not in drop_prefixes] + + subject = match.group(2) + match = prefix_re.match(subject) + + subject = normalise_space(subject) + + subject = subject.strip() + if prefixes: + subject = '[%s] %s' % (','.join(prefixes), subject) + + return subject + +sig_re = re.compile('^(-- |_+)\n.*', re.S | re.M) def clean_content(str): + """ Try to remove signature (-- ) and list footer (_____) cruft """ str = sig_re.sub('', str) return str.strip() @@ -253,7 +358,7 @@ def main(args): try: patch.save() except Exception, ex: - print ex.message + print str(ex) if comment: if save_required: @@ -267,7 +372,7 @@ def main(args): try: comment.save() except Exception, ex: - print ex.message + print str(ex) return 0