X-Git-Url: https://git.ozlabs.org/?a=blobdiff_plain;f=apps%2Fpatchwork%2Fbin%2Fparsemail.py;h=68bd94cd37174a4f25851f2a2596cea2e56f34d9;hb=7d9334e879857f8a380bc9509b6cbf9972cecc25;hp=13c6d783d5a5b7329ef8e144de21f9166c9d60aa;hpb=04767c65175f2a76127cd78649c5d92a82a39bb6;p=patchwork diff --git a/apps/patchwork/bin/parsemail.py b/apps/patchwork/bin/parsemail.py index 13c6d78..68bd94c 100755 --- a/apps/patchwork/bin/parsemail.py +++ b/apps/patchwork/bin/parsemail.py @@ -26,25 +26,48 @@ import time import operator from email import message_from_file try: - from email.header import Header + from email.header import Header, decode_header from email.utils import parsedate_tz, mktime_tz except ImportError: # Python 2.4 compatibility - from email.Header import Header + from email.Header import Header, decode_header from email.Utils import parsedate_tz, mktime_tz -from patchparser import parse_patch +from patchwork.parser import parse_patch from patchwork.models import Patch, Project, Person, Comment list_id_headers = ['List-ID', 'X-Mailing-List'] +whitespace_re = re.compile('\s+') +def normalise_space(str): + return whitespace_re.sub(' ', str).strip() + +def clean_header(header): + """ Decode (possibly non-ascii) headers """ + + def decode(fragment): + (frag_str, frag_encoding) = fragment + if frag_encoding: + return frag_str.decode(frag_encoding) + return frag_str.decode() + + fragments = map(decode, decode_header(header)) + + return normalise_space(u' '.join(fragments)) + def find_project(mail): project = None - listid_re = re.compile('.*<([^>]+)>.*', re.S) + listid_res = [re.compile('.*<([^>]+)>.*', re.S), + re.compile('^([\S]+)$', re.S)] for header in list_id_headers: if header in mail: - match = listid_re.match(mail.get(header)) + + for listid_re in listid_res: + match = listid_re.match(mail.get(header)) + if match: + break + if not match: continue @@ -60,7 +83,7 @@ def find_project(mail): def find_author(mail): - from_header = mail.get('From').strip() + from_header = clean_header(mail.get('From')) (name, email) = (None, None) # tuple of (regex, fn) @@ -93,7 +116,7 @@ def find_author(mail): new_person = False try: - person = Person.objects.get(email = email) + person = Person.objects.get(email__iexact = email) except Person.DoesNotExist: person = Person(name = name, email = email) new_person = True @@ -103,7 +126,6 @@ def find_author(mail): def mail_date(mail): t = parsedate_tz(mail.get('Date', '')) if not t: - print "using now()" return datetime.datetime.utcnow() return datetime.datetime.utcfromtimestamp(mktime_tz(t)) @@ -121,21 +143,21 @@ def find_content(project, mail): if part.get_content_maintype() != 'text': continue - #print "\t%s, %s" % \ - # (part.get_content_subtype(), part.get_content_charset()) - + payload = part.get_payload(decode=True) charset = part.get_content_charset() - if not charset: - charset = mail.get_charset() - if not charset: + subtype = part.get_content_subtype() + + # if we don't have a charset, assume utf-8 + if charset is None: charset = 'utf-8' - payload = unicode(part.get_payload(decode=True), charset, "replace") + if not isinstance(payload, unicode): + payload = unicode(payload, charset) - if part.get_content_subtype() == 'x-patch': + if subtype in ['x-patch', 'x-diff']: patchbuf = payload - if part.get_content_subtype() == 'plain': + elif subtype == 'plain': if not patchbuf: (patchbuf, c) = parse_patch(payload) else: @@ -149,15 +171,15 @@ def find_content(project, mail): if patchbuf: mail_headers(mail) - patch = Patch(name = clean_subject(mail.get('Subject')), - content = patchbuf, date = mail_date(mail), - headers = mail_headers(mail)) + name = clean_subject(mail.get('Subject'), [project.linkname]) + patch = Patch(name = name, content = patchbuf, + date = mail_date(mail), headers = mail_headers(mail)) if commentbuf: if patch: - cpatch = patch - else: - cpatch = find_patch_for_comment(mail) + cpatch = patch + else: + cpatch = find_patch_for_comment(project, mail) if not cpatch: return (None, None) comment = Comment(patch = cpatch, date = mail_date(mail), @@ -166,7 +188,7 @@ def find_content(project, mail): return (patch, comment) -def find_patch_for_comment(mail): +def find_patch_for_comment(project, mail): # construct a list of possible reply message ids refs = [] if 'In-Reply-To' in mail: @@ -184,14 +206,14 @@ def find_patch_for_comment(mail): # first, check for a direct reply try: - patch = Patch.objects.get(msgid = ref) + patch = Patch.objects.get(project = project, msgid = ref) return patch except Patch.DoesNotExist: pass # see if we have comments that refer to a patch try: - comment = Comment.objects.get(msgid = ref) + comment = Comment.objects.get(patch__project = project, msgid = ref) return comment.patch except Comment.DoesNotExist: pass @@ -199,23 +221,111 @@ def find_patch_for_comment(mail): return None -re_re = re.compile('^(re|fwd?)[:\s]\s*', re.I) -prefix_re = re.compile('^\[.*\]\s*') -whitespace_re = re.compile('\s+') +split_re = re.compile('[,\s]+') + +def split_prefixes(prefix): + """ Turn a prefix string into a list of prefix tokens + + >>> split_prefixes('PATCH') + ['PATCH'] + >>> split_prefixes('PATCH,RFC') + ['PATCH', 'RFC'] + >>> split_prefixes('') + [] + >>> split_prefixes('PATCH,') + ['PATCH'] + >>> split_prefixes('PATCH ') + ['PATCH'] + >>> split_prefixes('PATCH,RFC') + ['PATCH', 'RFC'] + >>> split_prefixes('PATCH 1/2') + ['PATCH', '1/2'] + """ + matches = split_re.split(prefix) + return [ s for s in matches if s != '' ] -def clean_subject(subject): +re_re = re.compile('^(re|fwd?)[:\s]\s*', re.I) +prefix_re = re.compile('^\[([^\]]*)\]\s*(.*)$') + +def clean_subject(subject, drop_prefixes = None): + """ Clean a Subject: header from an incoming patch. + + Removes Re: and Fwd: strings, as well as [PATCH]-style prefixes. By + default, only [PATCH] is removed, and we keep any other bracketed data + in the subject. If drop_prefixes is provided, remove those too, + comparing case-insensitively. + + >>> clean_subject('meep') + 'meep' + >>> clean_subject('Re: meep') + 'meep' + >>> clean_subject('[PATCH] meep') + 'meep' + >>> clean_subject('[PATCH] meep \\n meep') + 'meep meep' + >>> clean_subject('[PATCH RFC] meep') + '[RFC] meep' + >>> clean_subject('[PATCH,RFC] meep') + '[RFC] meep' + >>> clean_subject('[PATCH,1/2] meep') + '[1/2] meep' + >>> clean_subject('[PATCH RFC 1/2] meep') + '[RFC,1/2] meep' + >>> clean_subject('[PATCH] [RFC] meep') + '[RFC] meep' + >>> clean_subject('[PATCH] [RFC,1/2] meep') + '[RFC,1/2] meep' + >>> clean_subject('[PATCH] [RFC] [1/2] meep') + '[RFC,1/2] meep' + >>> clean_subject('[PATCH] rewrite [a-z] regexes') + 'rewrite [a-z] regexes' + >>> clean_subject('[PATCH] [RFC] rewrite [a-z] regexes') + '[RFC] rewrite [a-z] regexes' + >>> clean_subject('[foo] [bar] meep', ['foo']) + '[bar] meep' + >>> clean_subject('[FOO] [bar] meep', ['foo']) + '[bar] meep' + """ + + if drop_prefixes is None: + drop_prefixes = [] + else: + drop_prefixes = [ s.lower() for s in drop_prefixes ] + + drop_prefixes.append('patch') + + # remove Re:, Fwd:, etc subject = re_re.sub(' ', subject) - subject = prefix_re.sub('', subject) - subject = whitespace_re.sub(' ', subject) - return subject.strip() -sig_re = re.compile('^(-{2,3} ?|_+)\n.*', re.S | re.M) + subject = normalise_space(subject) + + prefixes = [] + + match = prefix_re.match(subject) + + while match: + prefix_str = match.group(1) + prefixes += [ p for p in split_prefixes(prefix_str) \ + if p.lower() not in drop_prefixes] + + subject = match.group(2) + match = prefix_re.match(subject) + + subject = normalise_space(subject) + + subject = subject.strip() + if prefixes: + subject = '[%s] %s' % (','.join(prefixes), subject) + + return subject + +sig_re = re.compile('^(-- |_+)\n.*', re.S | re.M) def clean_content(str): + """ Try to remove signature (-- ) and list footer (_____) cruft """ str = sig_re.sub('', str) return str.strip() -def main(args): - mail = message_from_file(sys.stdin) +def parse_mail(mail): # some basic sanity checks if 'From' not in mail: @@ -253,7 +363,7 @@ def main(args): try: patch.save() except Exception, ex: - print ex.message + print str(ex) if comment: if save_required: @@ -267,9 +377,13 @@ def main(args): try: comment.save() except Exception, ex: - print ex.message + print str(ex) return 0 +def main(args): + mail = message_from_file(sys.stdin) + return parse_mail(mail) + if __name__ == '__main__': sys.exit(main(sys.argv))