ごみです。 {{{ #!/usr/local/bin/python # -*- coding: utf-8 -*- import sys import trac.scripts.admin import cmd import getpass import os import shlex import shutil import StringIO import time import traceback import urllib import time import calendar import re import mimetypes import email #from email.Parser import Parser from email.Header import decode_header #from email.Utils import collapse_rfc2231_value from email.Utils import decode_rfc2231 import types import os import email.Errors import email.Utils import mailbox from trac import util from trac.wiki import wiki_to_html, IWikiSyntaxProvider from trac.util import Markup from trac.web.chrome import add_link, add_stylesheet, INavigationContributor, ITemplateProvider from trac.attachment import attachments_to_hdf, Attachment from trac.util import NaivePopen import tempfile import getpass, poplib try: sum except NameError: def sum(list): """Python2.2 doesn't have sum()""" tot = 0 for item in list: tot += item return tot class MailArchiveAdmin(trac.scripts.admin.TracAdmin): __env = None env = None def msgfactory(self,fp): try: return email.message_from_file(fp) except email.Errors.MessageParseError: # Don't return None since that will # stop the mailbox iterator return '' def decode_to_unicode(self, basestr): decodefrag = email.Header.decode_header(basestr) subj_fragments = ['',] for frag, enc in decodefrag: if enc: frag = self.to_unicode(frag, enc) subj_fragments.append(frag) return ''.join(subj_fragments) def to_unicode(self,text,charset): default_charset = self.env.config.get('mailarchive', 'default_charset',None) if default_charset : chaerset = default_charset # to unicode with codecaliases # codecaliases change mail charset to python charset charset = charset.lower( ) aliases = {} aliases_text = self.env.config.get('mailarchive', 'codecaliases') for alias in aliases_text.split(','): alias_s = alias.split(':') if len(alias_s) >=2: if alias_s[1] == 'cmd': aliases[alias_s[0].lower()] = ('cmd',alias_s[2]) else: aliases[alias_s[0].lower()] = ('codec',alias_s[1]) if aliases.has_key(charset): (type,alias) = aliases[charset] if type == 'codec': text = unicode(text,alias) elif type == 'cmd': np = NaivePopen(alias, text, capturestderr=1) if np.errorlevel or np.err: err = 'Running (%s) failed: %s, %s.' % (cmdline, np.errorlevel, np.err) raise Exception, err text = unicode(np.out,'utf-8') else: text = unicode(text,charset) return text def import_message(self, msg, author,mlid, db): OUTPUT_ENCODING = 'utf-8' subject = '' messageid = '' utcdate = 0 localdate = 0 zoneoffset = 0 text = '' fromtext = '' body = '' ref_messageid = '' cursor = db.cursor() is_newid = False if 'message-id' in msg: messageid = msg['message-id'] if messageid[:1] == '<': messageid = messageid[1:] if messageid[-1:] == '>': messageid = messageid[:-1] self.print_debug('Message-ID:%s' % messageid ) #check messageid is unique self.print_debug("Creating new mailarc '%s'" % 'mailarc') cursor.execute("SELECT id from mailarc WHERE messageid=%s",(messageid,)) row = cursor.fetchone() id = None if row: id = row[0] if id == None or id == "": # why? get_last_id return 0 at first. #id = db.get_last_id(cursor, 'mailarc') is_newid = True cursor.execute("SELECT Max(id)+1 as id from mailarc") row = cursor.fetchone() if row and row[0] != None: id = row[0] else: id = 1 id = int(id) # Because id might be 'n.0', int() is called. if 'date' in msg: datetuple_tz = email.Utils.parsedate_tz(msg['date']) localdate = calendar.timegm(datetuple_tz[:9]) #toDB zoneoffset = datetuple_tz[9] # toDB utcdate = localdate-zoneoffset # toDB #make zone ( +HHMM or -HHMM zone = '' if zoneoffset >0: zone = '+' + time.strftime('%H%M',time.gmtime(zoneoffset)) elif zoneoffset < 0: zone = '-' + time.strftime('%H%M',time.gmtime(-1*zoneoffset)) #self.print_debug( time.strftime("%y/%m/%d %H:%M:%S %z",datetuple_tz[:9])) self.print_debug( time.strftime("%Y/%m/%d %H:%M:%S",time.gmtime(utcdate))) self.print_debug( time.strftime("%Y/%m/%d %H:%M:%S",time.gmtime(localdate))) self.print_debug(zone) fromname,fromaddr = email.Utils.parseaddr(msg['from']) fromname = self.decode_to_unicode(fromname) fromaddr = self.decode_to_unicode(fromaddr) self.print_info( ' ' + time.strftime("%Y/%m/%d %H:%M:%S",time.gmtime(localdate))+' ' + zone +' '+ fromaddr) if 'subject' in msg: subject = self.decode_to_unicode(msg['subject']) self.print_debug( subject.encode(OUTPUT_ENCODING)) # make thread infomations ref_messageid = '' if 'in-reply-to' in msg: ref_messageid = ref_messageid + msg['In-Reply-To'] + ' ' self.print_debug('In-Reply-To:%s' % ref_messageid ) if 'references' in msg: ref_messageid = ref_messageid + msg['References'] + ' ' m = re.findall(r'<(.+?)>', ref_messageid) ref_messageid = '' for text in m: ref_messageid = ref_messageid + "'%s'," % text ref_messageid = ref_messageid.strip(',') self.print_debug('RefMessage-ID:%s' % ref_messageid ) # multipart mail if msg.is_multipart(): body = '' # delete all attachement at message-id Attachment.delete_all(self.env, 'mailarchive', id, db) for part in msg.walk(): content_type = part.get_content_type() self.print_debug('Content-Type:'+content_type) file_counter = 1 if content_type == 'multipart/mixed': pass elif content_type == 'text/html' and self.is_file(part) == False: body = part.get_payload(decode=1) elif content_type == 'text/plain' and self.is_file(part) == False: body = part.get_payload(decode=1) charset = part.get_content_charset() self.print_debug('charset:'+str(charset)) # Todo:need try if charset != None: body = self.to_unicode(body,charset) elif part.get_payload(decode=1) == None: pass else: self.print_debug( part.get_content_type()) # get filename # Applications should really sanitize the given filename so that an # email message can't be used to overwrite important files filename = self.get_filename(part) if not filename: ext = mimetypes.guess_extension(part.get_content_type()) if not ext: # Use a generic bag-of-bits extension ext = '.bin' filename = 'part-%03d%s' % (file_counter, ext) file_counter += 1 self.print_debug("filename:" + filename.encode(OUTPUT_ENCODING)) # make attachment tmp = os.tmpfile() tempsize =len(part.get_payload(decode=1)) tmp.write(part.get_payload(decode=1)) tmp.flush() tmp.seek(0,0) attachment = Attachment(self.env,'mailarchive', id) attachment.description = '' # req.args.get('description', '') attachment.author = author #req.args.get('author', '') attachment.ipnr = '127.0.0.1' try: attachment.insert(filename, tmp, tempsize,None,db) except Exception, e: try: ext = filename.split('.')[-1] if ext == filename: ext = '.bin' else: ext = '.' + ext filename = 'part-%03d%s' % (file_counter, ext) file_counter += 1 attachment.insert(filename, tmp, tempsize,None,db) self.print_warning('As name is too long, the attached file is renamed : '+filename) except Exception, e: self.print_error('Exception at attach file of Message-ID:'+messageid) self.print_error( e ) tmp.close() # not multipart mail else: # Todo:if Content-Type = text/html then convert htmlMail to text content_type = msg.get_content_type() self.print_debug('Content-Type:'+content_type) if content_type == 'text/html': body = 'html' else: #body #self.print_debug(msg.get_content_type()) body = msg.get_payload(decode=1) charset = msg.get_content_charset() # need try: if charset != None: self.print_debug("charset:"+charset) body = self.to_unicode(body,charset) #body = body.replace(os.linesep,'\n') self.print_debug('Thread') thread_parent = ref_messageid.replace("'",'').replace(',',' ') thread_root = '' if thread_parent !='': # sarch first parent id self.print_debug("SearchThread;"+thread_parent) cursor = db.cursor() sql = "SELECT threadroot,messageid FROM mailarc where messageid in (%s)" % ref_messageid self.print_debug(sql) cursor.execute(sql) row = cursor.fetchone() if row: #thread_parent = row[1] if row[0] == '': thread_root = thread_parent.split(' ').pop() self.print_debug("AddToThread;"+thread_root) else: thread_root = row[0] self.print_debug("NewThread;"+thread_root) else: self.print_debug("NoThread;"+thread_parent) thread_root = thread_root.strip() self.print_debug('Insert') if messageid != '': # insert or update mailarc_category yearmonth = time.strftime("%Y%m",time.gmtime(utcdate)) category = mlid+yearmonth cursor.execute("SELECT category,mlid,yearmonth,count FROM mailarc_category WHERE category=%s",(category.encode('utf-8'),)) row = cursor.fetchone() count = 0 if row: count = row[3] pass else: cursor.execute("INSERT INTO mailarc_category (category,mlid,yearmonth,count) VALUES(%s,%s,%s,%s)",(category.encode('utf-8'),mlid.encode('utf-8'),yearmonth,0)) if is_newid == True: count = count +1 cursor.execute("UPDATE mailarc_category SET count=%s WHERE category=%s" , (count,category.encode('utf-8'))) # insert or update mailarc #self.print_debug( # "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)" %(str(id), # category.encode('utf-8'), # messageid, # utcdate, # zoneoffset, # subject.encode('utf-8'), fromname.encode('utf-8'), # fromaddr.encode('utf-8'),'','', # thread_root,thread_parent)) cursor.execute("DELETE FROM mailarc where messageid=%s",(messageid,)) cursor.execute("INSERT INTO mailarc (" "id,category,messageid,utcdate,zoneoffset,subject," "fromname,fromaddr,header,text, threadroot,threadparent ) " "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", (str(id), category.encode('utf-8'), messageid, utcdate, zoneoffset, subject.encode('utf-8'), fromname.encode('utf-8'), fromaddr.encode('utf-8'),'',body.encode('utf-8'), thread_root,thread_parent)) db.commit() def do_refresh_category(self,line): db = self.db_open() self.env = self.env_open() cursor = db.cursor() cursor.execute("DELETE FROM mailarc_category") cursor.execute("SELECT category, count(*) as cnt from mailarc GROUP BY category ") for category,cnt in cursor: cursor2 = db.cursor() cursor2.execute("INSERT INTO mailarc_category (category,mlid,yearmonth,count) VALUES(%s,%s,%s,%s)",(category,category[:-6],category[-6:],cnt)) db.commit() ## Help _help_import = [('import ', 'import UnixMail')] def do_import(self,line): arg = self.arg_tokenize(line) if len(arg) < 2 : print "import MLname filepath" db = self.db_open() self.env = self.env_open() self._import_unixmailbox('cmd',db,arg[0],arg[1]) ## Help _help_pop3 = [('pop3 ', 'import from pop3 server')] def do_pop3(self,line): arg = self.arg_tokenize(line) if len(arg) < 1 : print "pop3 MLname" db = self.db_open() self.env = self.env_open() self._import_from_pop3('cmd',db,arg[0]) ## Help _help_help = [('help', 'Show documentation')] def do_help(self, line=None): arg = self.arg_tokenize(line) if arg[0]: try: doc = getattr(self, "_help_" + arg[0]) self.print_doc (doc) except AttributeError: print "No documentation found for '%s'" % arg[0] else: docs = (#self._help_about + self._help_help + self._help_import + self._help_pop3 ) print 'mailarc-admin - The Trac MailArchivePlugin Administration Console ' if not self.interactive: print print "Usage: mailarc-admin [command [subcommand] [option ...]]\n" print "Invoking mailarc-admin without command starts "\ "interactive mode." self.print_doc (docs) def print_info(self,line): print "%s" % line def print_debug(self,line): #print "[Debug] %s" % line pass def print_error(self,line): print "[Error] %s" % line def print_warning(self,line): print "[Warning] %s" % line def _import_unixmailbox(self,author, db, mlid, msgfile_path): self.print_debug('import_mail') if not db: #db = self.env.get_db_cnx() handle_ta = True else: handle_ta = False #paser = Parser() self.print_info("%s Start Importing %s ..." % (time.strftime("%Y/%m/%d %H:%M:%S",time.gmtime()),msgfile_path)) fp = open(msgfile_path,"rb") mbox = mailbox.UnixMailbox(fp, self.msgfactory) counter =1 msg = mbox.next() while msg is not None: messageid = '' try: messageid = msg['message-id'] self.import_message(msg,author,mlid,db) except Exception, e: exception_flag = True self.print_error('Exception At Message-ID:'+messageid) self.print_error( e ) #traceback.print_exc() if counter > 10000: break msg = mbox.next() counter = counter + 1 fp.close() #if handle_ta: db.commit() self.print_info("End Imporing %s. " % msgfile_path) def _import_from_pop3(self,author, db, mlid): pop_server = self.env.config.get('mailarchive', 'pop3_server') pop_user = self.env.config.get('mailarchive', 'pop3_user') pop_password = self.env.config.get('mailarchive', 'pop3_password') pop_delete = self.env.config.get('mailarchive', 'pop3_delete','none') if pop_server =='': self.print_error('trac.ini mailarchive pop3_server is null!') elif pop_user == '': self.print_error('trac.ini mailarchive pop3_user is null!') elif pop_password == '': self.print_error('trac.ini mailarchive pop3_password is null!') self.print_info("%s Start Connction pop3 %s:%s ..." % (time.strftime("%Y/%m/%d %H:%M:%S",time.gmtime()), pop_server,pop_user)) pop = poplib.POP3(pop_server) pop.user(pop_user) pop.pass_(pop_password) num_messages = len(pop.list()[1]) counter = 1 for i in range(num_messages): #lines = ['',] #for j in pop.retr(i+1)[1]: # lines.append(j + os.linesep) #mes_text = ''.join(lines) mes_text = ''.join(['%s\n' % line for line in pop.retr(i+1)[1]]) messageid = '' exception_flag = False try: msg = email.message_from_string(mes_text) messageid = msg['message-id'] self.import_message(msg,author,mlid,db) except Exception, e: exception_flag = True self.print_error('Exception At Message-ID:'+messageid) self.print_error( e ) #if exception_flag == False: # self.print_info(" Import Message Success") # delete mail if pop_delete == 'all': pop.dele(i+1) self.print_info(" Delete MailServer Message ") elif pop_delete == 'imported': if exception_flag == False: pop.dele(i+1) self.print_info(" Delete MailServer Message ") else: pass if counter > 10000: break counter = counter + 1 pop.quit() #if handle_ta: db.commit() self.print_info("End Reciving. " ) def is_file(self,part ): """Return True:filename associated with the payload if present. """ missing = object() filename = part.get_param('filename', missing, 'content-disposition') if filename is missing: filename = part.get_param('name', missing, 'content-disposition') if filename is missing: return False return True def get_filename(self,part , failobj=None): """Return the filename associated with the payload if present. The filename is extracted from the Content-Disposition header's `filename' parameter, and it is unquoted. If that header is missing the `filename' parameter, this method falls back to looking for the `name' parameter. """ missing = object() filename = part.get_param('filename', missing, 'content-disposition') if filename is missing: filename = part.get_param('name', missing, 'content-disposition') if filename is missing: return failobj errors='replace' fallback_charset='us-ascii' if isinstance(filename, tuple): rawval = self.unquote(filename[2]) charset = filename[0] or 'us-ascii' try: return self.to_unicode(rawval, charset) except LookupError: # XXX charset is unknown to Python. return unicode(rawval, fallback_charset, errors) else: return self.decode_to_unicode(self.unquote(value)) def unquote(self,str): """Remove quotes from a string.""" if len(str) > 1: if str.startswith('"') and str.endswith('"'): return str[1:-1].replace('\\\\', '\\').replace('\\"', '"') if str.startswith('<') and str.endswith('>'): return str[1:-1] return str def _delete_message(self,db,id): pass def run(args): """Main entry point.""" admin = MailArchiveAdmin() if len(args) > 0: if args[0] in ('-h', '--help', 'help'): return admin.onecmd("help") elif args[0] in ('-v','--version','about'): return admin.onecmd("about") else: admin.env_set(os.path.abspath(args[0])) if len(args) > 1: s_args = ' '.join(["'%s'" % c for c in args[2:]]) command = args[1] + ' ' +s_args return admin.onecmd(command) else: while True: admin.run() else: return admin.onecmd("help") sys.exit(run(sys.argv[1:])) }}}