ごみです。

#!/usr/local/bin/python
# -*- coding: utf-8 -*-

import sys

import trac.scripts.admin 


import cmd
import getpass
import os
import shlex
import shutil
import StringIO
import time
import traceback
import urllib

import time
import calendar
import re

import mimetypes
import email
#from email.Parser  import Parser 
from email.Header import decode_header
#from email.Utils import collapse_rfc2231_value
from email.Utils import decode_rfc2231
import types

import os
import email.Errors
import email.Utils
import mailbox

from trac import util
from trac.wiki import wiki_to_html, IWikiSyntaxProvider
from trac.util import  Markup
from trac.web.chrome import add_link, add_stylesheet, INavigationContributor, ITemplateProvider
from trac.attachment import attachments_to_hdf, Attachment
from trac.util import NaivePopen
import tempfile

import getpass, poplib


try:
    sum
except NameError:
    def sum(list):
        """Python2.2 doesn't have sum()"""
        tot = 0
        for item in list:
            tot += item
        return tot


class MailArchiveAdmin(trac.scripts.admin.TracAdmin):

    __env = None
    env = None

        
    def msgfactory(self,fp):
        try:
            return email.message_from_file(fp)
        except email.Errors.MessageParseError:
            # Don't return None since that will
	    # stop the mailbox iterator
	    return ''

    def decode_to_unicode(self, basestr):
        decodefrag = email.Header.decode_header(basestr)
        subj_fragments = ['',]
        for frag, enc in decodefrag:
            if enc:
                frag = self.to_unicode(frag, enc)
            subj_fragments.append(frag)
        return ''.join(subj_fragments)

    def to_unicode(self,text,charset):
        default_charset = self.env.config.get('mailarchive', 'default_charset',None)
        if default_charset :
            chaerset = default_charset
        
        # to unicode with codecaliases
        # codecaliases change mail charset to python charset
        charset = charset.lower( ) 
        aliases = {}
        aliases_text = self.env.config.get('mailarchive', 'codecaliases')
        for alias in aliases_text.split(','):
            alias_s = alias.split(':')
            if len(alias_s) >=2:
                if alias_s[1] == 'cmd':
                    aliases[alias_s[0].lower()] = ('cmd',alias_s[2])
                else:
                    aliases[alias_s[0].lower()] = ('codec',alias_s[1])
        
        if aliases.has_key(charset):
            (type,alias) = aliases[charset]
            if type == 'codec':
                text = unicode(text,alias)
            elif type == 'cmd':
                np = NaivePopen(alias, text, capturestderr=1)
                if np.errorlevel or np.err:
                    err = 'Running (%s) failed: %s, %s.' % (cmdline, np.errorlevel,
                                                            np.err)
                    raise Exception, err
                text = unicode(np.out,'utf-8')
        else:
            text = unicode(text,charset)
        return text

    def import_message(self, msg, author,mlid, db):
        OUTPUT_ENCODING = 'utf-8'
        subject = ''
        messageid = ''
        utcdate = 0
        localdate = 0
        zoneoffset = 0
        text = ''
        fromtext = ''
        body = ''
        ref_messageid = ''

        cursor = db.cursor()
        is_newid = False
        
        if 'message-id' in msg:
            messageid = msg['message-id'] 
            if messageid[:1] == '<':
                messageid = messageid[1:]
            if messageid[-1:] == '>':
                messageid = messageid[:-1]
            self.print_debug('Message-ID:%s' % messageid )

            #check messageid is unique
            self.print_debug("Creating new mailarc '%s'" % 'mailarc')
            cursor.execute("SELECT id from mailarc WHERE messageid=%s",(messageid,))
            row = cursor.fetchone()
            id = None
            if row:
                id = row[0]
            if id == None or id == "":
                # why? get_last_id return 0 at first.
                #id = db.get_last_id(cursor, 'mailarc')
                is_newid = True
                cursor.execute("SELECT Max(id)+1 as id from mailarc")
                row = cursor.fetchone()
                if row and row[0] != None:
                    id = row[0]
                else:
                    id = 1
            id = int(id) # Because id might be 'n.0', int() is called. 


        if 'date' in msg:
            datetuple_tz = email.Utils.parsedate_tz(msg['date']) 
            localdate = calendar.timegm(datetuple_tz[:9]) #toDB
            zoneoffset = datetuple_tz[9] # toDB
            utcdate = localdate-zoneoffset # toDB
            #make zone ( +HHMM or -HHMM
            zone = ''
            if zoneoffset >0:
                zone = '+' + time.strftime('%H%M',time.gmtime(zoneoffset))
            elif zoneoffset < 0:
                zone = '-' + time.strftime('%H%M',time.gmtime(-1*zoneoffset))

            #self.print_debug( time.strftime("%y/%m/%d %H:%M:%S %z",datetuple_tz[:9]))
            self.print_debug( time.strftime("%Y/%m/%d %H:%M:%S",time.gmtime(utcdate)))
            self.print_debug( time.strftime("%Y/%m/%d %H:%M:%S",time.gmtime(localdate)))
            self.print_debug(zone)

        fromname,fromaddr = email.Utils.parseaddr(msg['from'])
        fromname = self.decode_to_unicode(fromname)
        fromaddr = self.decode_to_unicode(fromaddr)

        self.print_info( '  ' + time.strftime("%Y/%m/%d %H:%M:%S",time.gmtime(localdate))+' ' + zone +' '+ fromaddr)

        if 'subject' in msg:
            subject = self.decode_to_unicode(msg['subject'])
            self.print_debug( subject.encode(OUTPUT_ENCODING))

        # make thread infomations
        ref_messageid = ''
        if 'in-reply-to' in msg:
            ref_messageid = ref_messageid + msg['In-Reply-To'] + ' '
            self.print_debug('In-Reply-To:%s' % ref_messageid )
        
        if 'references' in msg:
            ref_messageid = ref_messageid + msg['References'] + ' '

        m = re.findall(r'<(.+?)>', ref_messageid)
        ref_messageid = ''
        for text in m:
            ref_messageid = ref_messageid + "'%s'," % text
        ref_messageid = ref_messageid.strip(',')
        self.print_debug('RefMessage-ID:%s' % ref_messageid )
        

        # multipart mail
        if msg.is_multipart():
            body = ''
            # delete all attachement at message-id
            Attachment.delete_all(self.env, 'mailarchive', id, db)

            for part in msg.walk():
                content_type = part.get_content_type()
                self.print_debug('Content-Type:'+content_type)
                file_counter = 1

                if content_type == 'multipart/mixed':
                    pass
                elif content_type == 'text/html' and self.is_file(part) == False:
                    body = part.get_payload(decode=1)
                elif content_type == 'text/plain' and self.is_file(part) == False:
                    body = part.get_payload(decode=1)
                    charset = part.get_content_charset()
                    self.print_debug('charset:'+str(charset))
                    # Todo:need try 
                    if charset != None:
                        body = self.to_unicode(body,charset)
                elif part.get_payload(decode=1) == None:
                    pass
                else:
                    self.print_debug( part.get_content_type())
                    # get filename
                    # Applications should really sanitize the given filename so that an
                    # email message can't be used to overwrite important files
                    filename = self.get_filename(part)
                    if not filename:
                        ext = mimetypes.guess_extension(part.get_content_type())
                        if not ext:
                            # Use a generic bag-of-bits extension
                            ext = '.bin'
                        filename = 'part-%03d%s' % (file_counter, ext)
                        file_counter += 1

                    self.print_debug("filename:" + filename.encode(OUTPUT_ENCODING))

                    # make attachment 
                    tmp = os.tmpfile()
                    tempsize =len(part.get_payload(decode=1))
                    tmp.write(part.get_payload(decode=1))
                    
                    tmp.flush()
                    tmp.seek(0,0)

                    attachment = Attachment(self.env,'mailarchive', id)

                    attachment.description = '' # req.args.get('description', '')
                    attachment.author = author #req.args.get('author', '')
                    attachment.ipnr = '127.0.0.1'

                    try:
                        attachment.insert(filename, 
                                tmp, tempsize,None,db)
                    except Exception, e:
                        try:
                            ext = filename.split('.')[-1]
                            if ext == filename:
                                ext = '.bin'
                            else:
                                ext = '.' + ext
                            filename = 'part-%03d%s' % (file_counter, ext)
                            file_counter += 1
                            attachment.insert(filename, 
                                    tmp, tempsize,None,db)
                            self.print_warning('As name is too long, the attached file is renamed : '+filename)

                        except Exception, e:
                            self.print_error('Exception at attach file of Message-ID:'+messageid)
                            self.print_error( e )

                    tmp.close()

        # not multipart mail
        else:
            # Todo:if Content-Type = text/html then convert htmlMail to text
            content_type = msg.get_content_type()
            self.print_debug('Content-Type:'+content_type)
            if content_type == 'text/html':
                body = 'html'
            else:
                #body
                #self.print_debug(msg.get_content_type())
                body = msg.get_payload(decode=1)
                charset = msg.get_content_charset()

                # need try:
                if charset != None:
                    self.print_debug("charset:"+charset)
                    body = self.to_unicode(body,charset)
            
            
        #body = body.replace(os.linesep,'\n')
        self.print_debug('Thread')

        thread_parent = ref_messageid.replace("'",'').replace(',',' ')
        thread_root = ''
        if thread_parent !='':
        # sarch first parent id
            self.print_debug("SearchThread;"+thread_parent)
            cursor = db.cursor()
            sql = "SELECT threadroot,messageid FROM mailarc where messageid in (%s)" % ref_messageid
            self.print_debug(sql)
            cursor.execute(sql)

            row = cursor.fetchone()
            if row:
                #thread_parent = row[1]
                if row[0] == '':
                    thread_root = thread_parent.split(' ').pop()
                    self.print_debug("AddToThread;"+thread_root)
                else:
                    thread_root = row[0]
                    self.print_debug("NewThread;"+thread_root)
            else:
                    self.print_debug("NoThread;"+thread_parent)
        thread_root = thread_root.strip()
        
        self.print_debug('Insert')
            
        if messageid != '':

            # insert or update  mailarc_category
            
            yearmonth = time.strftime("%Y%m",time.gmtime(utcdate))
            category = mlid+yearmonth
            cursor.execute("SELECT category,mlid,yearmonth,count FROM mailarc_category WHERE category=%s",(category.encode('utf-8'),))
            row = cursor.fetchone()
            count = 0
            if row:
                count = row[3]
                pass
            else:
                cursor.execute("INSERT INTO mailarc_category (category,mlid,yearmonth,count) VALUES(%s,%s,%s,%s)",(category.encode('utf-8'),mlid.encode('utf-8'),yearmonth,0))
            if is_newid == True:
                count = count +1
            cursor.execute("UPDATE mailarc_category SET count=%s WHERE category=%s" ,
                (count,category.encode('utf-8')))

            # insert or update mailarc
                
            #self.print_debug(
            #    "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)" %(str(id),
            #    category.encode('utf-8'),
            #    messageid,
            #     utcdate,
            #      zoneoffset,
            #     subject.encode('utf-8'), fromname.encode('utf-8'),
            #     fromaddr.encode('utf-8'),'','',
            #     thread_root,thread_parent))
            cursor.execute("DELETE FROM mailarc where messageid=%s",(messageid,))
            cursor.execute("INSERT INTO mailarc ("
                "id,category,messageid,utcdate,zoneoffset,subject,"
                "fromname,fromaddr,header,text, threadroot,threadparent ) "
                "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)",
                (str(id),
                category.encode('utf-8'),
                messageid,
                 utcdate,
                  zoneoffset,
                 subject.encode('utf-8'), fromname.encode('utf-8'),
                 fromaddr.encode('utf-8'),'',body.encode('utf-8'),
                 thread_root,thread_parent))

        db.commit()

    def do_refresh_category(self,line):
        db = self.db_open()
        self.env = self.env_open()
        cursor = db.cursor()
        cursor.execute("DELETE FROM mailarc_category")
        cursor.execute("SELECT category, count(*) as cnt from mailarc GROUP BY category ")
        for category,cnt in cursor:
            cursor2 =  db.cursor()
            cursor2.execute("INSERT INTO mailarc_category (category,mlid,yearmonth,count) VALUES(%s,%s,%s,%s)",(category,category[:-6],category[-6:],cnt))
        db.commit()

    ## Help
    _help_import = [('import <mlname> <filepath>', 'import UnixMail')]
            
    def do_import(self,line):
        arg = self.arg_tokenize(line)
        if len(arg) < 2 :
            print "import MLname filepath"
        db = self.db_open()
        self.env = self.env_open()
        self._import_unixmailbox('cmd',db,arg[0],arg[1])

    ## Help
    _help_pop3 = [('pop3 <mlname>', 'import from pop3 server')]

    def do_pop3(self,line):
        arg = self.arg_tokenize(line)
        if len(arg) < 1 :
            print "pop3 MLname"
        db = self.db_open()
        self.env = self.env_open()
        self._import_from_pop3('cmd',db,arg[0])

    ## Help
    _help_help = [('help', 'Show documentation')]

    def do_help(self, line=None):
        arg = self.arg_tokenize(line)
        if arg[0]:
            try:
                doc = getattr(self, "_help_" + arg[0])
                self.print_doc (doc)
            except AttributeError:
                print "No documentation found for '%s'" % arg[0]
        else:
            docs = (#self._help_about +
                    self._help_help +
                    self._help_import + self._help_pop3 
                    )
            print 'mailarc-admin - The Trac MailArchivePlugin Administration Console ' 
            if not self.interactive:
                print
                print "Usage: mailarc-admin </path/to/projenv> [command [subcommand] [option ...]]\n"
                print "Invoking mailarc-admin without command starts "\
                      "interactive mode."
            self.print_doc (docs)



    def print_info(self,line):
        print "%s" % line

    def print_debug(self,line):
        #print "[Debug] %s" % line
        pass

    def print_error(self,line):
        print "[Error] %s" % line

    def print_warning(self,line):
        print "[Warning] %s" % line

    def _import_unixmailbox(self,author, db, mlid, msgfile_path):
        self.print_debug('import_mail')
        if not db:
            #db = self.env.get_db_cnx()
            handle_ta = True
        else:
            handle_ta = False


        #paser = Parser()

        self.print_info("%s Start Importing %s ..." % 
            (time.strftime("%Y/%m/%d %H:%M:%S",time.gmtime()),msgfile_path))

        fp = open(msgfile_path,"rb")
        mbox = mailbox.UnixMailbox(fp, self.msgfactory)
        
        counter =1
        msg = mbox.next()
        while msg is not None:
            messageid = ''
            try:
                messageid = msg['message-id'] 
                self.import_message(msg,author,mlid,db)
            except Exception, e:
                exception_flag = True
                self.print_error('Exception At Message-ID:'+messageid)
                self.print_error( e )
                #traceback.print_exc()

            if counter > 10000:
                break
            msg = mbox.next()
            counter = counter + 1


        fp.close()
        #if handle_ta:
        db.commit()
        self.print_info("End Imporing %s. " % msgfile_path)

    def _import_from_pop3(self,author, db, mlid):

        pop_server = self.env.config.get('mailarchive', 'pop3_server')
        pop_user = self.env.config.get('mailarchive', 'pop3_user')
        pop_password = self.env.config.get('mailarchive', 'pop3_password')
        pop_delete = self.env.config.get('mailarchive', 'pop3_delete','none')

        if pop_server =='':
            self.print_error('trac.ini mailarchive pop3_server is null!')
        elif pop_user == '':
            self.print_error('trac.ini mailarchive pop3_user is null!')
        elif pop_password == '':
            self.print_error('trac.ini mailarchive pop3_password is null!')

        self.print_info("%s Start Connction pop3 %s:%s ..." % 
            (time.strftime("%Y/%m/%d %H:%M:%S",time.gmtime()),
            pop_server,pop_user))

        pop = poplib.POP3(pop_server)
        pop.user(pop_user)
        pop.pass_(pop_password)
        num_messages = len(pop.list()[1])
        counter = 1
        for i in range(num_messages):
            #lines = ['',]
            #for j in pop.retr(i+1)[1]:
            #    lines.append(j + os.linesep)
            #mes_text = ''.join(lines)
            mes_text = ''.join(['%s\n' % line for line in  pop.retr(i+1)[1]])
            messageid = ''
            exception_flag = False
            try:
                msg = email.message_from_string(mes_text)
                messageid = msg['message-id'] 
                self.import_message(msg,author,mlid,db)
            except Exception, e:
                exception_flag = True
                self.print_error('Exception At Message-ID:'+messageid)
                self.print_error( e )

            #if exception_flag == False:
            #    self.print_info("    Import Message Success")
                
            
            # delete mail
            if pop_delete == 'all':
                pop.dele(i+1)
                self.print_info("    Delete MailServer Message ")
            elif pop_delete == 'imported':
                if exception_flag == False:
                    pop.dele(i+1)
                    self.print_info("    Delete MailServer Message ")
            else:
                pass

            if counter > 10000:
                break
            counter = counter + 1

        pop.quit()

        #if handle_ta:
        db.commit()
        self.print_info("End Reciving. "  )

    def is_file(self,part ):
        """Return True:filename associated with the payload if present.
        """
        missing = object()
        filename = part.get_param('filename', missing, 'content-disposition')
        if filename is missing:
            filename = part.get_param('name', missing, 'content-disposition')
        if filename is missing:
            return False
        return True

    def get_filename(self,part , failobj=None):
        """Return the filename associated with the payload if present.

        The filename is extracted from the Content-Disposition header's
        `filename' parameter, and it is unquoted.  If that header is missing
        the `filename' parameter, this method falls back to looking for the
        `name' parameter.
        """
        missing = object()
        filename = part.get_param('filename', missing, 'content-disposition')
        if filename is missing:
            filename = part.get_param('name', missing, 'content-disposition')
        if filename is missing:
            return failobj
        
        errors='replace'
        fallback_charset='us-ascii'
        if isinstance(filename, tuple):
            rawval = self.unquote(filename[2])
            charset = filename[0] or 'us-ascii'
            try:
                return self.to_unicode(rawval, charset)
            except LookupError:
                # XXX charset is unknown to Python.
                return unicode(rawval, fallback_charset, errors)
        else:
            return self.decode_to_unicode(self.unquote(value))

    def unquote(self,str):
        """Remove quotes from a string."""
        if len(str) > 1:
            if str.startswith('"') and str.endswith('"'):
                return str[1:-1].replace('\\\\', '\\').replace('\\"', '"')
            if str.startswith('<') and str.endswith('>'):
                return str[1:-1]
        return str

    def _delete_message(self,db,id):
        pass

def run(args):
    """Main entry point."""
    admin = MailArchiveAdmin()
    if len(args) > 0:
        if args[0] in ('-h', '--help', 'help'):
            return admin.onecmd("help")
        elif args[0] in ('-v','--version','about'):
            return admin.onecmd("about")
        else:
            admin.env_set(os.path.abspath(args[0]))
            if len(args) > 1:
                s_args = ' '.join(["'%s'" % c for c in args[2:]])
                command = args[1] + ' ' +s_args
                return admin.onecmd(command)
            else:
                while True:
                    admin.run()
    else:
        return admin.onecmd("help")

sys.exit(run(sys.argv[1:]))