バージョン 1 から バージョン 2 における更新: test

更新日時:

2007/02/15 23:53:35 (19 年 前)

更新者:

weekbuild (IP アドレス: 218.110.157.68)

コメント:

--

test

@@ -73 +73 @@
 
     def decode_to_unicode(self, basestr):
-        #RFC2231の添付ファイル名はget_filenameではunicodeTypeでくるみたいだ
-        if type(basestr) == types.UnicodeType :
-            return basestr
-        
         decodefrag = email.Header.decode_header(basestr)
         subj_fragments = ['',]
@@ -84 +80 @@
             subj_fragments.append(frag)
         return ''.join(subj_fragments)
+
+    def to_unicode(self,text,charset):
+        default_charset = self.env.config.get('mailarchive', 'default_charset',None)
+        if default_charset :
+            chaerset = default_charset
+        
+        # to unicode with codecaliases
+        # codecaliases change mail charset to python charset
+        charset = charset.lower( ) 
+        aliases = {}
+        aliases_text = self.env.config.get('mailarchive', 'codecaliases')
+        for alias in aliases_text.split(','):
+            alias_s = alias.split(':')
+            if len(alias_s) >=2:
+                if alias_s[1] == 'cmd':
+                    aliases[alias_s[0].lower()] = ('cmd',alias_s[2])
+                else:
+                    aliases[alias_s[0].lower()] = ('codec',alias_s[1])
+        
+        if aliases.has_key(charset):
+            (type,alias) = aliases[charset]
+            if type == 'codec':
+                text = unicode(text,alias)
+            elif type == 'cmd':
+                np = NaivePopen(alias, text, capturestderr=1)
+                if np.errorlevel or np.err:
+                    err = 'Running (%s) failed: %s, %s.' % (cmdline, np.errorlevel,
+                                                            np.err)
+                    raise Exception, err
+                text = unicode(np.out,'utf-8')
+        else:
+            text = unicode(text,charset)
+        return text
+
+    def import_message(self, msg, author,mlid, db):
+        OUTPUT_ENCODING = 'utf-8'
+        subject = ''
+        messageid = ''
+        utcdate = 0
+        localdate = 0
+        zoneoffset = 0
+        text = ''
+        fromtext = ''
+        body = ''
+        ref_messageid = ''
+
+        cursor = db.cursor()
+        is_newid = False
+        
+        if 'message-id' in msg:
+            messageid = msg['message-id'] 
+            if messageid[:1] == '<':
+                messageid = messageid[1:]
+            if messageid[-1:] == '>':
+                messageid = messageid[:-1]
+            self.print_debug('Message-ID:%s' % messageid )
+
+            #check messageid is unique
+            self.print_debug("Creating new mailarc '%s'" % 'mailarc')
+            cursor.execute("SELECT id from mailarc WHERE messageid=%s",(messageid,))
+            row = cursor.fetchone()
+            id = None
+            if row:
+                id = row[0]
+            if id == None or id == "":
+                # why? get_last_id return 0 at first.
+                #id = db.get_last_id(cursor, 'mailarc')
+                is_newid = True
+                cursor.execute("SELECT Max(id)+1 as id from mailarc")
+                row = cursor.fetchone()
+                if row and row[0] != None:
+                    id = row[0]
+                else:
+                    id = 1
+            id = int(id) # Because id might be 'n.0', int() is called. 
+
+
+        if 'date' in msg:
+            datetuple_tz = email.Utils.parsedate_tz(msg['date']) 
+            localdate = calendar.timegm(datetuple_tz[:9]) #toDB
+            zoneoffset = datetuple_tz[9] # toDB
+            utcdate = localdate-zoneoffset # toDB
+            #make zone ( +HHMM or -HHMM
+            zone = ''
+            if zoneoffset >0:
+                zone = '+' + time.strftime('%H%M',time.gmtime(zoneoffset))
+            elif zoneoffset < 0:
+                zone = '-' + time.strftime('%H%M',time.gmtime(-1*zoneoffset))
+
+            #self.print_debug( time.strftime("%y/%m/%d %H:%M:%S %z",datetuple_tz[:9]))
+            self.print_debug( time.strftime("%Y/%m/%d %H:%M:%S",time.gmtime(utcdate)))
+            self.print_debug( time.strftime("%Y/%m/%d %H:%M:%S",time.gmtime(localdate)))
+            self.print_debug(zone)
+
+        fromname,fromaddr = email.Utils.parseaddr(msg['from'])
+        fromname = self.decode_to_unicode(fromname)
+        fromaddr = self.decode_to_unicode(fromaddr)
+
+        self.print_info( '  ' + time.strftime("%Y/%m/%d %H:%M:%S",time.gmtime(localdate))+' ' + zone +' '+ fromaddr)
+
+        if 'subject' in msg:
+            subject = self.decode_to_unicode(msg['subject'])
+            self.print_debug( subject.encode(OUTPUT_ENCODING))
+
+        # make thread infomations
+        ref_messageid = ''
+        if 'in-reply-to' in msg:
+            ref_messageid = ref_messageid + msg['In-Reply-To'] + ' '
+            self.print_debug('In-Reply-To:%s' % ref_messageid )
+        
+        if 'references' in msg:
+            ref_messageid = ref_messageid + msg['References'] + ' '
+
+        m = re.findall(r'<(.+?)>', ref_messageid)
+        ref_messageid = ''
+        for text in m:
+            ref_messageid = ref_messageid + "'%s'," % text
+        ref_messageid = ref_messageid.strip(',')
+        self.print_debug('RefMessage-ID:%s' % ref_messageid )
+        
+
+        # multipart mail
+        if msg.is_multipart():
+            body = ''
+            # delete all attachement at message-id
+            Attachment.delete_all(self.env, 'mailarchive', id, db)
+
+            for part in msg.walk():
+                content_type = part.get_content_type()
+                self.print_debug('Content-Type:'+content_type)
+                file_counter = 1
+
+                if content_type == 'multipart/mixed':
+                    pass
+                elif content_type == 'text/html' and self.is_file(part) == False:
+                    body = part.get_payload(decode=1)
+                elif content_type == 'text/plain' and self.is_file(part) == False:
+                    body = part.get_payload(decode=1)
+                    charset = part.get_content_charset()
+                    self.print_debug('charset:'+str(charset))
+                    # Todo:need try 
+                    if charset != None:
+                        body = self.to_unicode(body,charset)
+                elif part.get_payload(decode=1) == None:
+                    pass
+                else:
+                    self.print_debug( part.get_content_type())
+                    # get filename
+                    # Applications should really sanitize the given filename so that an
+                    # email message can't be used to overwrite important files
+                    filename = self.get_filename(part)
+                    if not filename:
+                        ext = mimetypes.guess_extension(part.get_content_type())
+                        if not ext:
+                            # Use a generic bag-of-bits extension
+                            ext = '.bin'
+                        filename = 'part-%03d%s' % (file_counter, ext)
+                        file_counter += 1
+
+                    self.print_debug("filename:" + filename.encode(OUTPUT_ENCODING))
+
+                    # make attachment 
+                    tmp = os.tmpfile()
+                    tempsize =len(part.get_payload(decode=1))
+                    tmp.write(part.get_payload(decode=1))
+                    
+                    tmp.flush()
+                    tmp.seek(0,0)
+
+                    attachment = Attachment(self.env,'mailarchive', id)
+
+                    attachment.description = '' # req.args.get('description', '')
+                    attachment.author = author #req.args.get('author', '')
+                    attachment.ipnr = '127.0.0.1'
+
+                    try:
+                        attachment.insert(filename, 
+                                tmp, tempsize,None,db)
+                    except Exception, e:
+                        try:
+                            ext = filename.split('.')[-1]
+                            if ext == filename:
+                                ext = '.bin'
+                            else:
+                                ext = '.' + ext
+                            filename = 'part-%03d%s' % (file_counter, ext)
+                            file_counter += 1
+                            attachment.insert(filename, 
+                                    tmp, tempsize,None,db)
+                            self.print_warning('As name is too long, the attached file is renamed : '+filename)
+
+                        except Exception, e:
+                            self.print_error('Exception at attach file of Message-ID:'+messageid)
+                            self.print_error( e )
+
+                    tmp.close()
+
+        # not multipart mail
+        else:
+            # Todo:if Content-Type = text/html then convert htmlMail to text
+            content_type = msg.get_content_type()
+            self.print_debug('Content-Type:'+content_type)
+            if content_type == 'text/html':
+                body = 'html'
+            else:
+                #body
+                #self.print_debug(msg.get_content_type())
+                body = msg.get_payload(decode=1)
+                charset = msg.get_content_charset()
+
+                # need try:
+                if charset != None:
+                    self.print_debug("charset:"+charset)
+                    body = self.to_unicode(body,charset)
+            
+            
+        #body = body.replace(os.linesep,'\n')
+        self.print_debug('Thread')
+
+        thread_parent = ref_messageid.replace("'",'').replace(',',' ')
+        thread_root = ''
+        if thread_parent !='':
+        # sarch first parent id
+            self.print_debug("SearchThread;"+thread_parent)
+            cursor = db.cursor()
+            sql = "SELECT threadroot,messageid FROM mailarc where messageid in (%s)" % ref_messageid
+            self.print_debug(sql)
+            cursor.execute(sql)
+
+            row = cursor.fetchone()
+            if row:
+                #thread_parent = row[1]
+                if row[0] == '':
+                    thread_root = thread_parent.split(' ').pop()
+                    self.print_debug("AddToThread;"+thread_root)
+                else:
+                    thread_root = row[0]
+                    self.print_debug("NewThread;"+thread_root)
+            else:
+                    self.print_debug("NoThread;"+thread_parent)
+        thread_root = thread_root.strip()
+        
+        self.print_debug('Insert')
+            
+        if messageid != '':
+
+            # insert or update  mailarc_category
+            
+            yearmonth = time.strftime("%Y%m",time.gmtime(utcdate))
+            category = mlid+yearmonth
+            cursor.execute("SELECT category,mlid,yearmonth,count FROM mailarc_category WHERE category=%s",(category.encode('utf-8'),))
+            row = cursor.fetchone()
+            count = 0
+            if row:
+                count = row[3]
+                pass
+            else:
+                cursor.execute("INSERT INTO mailarc_category (category,mlid,yearmonth,count) VALUES(%s,%s,%s,%s)",(category.encode('utf-8'),mlid.encode('utf-8'),yearmonth,0))
+            if is_newid == True:
+                count = count +1
+            cursor.execute("UPDATE mailarc_category SET count=%s WHERE category=%s" ,
+                (count,category.encode('utf-8')))
+
+            # insert or update mailarc
+                
+            #self.print_debug(
+            #    "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)" %(str(id),
+            #    category.encode('utf-8'),
+            #    messageid,
+            #     utcdate,
+            #      zoneoffset,
+            #     subject.encode('utf-8'), fromname.encode('utf-8'),
+            #     fromaddr.encode('utf-8'),'','',
+            #     thread_root,thread_parent))
+            cursor.execute("DELETE FROM mailarc where messageid=%s",(messageid,))
+            cursor.execute("INSERT INTO mailarc ("
+                "id,category,messageid,utcdate,zoneoffset,subject,"
+                "fromname,fromaddr,header,text, threadroot,threadparent ) "
+                "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)",
+                (str(id),
+                category.encode('utf-8'),
+                messageid,
+                 utcdate,
+                  zoneoffset,
+                 subject.encode('utf-8'), fromname.encode('utf-8'),
+                 fromaddr.encode('utf-8'),'',body.encode('utf-8'),
+                 thread_root,thread_parent))
+
+        db.commit()
+
+    def do_refresh_category(self,line):
+        db = self.db_open()
+        self.env = self.env_open()
+        cursor = db.cursor()
+        cursor.execute("DELETE FROM mailarc_category")
+        cursor.execute("SELECT category, count(*) as cnt from mailarc GROUP BY category ")
+        for category,cnt in cursor:
+            cursor2 =  db.cursor()
+            cursor2.execute("INSERT INTO mailarc_category (category,mlid,yearmonth,count) VALUES(%s,%s,%s,%s)",(category,category[:-6],category[-6:],cnt))
+        db.commit()
+
+    ## Help
+    _help_import = [('import <mlname> <filepath>', 'import UnixMail')]
+            
+    def do_import(self,line):
+        arg = self.arg_tokenize(line)
+        if len(arg) < 2 :
+            print "import MLname filepath"
+        db = self.db_open()
+        self.env = self.env_open()
+        self._import_unixmailbox('cmd',db,arg[0],arg[1])
+
+    ## Help
+    _help_pop3 = [('pop3 <mlname>', 'import from pop3 server')]
+
+    def do_pop3(self,line):
+        arg = self.arg_tokenize(line)
+        if len(arg) < 1 :
+            print "pop3 MLname"
+        db = self.db_open()
+        self.env = self.env_open()
+        self._import_from_pop3('cmd',db,arg[0])
+
+    ## Help
+    _help_help = [('help', 'Show documentation')]
+
+    def do_help(self, line=None):
+        arg = self.arg_tokenize(line)
+        if arg[0]:
+            try:
+                doc = getattr(self, "_help_" + arg[0])
+                self.print_doc (doc)
+            except AttributeError:
+                print "No documentation found for '%s'" % arg[0]
+        else:
+            docs = (#self._help_about +
+                    self._help_help +
+                    self._help_import + self._help_pop3 
+                    )
+            print 'mailarc-admin - The Trac MailArchivePlugin Administration Console ' 
+            if not self.interactive:
+                print
+                print "Usage: mailarc-admin </path/to/projenv> [command [subcommand] [option ...]]\n"
+                print "Invoking mailarc-admin without command starts "\
+                      "interactive mode."
+            self.print_doc (docs)
+
+
+
+    def print_info(self,line):
+        print "%s" % line
+
+    def print_debug(self,line):
+        #print "[Debug] %s" % line
+        pass
+
+    def print_error(self,line):
+        print "[Error] %s" % line
+
+    def print_warning(self,line):
+        print "[Warning] %s" % line
+
+    def _import_unixmailbox(self,author, db, mlid, msgfile_path):
+        self.print_debug('import_mail')
+        if not db:
+            #db = self.env.get_db_cnx()
+            handle_ta = True
+        else:
+            handle_ta = False
+
+
+        #paser = Parser()
+
+        self.print_info("%s Start Importing %s ..." % 
+            (time.strftime("%Y/%m/%d %H:%M:%S",time.gmtime()),msgfile_path))
+
+        fp = open(msgfile_path,"rb")
+        mbox = mailbox.UnixMailbox(fp, self.msgfactory)
+        
+        counter =1
+        msg = mbox.next()
+        while msg is not None:
+            messageid = ''
+            try:
+                messageid = msg['message-id'] 
+                self.import_message(msg,author,mlid,db)
+            except Exception, e:
+                exception_flag = True
+                self.print_error('Exception At Message-ID:'+messageid)
+                self.print_error( e )
+                #traceback.print_exc()
+
+            if counter > 10000:
+                break
+            msg = mbox.next()
+            counter = counter + 1
+
+
+        fp.close()
+        #if handle_ta:
+        db.commit()
+        self.print_info("End Imporing %s. " % msgfile_path)
+
+    def _import_from_pop3(self,author, db, mlid):
+
+        pop_server = self.env.config.get('mailarchive', 'pop3_server')
+        pop_user = self.env.config.get('mailarchive', 'pop3_user')
+        pop_password = self.env.config.get('mailarchive', 'pop3_password')
+        pop_delete = self.env.config.get('mailarchive', 'pop3_delete','none')
+
+        if pop_server =='':
+            self.print_error('trac.ini mailarchive pop3_server is null!')
+        elif pop_user == '':
+            self.print_error('trac.ini mailarchive pop3_user is null!')
+        elif pop_password == '':
+            self.print_error('trac.ini mailarchive pop3_password is null!')
+
+        self.print_info("%s Start Connction pop3 %s:%s ..." % 
+            (time.strftime("%Y/%m/%d %H:%M:%S",time.gmtime()),
+            pop_server,pop_user))
+
+        pop = poplib.POP3(pop_server)
+        pop.user(pop_user)
+        pop.pass_(pop_password)
+        num_messages = len(pop.list()[1])
+        counter = 1
+        for i in range(num_messages):
+            #lines = ['',]
+            #for j in pop.retr(i+1)[1]:
+            #    lines.append(j + os.linesep)
+            #mes_text = ''.join(lines)
+            mes_text = ''.join(['%s\n' % line for line in  pop.retr(i+1)[1]])
+            messageid = ''
+            exception_flag = False
+            try:
+                msg = email.message_from_string(mes_text)
+                messageid = msg['message-id'] 
+                self.import_message(msg,author,mlid,db)
+            except Exception, e:
+                exception_flag = True
+                self.print_error('Exception At Message-ID:'+messageid)
+                self.print_error( e )
+
+            #if exception_flag == False:
+            #    self.print_info("    Import Message Success")
+                
+            
+            # delete mail
+            if pop_delete == 'all':
+                pop.dele(i+1)
+                self.print_info("    Delete MailServer Message ")
+            elif pop_delete == 'imported':
+                if exception_flag == False:
+                    pop.dele(i+1)
+                    self.print_info("    Delete MailServer Message ")
+            else:
+                pass
+
+            if counter > 10000:
+                break
+            counter = counter + 1
+
+        pop.quit()
+
+        #if handle_ta:
+        db.commit()
+        self.print_info("End Reciving. "  )
+
+    def is_file(self,part ):
+        """Return True:filename associated with the payload if present.
+        """
+        missing = object()
+        filename = part.get_param('filename', missing, 'content-disposition')
+        if filename is missing:
+            filename = part.get_param('name', missing, 'content-disposition')
+        if filename is missing:
+            return False
+        return True
+
+    def get_filename(self,part , failobj=None):
+        """Return the filename associated with the payload if present.
+
+        The filename is extracted from the Content-Disposition header's
+        `filename' parameter, and it is unquoted.  If that header is missing
+        the `filename' parameter, this method falls back to looking for the
+        `name' parameter.
+        """
+        missing = object()
+        filename = part.get_param('filename', missing, 'content-disposition')
+        if filename is missing:
+            filename = part.get_param('name', missing, 'content-disposition')
+        if filename is missing:
+            return failobj
+        
+        errors='replace'
+        fallback_charset='us-ascii'
+        if isinstance(filename, tuple):
+            rawval = self.unquote(filename[2])
+            charset = filename[0] or 'us-ascii'
+            try:
+                return self.to_unicode(rawval, charset)
+            except LookupError:
+                # XXX charset is unknown to Python.
+                return unicode(rawval, fallback_charset, errors)
+        else:
+            return self.decode_to_unicode(self.unquote(value))
+
+    def unquote(self,str):
+        """Remove quotes from a string."""
+        if len(str) > 1:
+            if str.startswith('"') and str.endswith('"'):
+                return str[1:-1].replace('\\\\', '\\').replace('\\"', '"')
+            if str.startswith('<') and str.endswith('>'):
+                return str[1:-1]
+        return str
+
+    def _delete_message(self,db,id):
+        pass
+
+def run(args):
+    """Main entry point."""
+    admin = MailArchiveAdmin()
+    if len(args) > 0:
+        if args[0] in ('-h', '--help', 'help'):
+            return admin.onecmd("help")
+        elif args[0] in ('-v','--version','about'):
+            return admin.onecmd("about")
+        else:
+            admin.env_set(os.path.abspath(args[0]))
+            if len(args) > 1:
+                s_args = ' '.join(["'%s'" % c for c in args[2:]])
+                command = args[1] + ' ' +s_args
+                return admin.onecmd(command)
+            else:
+                while True:
+                    admin.run()
+    else:
+        return admin.onecmd("help")
+
+sys.exit(run(sys.argv[1:]))
 }}}