33 | | import os |
34 | | import email.Errors |
35 | | import email.Utils |
36 | | import mailbox |
37 | | |
38 | | from trac import util |
39 | | from trac.wiki import wiki_to_html, IWikiSyntaxProvider |
40 | | from trac.util import Markup |
41 | | from trac.web.chrome import add_link, add_stylesheet, INavigationContributor, ITemplateProvider |
42 | | from trac.attachment import attachments_to_hdf, Attachment |
43 | | from trac.util import NaivePopen |
44 | | import tempfile |
45 | | |
46 | | import getpass, poplib |
47 | | |
48 | | |
49 | | try: |
50 | | sum |
51 | | except NameError: |
52 | | def sum(list): |
53 | | """Python2.2 doesn't have sum()""" |
54 | | tot = 0 |
55 | | for item in list: |
56 | | tot += item |
57 | | return tot |
58 | | |
59 | | |
60 | | class MailArchiveAdmin(trac.scripts.admin.TracAdmin): |
61 | | |
62 | | __env = None |
63 | | env = None |
64 | | |
65 | | |
66 | | def msgfactory(self,fp): |
67 | | try: |
68 | | return email.message_from_file(fp) |
69 | | except email.Errors.MessageParseError: |
70 | | # Don't return None since that will |
71 | | # stop the mailbox iterator |
72 | | return '' |
73 | | |
74 | | def decode_to_unicode(self, basestr): |
75 | | decodefrag = email.Header.decode_header(basestr) |
76 | | subj_fragments = ['',] |
77 | | for frag, enc in decodefrag: |
78 | | if enc: |
79 | | frag = self.to_unicode(frag, enc) |
80 | | subj_fragments.append(frag) |
81 | | return ''.join(subj_fragments) |
82 | | |
83 | | def to_unicode(self,text,charset): |
84 | | default_charset = self.env.config.get('mailarchive', 'default_charset',None) |
85 | | if default_charset : |
86 | | chaerset = default_charset |
87 | | |
88 | | # to unicode with codecaliases |
89 | | # codecaliases change mail charset to python charset |
90 | | charset = charset.lower( ) |
91 | | aliases = {} |
92 | | aliases_text = self.env.config.get('mailarchive', 'codecaliases') |
93 | | for alias in aliases_text.split(','): |
94 | | alias_s = alias.split(':') |
95 | | if len(alias_s) >=2: |
96 | | if alias_s[1] == 'cmd': |
97 | | aliases[alias_s[0].lower()] = ('cmd',alias_s[2]) |
98 | | else: |
99 | | aliases[alias_s[0].lower()] = ('codec',alias_s[1]) |
100 | | |
101 | | if aliases.has_key(charset): |
102 | | (type,alias) = aliases[charset] |
103 | | if type == 'codec': |
104 | | text = unicode(text,alias) |
105 | | elif type == 'cmd': |
106 | | np = NaivePopen(alias, text, capturestderr=1) |
107 | | if np.errorlevel or np.err: |
108 | | err = 'Running (%s) failed: %s, %s.' % (cmdline, np.errorlevel, |
109 | | np.err) |
110 | | raise Exception, err |
111 | | text = unicode(np.out,'utf-8') |
112 | | else: |
113 | | text = unicode(text,charset) |
114 | | return text |
115 | | |
116 | | def import_message(self, msg, author,mlid, db): |
117 | | OUTPUT_ENCODING = 'utf-8' |
118 | | subject = '' |
119 | | messageid = '' |
120 | | utcdate = 0 |
121 | | localdate = 0 |
122 | | zoneoffset = 0 |
123 | | text = '' |
124 | | fromtext = '' |
125 | | body = '' |
126 | | ref_messageid = '' |
127 | | |
128 | | cursor = db.cursor() |
129 | | is_newid = False |
130 | | |
131 | | if 'message-id' in msg: |
132 | | messageid = msg['message-id'] |
133 | | if messageid[:1] == '<': |
134 | | messageid = messageid[1:] |
135 | | if messageid[-1:] == '>': |
136 | | messageid = messageid[:-1] |
137 | | self.print_debug('Message-ID:%s' % messageid ) |
138 | | |
139 | | #check messageid is unique |
140 | | self.print_debug("Creating new mailarc '%s'" % 'mailarc') |
141 | | cursor.execute("SELECT id from mailarc WHERE messageid=%s",(messageid,)) |
142 | | row = cursor.fetchone() |
143 | | id = None |
144 | | if row: |
145 | | id = row[0] |
146 | | if id == None or id == "": |
147 | | # why? get_last_id return 0 at first. |
148 | | #id = db.get_last_id(cursor, 'mailarc') |
149 | | is_newid = True |
150 | | cursor.execute("SELECT Max(id)+1 as id from mailarc") |
151 | | row = cursor.fetchone() |
152 | | if row and row[0] != None: |
153 | | id = row[0] |
154 | | else: |
155 | | id = 1 |
156 | | id = int(id) # Because id might be 'n.0', int() is called. |
157 | | |
158 | | |
159 | | if 'date' in msg: |
160 | | datetuple_tz = email.Utils.parsedate_tz(msg['date']) |
161 | | localdate = calendar.timegm(datetuple_tz[:9]) #toDB |
162 | | zoneoffset = datetuple_tz[9] # toDB |
163 | | utcdate = localdate-zoneoffset # toDB |
164 | | #make zone ( +HHMM or -HHMM |
165 | | zone = '' |
166 | | if zoneoffset >0: |
167 | | zone = '+' + time.strftime('%H%M',time.gmtime(zoneoffset)) |
168 | | elif zoneoffset < 0: |
169 | | zone = '-' + time.strftime('%H%M',time.gmtime(-1*zoneoffset)) |
170 | | |
171 | | #self.print_debug( time.strftime("%y/%m/%d %H:%M:%S %z",datetuple_tz[:9])) |
172 | | self.print_debug( time.strftime("%Y/%m/%d %H:%M:%S",time.gmtime(utcdate))) |
173 | | self.print_debug( time.strftime("%Y/%m/%d %H:%M:%S",time.gmtime(localdate))) |
174 | | self.print_debug(zone) |
175 | | |
176 | | fromname,fromaddr = email.Utils.parseaddr(msg['from']) |
177 | | fromname = self.decode_to_unicode(fromname) |
178 | | fromaddr = self.decode_to_unicode(fromaddr) |
179 | | |
180 | | self.print_info( ' ' + time.strftime("%Y/%m/%d %H:%M:%S",time.gmtime(localdate))+' ' + zone +' '+ fromaddr) |
181 | | |
182 | | if 'subject' in msg: |
183 | | subject = self.decode_to_unicode(msg['subject']) |
184 | | self.print_debug( subject.encode(OUTPUT_ENCODING)) |
185 | | |
186 | | # make thread infomations |
187 | | ref_messageid = '' |
188 | | if 'in-reply-to' in msg: |
189 | | ref_messageid = ref_messageid + msg['In-Reply-To'] + ' ' |
190 | | self.print_debug('In-Reply-To:%s' % ref_messageid ) |
191 | | |
192 | | if 'references' in msg: |
193 | | ref_messageid = ref_messageid + msg['References'] + ' ' |
194 | | |
195 | | m = re.findall(r'<(.+?)>', ref_messageid) |
196 | | ref_messageid = '' |
197 | | for text in m: |
198 | | ref_messageid = ref_messageid + "'%s'," % text |
199 | | ref_messageid = ref_messageid.strip(',') |
200 | | self.print_debug('RefMessage-ID:%s' % ref_messageid ) |
201 | | |
202 | | |
203 | | # multipart mail |
204 | | if msg.is_multipart(): |
205 | | body = '' |
206 | | # delete all attachement at message-id |
207 | | Attachment.delete_all(self.env, 'mailarchive', id, db) |
208 | | |
209 | | for part in msg.walk(): |
210 | | content_type = part.get_content_type() |
211 | | self.print_debug('Content-Type:'+content_type) |
212 | | file_counter = 1 |
213 | | |
214 | | if content_type == 'multipart/mixed': |
215 | | pass |
216 | | elif content_type == 'text/html' and self.is_file(part) == False: |
217 | | body = part.get_payload(decode=1) |
218 | | elif content_type == 'text/plain' and self.is_file(part) == False: |
219 | | body = part.get_payload(decode=1) |
220 | | charset = part.get_content_charset() |
221 | | self.print_debug('charset:'+str(charset)) |
222 | | # Todo:need try |
223 | | if charset != None: |
224 | | body = self.to_unicode(body,charset) |
225 | | elif part.get_payload(decode=1) == None: |
226 | | pass |
227 | | else: |
228 | | self.print_debug( part.get_content_type()) |
229 | | # get filename |
230 | | # Applications should really sanitize the given filename so that an |
231 | | # email message can't be used to overwrite important files |
232 | | filename = self.get_filename(part) |
233 | | if not filename: |
234 | | ext = mimetypes.guess_extension(part.get_content_type()) |
235 | | if not ext: |
236 | | # Use a generic bag-of-bits extension |
237 | | ext = '.bin' |
238 | | filename = 'part-%03d%s' % (file_counter, ext) |
239 | | file_counter += 1 |
240 | | |
241 | | self.print_debug("filename:" + filename.encode(OUTPUT_ENCODING)) |
242 | | |
243 | | # make attachment |
244 | | tmp = os.tmpfile() |
245 | | tempsize =len(part.get_payload(decode=1)) |
246 | | tmp.write(part.get_payload(decode=1)) |
247 | | |
248 | | tmp.flush() |
249 | | tmp.seek(0,0) |
250 | | |
251 | | attachment = Attachment(self.env,'mailarchive', id) |
252 | | |
253 | | attachment.description = '' # req.args.get('description', '') |
254 | | attachment.author = author #req.args.get('author', '') |
255 | | attachment.ipnr = '127.0.0.1' |
256 | | |
257 | | try: |
258 | | attachment.insert(filename, |
259 | | tmp, tempsize,None,db) |
260 | | except Exception, e: |
261 | | try: |
262 | | ext = filename.split('.')[-1] |
263 | | if ext == filename: |
264 | | ext = '.bin' |
265 | | else: |
266 | | ext = '.' + ext |
267 | | filename = 'part-%03d%s' % (file_counter, ext) |
268 | | file_counter += 1 |
269 | | attachment.insert(filename, |
270 | | tmp, tempsize,None,db) |
271 | | self.print_warning('As name is too long, the attached file is renamed : '+filename) |
272 | | |
273 | | except Exception, e: |
274 | | self.print_error('Exception at attach file of Message-ID:'+messageid) |
275 | | self.print_error( e ) |
276 | | |
277 | | tmp.close() |
278 | | |
279 | | # not multipart mail |
280 | | else: |
281 | | # Todo:if Content-Type = text/html then convert htmlMail to text |
282 | | content_type = msg.get_content_type() |
283 | | self.print_debug('Content-Type:'+content_type) |
284 | | if content_type == 'text/html': |
285 | | body = 'html' |
286 | | else: |
287 | | #body |
288 | | #self.print_debug(msg.get_content_type()) |
289 | | body = msg.get_payload(decode=1) |
290 | | charset = msg.get_content_charset() |
291 | | |
292 | | # need try: |
293 | | if charset != None: |
294 | | self.print_debug("charset:"+charset) |
295 | | body = self.to_unicode(body,charset) |
296 | | |
297 | | |
298 | | #body = body.replace(os.linesep,'\n') |
299 | | self.print_debug('Thread') |
300 | | |
301 | | thread_parent = ref_messageid.replace("'",'').replace(',',' ') |
302 | | thread_root = '' |
303 | | if thread_parent !='': |
304 | | # sarch first parent id |
305 | | self.print_debug("SearchThread;"+thread_parent) |
306 | | cursor = db.cursor() |
307 | | sql = "SELECT threadroot,messageid FROM mailarc where messageid in (%s)" % ref_messageid |
308 | | self.print_debug(sql) |
309 | | cursor.execute(sql) |
310 | | |
311 | | row = cursor.fetchone() |
312 | | if row: |
313 | | #thread_parent = row[1] |
314 | | if row[0] == '': |
315 | | thread_root = thread_parent.split(' ').pop() |
316 | | self.print_debug("AddToThread;"+thread_root) |
317 | | else: |
318 | | thread_root = row[0] |
319 | | self.print_debug("NewThread;"+thread_root) |
320 | | else: |
321 | | self.print_debug("NoThread;"+thread_parent) |
322 | | thread_root = thread_root.strip() |
323 | | |
324 | | self.print_debug('Insert') |
325 | | |
326 | | if messageid != '': |
327 | | |
328 | | # insert or update mailarc_category |
329 | | |
330 | | yearmonth = time.strftime("%Y%m",time.gmtime(utcdate)) |
331 | | category = mlid+yearmonth |
332 | | cursor.execute("SELECT category,mlid,yearmonth,count FROM mailarc_category WHERE category=%s",(category.encode('utf-8'),)) |
333 | | row = cursor.fetchone() |
334 | | count = 0 |
335 | | if row: |
336 | | count = row[3] |
337 | | pass |
338 | | else: |
339 | | cursor.execute("INSERT INTO mailarc_category (category,mlid,yearmonth,count) VALUES(%s,%s,%s,%s)",(category.encode('utf-8'),mlid.encode('utf-8'),yearmonth,0)) |
340 | | if is_newid == True: |
341 | | count = count +1 |
342 | | cursor.execute("UPDATE mailarc_category SET count=%s WHERE category=%s" , |
343 | | (count,category.encode('utf-8'))) |
344 | | |
345 | | # insert or update mailarc |
346 | | |
347 | | #self.print_debug( |
348 | | # "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)" %(str(id), |
349 | | # category.encode('utf-8'), |
350 | | # messageid, |
351 | | # utcdate, |
352 | | # zoneoffset, |
353 | | # subject.encode('utf-8'), fromname.encode('utf-8'), |
354 | | # fromaddr.encode('utf-8'),'','', |
355 | | # thread_root,thread_parent)) |
356 | | cursor.execute("DELETE FROM mailarc where messageid=%s",(messageid,)) |
357 | | cursor.execute("INSERT INTO mailarc (" |
358 | | "id,category,messageid,utcdate,zoneoffset,subject," |
359 | | "fromname,fromaddr,header,text, threadroot,threadparent ) " |
360 | | "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", |
361 | | (str(id), |
362 | | category.encode('utf-8'), |
363 | | messageid, |
364 | | utcdate, |
365 | | zoneoffset, |
366 | | subject.encode('utf-8'), fromname.encode('utf-8'), |
367 | | fromaddr.encode('utf-8'),'',body.encode('utf-8'), |
368 | | thread_root,thread_parent)) |
369 | | |
370 | | db.commit() |
371 | | |
372 | | def do_refresh_category(self,line): |
373 | | db = self.db_open() |
374 | | self.env = self.env_open() |
375 | | cursor = db.cursor() |
376 | | cursor.execute("DELETE FROM mailarc_category") |
377 | | cursor.execute("SELECT category, count(*) as cnt from mailarc GROUP BY category ") |
378 | | for category,cnt in cursor: |
379 | | cursor2 = db.cursor() |
380 | | cursor2.execute("INSERT INTO mailarc_category (category,mlid,yearmonth,count) VALUES(%s,%s,%s,%s)",(category,category[:-6],category[-6:],cnt)) |
381 | | db.commit() |
382 | | |
383 | | ## Help |
384 | | _help_import = [('import <mlname> <filepath>', 'import UnixMail')] |
385 | | |
386 | | def do_import(self,line): |
387 | | arg = self.arg_tokenize(line) |
388 | | if len(arg) < 2 : |
389 | | print "import MLname filepath" |
390 | | db = self.db_open() |
391 | | self.env = self.env_open() |
392 | | self._import_unixmailbox('cmd',db,arg[0],arg[1]) |
393 | | |
394 | | ## Help |
395 | | _help_pop3 = [('pop3 <mlname>', 'import from pop3 server')] |
396 | | |
397 | | def do_pop3(self,line): |
398 | | arg = self.arg_tokenize(line) |
399 | | if len(arg) < 1 : |
400 | | print "pop3 MLname" |
401 | | db = self.db_open() |
402 | | self.env = self.env_open() |
403 | | self._import_from_pop3('cmd',db,arg[0]) |
404 | | |
405 | | ## Help |
406 | | _help_help = [('help', 'Show documentation')] |
407 | | |
408 | | def do_help(self, line=None): |
409 | | arg = self.arg_tokenize(line) |
410 | | if arg[0]: |
411 | | try: |
412 | | doc = getattr(self, "_help_" + arg[0]) |
413 | | self.print_doc (doc) |
414 | | except AttributeError: |
415 | | print "No documentation found for '%s'" % arg[0] |
416 | | else: |
417 | | docs = (#self._help_about + |
418 | | self._help_help + |
419 | | self._help_import + self._help_pop3 |
420 | | ) |
421 | | print 'mailarc-admin - The Trac MailArchivePlugin Administration Console ' |
422 | | if not self.interactive: |
423 | | print |
424 | | print "Usage: mailarc-admin </path/to/projenv> [command [subcommand] [option ...]]\n" |
425 | | print "Invoking mailarc-admin without command starts "\ |
426 | | "interactive mode." |
427 | | self.print_doc (docs) |
| 22 | ちなみに[wiki:TracDoc/MailArchivePlugin MailArchive]プラグインでも、添付ファイルのサイズはこの指定に従います。 |
458 | | fp = open(msgfile_path,"rb") |
459 | | mbox = mailbox.UnixMailbox(fp, self.msgfactory) |
460 | | |
461 | | counter =1 |
462 | | msg = mbox.next() |
463 | | while msg is not None: |
464 | | messageid = '' |
465 | | try: |
466 | | messageid = msg['message-id'] |
467 | | self.import_message(msg,author,mlid,db) |
468 | | except Exception, e: |
469 | | exception_flag = True |
470 | | self.print_error('Exception At Message-ID:'+messageid) |
471 | | self.print_error( e ) |
472 | | #traceback.print_exc() |
473 | | |
474 | | if counter > 10000: |
475 | | break |
476 | | msg = mbox.next() |
477 | | counter = counter + 1 |
478 | | |
479 | | |
480 | | fp.close() |
481 | | #if handle_ta: |
482 | | db.commit() |
483 | | self.print_info("End Imporing %s. " % msgfile_path) |
484 | | |
485 | | def _import_from_pop3(self,author, db, mlid): |
486 | | |
487 | | pop_server = self.env.config.get('mailarchive', 'pop3_server') |
488 | | pop_user = self.env.config.get('mailarchive', 'pop3_user') |
489 | | pop_password = self.env.config.get('mailarchive', 'pop3_password') |
490 | | pop_delete = self.env.config.get('mailarchive', 'pop3_delete','none') |
491 | | |
492 | | if pop_server =='': |
493 | | self.print_error('trac.ini mailarchive pop3_server is null!') |
494 | | elif pop_user == '': |
495 | | self.print_error('trac.ini mailarchive pop3_user is null!') |
496 | | elif pop_password == '': |
497 | | self.print_error('trac.ini mailarchive pop3_password is null!') |
498 | | |
499 | | self.print_info("%s Start Connction pop3 %s:%s ..." % |
500 | | (time.strftime("%Y/%m/%d %H:%M:%S",time.gmtime()), |
501 | | pop_server,pop_user)) |
502 | | |
503 | | pop = poplib.POP3(pop_server) |
504 | | pop.user(pop_user) |
505 | | pop.pass_(pop_password) |
506 | | num_messages = len(pop.list()[1]) |
507 | | counter = 1 |
508 | | for i in range(num_messages): |
509 | | #lines = ['',] |
510 | | #for j in pop.retr(i+1)[1]: |
511 | | # lines.append(j + os.linesep) |
512 | | #mes_text = ''.join(lines) |
513 | | mes_text = ''.join(['%s\n' % line for line in pop.retr(i+1)[1]]) |
514 | | messageid = '' |
515 | | exception_flag = False |
516 | | try: |
517 | | msg = email.message_from_string(mes_text) |
518 | | messageid = msg['message-id'] |
519 | | self.import_message(msg,author,mlid,db) |
520 | | except Exception, e: |
521 | | exception_flag = True |
522 | | self.print_error('Exception At Message-ID:'+messageid) |
523 | | self.print_error( e ) |
524 | | |
525 | | #if exception_flag == False: |
526 | | # self.print_info(" Import Message Success") |
527 | | |
528 | | |
529 | | # delete mail |
530 | | if pop_delete == 'all': |
531 | | pop.dele(i+1) |
532 | | self.print_info(" Delete MailServer Message ") |
533 | | elif pop_delete == 'imported': |
534 | | if exception_flag == False: |
535 | | pop.dele(i+1) |
536 | | self.print_info(" Delete MailServer Message ") |
537 | | else: |
538 | | pass |
539 | | |
540 | | if counter > 10000: |
541 | | break |
542 | | counter = counter + 1 |
543 | | |
544 | | pop.quit() |
545 | | |
546 | | #if handle_ta: |
547 | | db.commit() |
548 | | self.print_info("End Reciving. " ) |
549 | | |
550 | | def is_file(self,part ): |
551 | | """Return True:filename associated with the payload if present. |
552 | | """ |
553 | | missing = object() |
554 | | filename = part.get_param('filename', missing, 'content-disposition') |
555 | | if filename is missing: |
556 | | filename = part.get_param('name', missing, 'content-disposition') |
557 | | if filename is missing: |
558 | | return False |
559 | | return True |
560 | | |
561 | | def get_filename(self,part , failobj=None): |
562 | | """Return the filename associated with the payload if present. |
563 | | |
564 | | The filename is extracted from the Content-Disposition header's |
565 | | `filename' parameter, and it is unquoted. If that header is missing |
566 | | the `filename' parameter, this method falls back to looking for the |
567 | | `name' parameter. |
568 | | """ |
569 | | missing = object() |
570 | | filename = part.get_param('filename', missing, 'content-disposition') |
571 | | if filename is missing: |
572 | | filename = part.get_param('name', missing, 'content-disposition') |
573 | | if filename is missing: |
574 | | return failobj |
575 | | |
576 | | errors='replace' |
577 | | fallback_charset='us-ascii' |
578 | | if isinstance(filename, tuple): |
579 | | rawval = unquote(filename[2]) |
580 | | charset = filename[0] or 'us-ascii' |
581 | | try: |
582 | | return self.to_unicode(rawval, charset) |
583 | | except LookupError: |
584 | | # XXX charset is unknown to Python. |
585 | | return unicode(rawval, fallback_charset, errors) |
586 | | else: |
587 | | return self.decode_to_unicode(unquote(value)) |
588 | | |
589 | | |
590 | | def _delete_message(self,db,id): |
591 | | pass |
592 | | |
593 | | def run(args): |
594 | | """Main entry point.""" |
595 | | admin = MailArchiveAdmin() |
596 | | if len(args) > 0: |
597 | | if args[0] in ('-h', '--help', 'help'): |
598 | | return admin.onecmd("help") |
599 | | elif args[0] in ('-v','--version','about'): |
600 | | return admin.onecmd("about") |
601 | | else: |
602 | | admin.env_set(os.path.abspath(args[0])) |
603 | | if len(args) > 1: |
604 | | s_args = ' '.join(["'%s'" % c for c in args[2:]]) |
605 | | command = args[1] + ' ' +s_args |
606 | | return admin.onecmd(command) |
607 | | else: |
608 | | while True: |
609 | | admin.run() |
610 | | else: |
611 | | return admin.onecmd("help") |
612 | | |
613 | | sys.exit(run(sys.argv[1:])) |
614 | | }}} |