Преглед на файлове

Merge pull request #522 from inkhey/feature/imapclient

Bastien Sevajol преди 6 години
родител
ревизия
181d78dc63
No account linked to committer's email
променени са 5 файла, в които са добавени 179 реда и са изтрити 162 реда
  1. 1 0
      install/requirements.txt
  2. 3 0
      tracim/development.ini.base
  3. 8 0
      tracim/tracim/config/app_cfg.py
  4. 3 1
      tracim/tracim/lib/daemons.py
  5. 164 161
      tracim/tracim/lib/email_fetcher.py

+ 1 - 0
install/requirements.txt Целия файл

@@ -68,3 +68,4 @@ click==6.7
68 68
 markdown==2.6.9
69 69
 email_reply_parser==0.5.9
70 70
 filelock==2.0.13
71
+imapclient==1.1.0

+ 3 - 0
tracim/development.ini.base Целия файл

@@ -222,6 +222,9 @@ email.reply.imap.user = your_imap_user
222 222
 email.reply.imap.password = your_imap_password
223 223
 email.reply.imap.folder = INBOX
224 224
 email.reply.imap.use_ssl = true
225
+email.reply.imap.use_idle = true
226
+# Re-new connection each 10 minutes
227
+email.reply.connection.max_lifetime = 600
225 228
 # Token for communication between mail fetcher and tracim controller
226 229
 email.reply.token = mysecuretoken
227 230
 # Delay in seconds between each check

+ 8 - 0
tracim/tracim/config/app_cfg.py Целия файл

@@ -384,6 +384,14 @@ class CFG(object):
384 384
         self.EMAIL_REPLY_IMAP_USE_SSL = asbool(tg.config.get(
385 385
             'email.reply.imap.use_ssl',
386 386
         ))
387
+        self.EMAIL_REPLY_IMAP_USE_IDLE = asbool(tg.config.get(
388
+            'email.reply.imap.use_idle',
389
+            True,
390
+        ))
391
+        self.EMAIL_REPLY_CONNECTION_MAX_LIFETIME = int(tg.config.get(
392
+            'email.reply.connection.max_lifetime',
393
+            600, # 10 minutes
394
+        ))
387 395
         self.EMAIL_REPLY_USE_HTML_PARSING = asbool(tg.config.get(
388 396
             'email.reply.use_html_parsing',
389 397
             True,

+ 3 - 1
tracim/tracim/lib/daemons.py Целия файл

@@ -173,7 +173,9 @@ class MailFetcherDaemon(Daemon):
173 173
             password=cfg.EMAIL_REPLY_IMAP_PASSWORD,
174 174
             use_ssl=cfg.EMAIL_REPLY_IMAP_USE_SSL,
175 175
             folder=cfg.EMAIL_REPLY_IMAP_FOLDER,
176
-            delay=cfg.EMAIL_REPLY_CHECK_HEARTBEAT,
176
+            heartbeat=cfg.EMAIL_REPLY_CHECK_HEARTBEAT,
177
+            use_idle=cfg.EMAIL_REPLY_IMAP_USE_IDLE,
178
+            connection_max_lifetime=cfg.EMAIL_REPLY_CONNECTION_MAX_LIFETIME,
177 179
             # FIXME - G.M - 2017-11-15 - proper tracim url formatting
178 180
             endpoint=cfg.WEBSITE_BASE_URL + "/events",
179 181
             token=cfg.EMAIL_REPLY_TOKEN,

+ 164 - 161
tracim/tracim/lib/email_fetcher.py Целия файл

@@ -1,9 +1,11 @@
1 1
 # -*- coding: utf-8 -*-
2 2
 
3 3
 import time
4
-import imaplib
5 4
 import json
6 5
 import typing
6
+import socket
7
+import ssl
8
+
7 9
 from email import message_from_bytes
8 10
 from email.header import decode_header
9 11
 from email.header import make_header
@@ -13,6 +15,8 @@ from email.utils import parseaddr
13 15
 import filelock
14 16
 import markdown
15 17
 import requests
18
+import imapclient
19
+
16 20
 from email_reply_parser import EmailReplyParser
17 21
 from tracim.lib.base import logger
18 22
 from tracim.lib.email_processing.parser import ParsedHTMLMail
@@ -22,9 +26,13 @@ TRACIM_SPECIAL_KEY_HEADER = 'X-Tracim-Key'
22 26
 CONTENT_TYPE_TEXT_PLAIN = 'text/plain'
23 27
 CONTENT_TYPE_TEXT_HTML = 'text/html'
24 28
 
25
-IMAP_SEEN_FLAG = '\\Seen'
26
-IMAP_CHECKED_FLAG = '\\Flagged'
29
+IMAP_SEEN_FLAG = imapclient.SEEN
30
+IMAP_CHECKED_FLAG = imapclient.FLAGGED
31
+
27 32
 MAIL_FETCHER_FILELOCK_TIMEOUT = 10
33
+MAIL_FETCHER_CONNECTION_TIMEOUT = 60*3
34
+MAIL_FETCHER_IDLE_RESPONSE_TIMEOUT = 60*9   # this should be not more
35
+# that 29 minutes according to rfc2177.(server wait 30min by default)
28 36
 
29 37
 
30 38
 class MessageContainer(object):
@@ -153,7 +161,9 @@ class MailFetcher(object):
153 161
         password: str,
154 162
         use_ssl: bool,
155 163
         folder: str,
156
-        delay: int,
164
+        use_idle: bool,
165
+        connection_max_lifetime: int,
166
+        heartbeat: int,
157 167
         endpoint: str,
158 168
         token: str,
159 169
         use_html_parsing: bool,
@@ -170,20 +180,25 @@ class MailFetcher(object):
170 180
         :param password: user password of mailbox
171 181
         :param use_ssl: use imap over ssl connection
172 182
         :param folder: mail folder where new mail are fetched
173
-        :param delay: seconds to wait before fetching new mail again
183
+        :param use_idle: use IMAP IDLE(server notification) when available
184
+        :param heartbeat: seconds to wait before fetching new mail again
185
+        :param connection_max_lifetime: maximum duration allowed for a
186
+             connection . connection are automatically renew when their
187
+             lifetime excess this duration.
174 188
         :param endpoint: tracim http endpoint where decoded mail are send.
175 189
         :param token: token to authenticate http connexion
176 190
         :param use_html_parsing: parse html mail
177 191
         :param use_txt_parsing: parse txt mail
178 192
         """
179
-        self._connection = None
180 193
         self.host = host
181 194
         self.port = port
182 195
         self.user = user
183 196
         self.password = password
184 197
         self.use_ssl = use_ssl
185 198
         self.folder = folder
186
-        self.delay = delay
199
+        self.heartbeat = heartbeat
200
+        self.use_idle = use_idle
201
+        self.connection_max_lifetime = connection_max_lifetime
187 202
         self.endpoint = endpoint
188 203
         self.token = token
189 204
         self.use_html_parsing = use_html_parsing
@@ -194,137 +209,174 @@ class MailFetcher(object):
194 209
     def run(self) -> None:
195 210
         logger.info(self, 'Starting MailFetcher')
196 211
         while self._is_active:
197
-            logger.debug(self, 'sleep for {}'.format(self.delay))
198
-            time.sleep(self.delay)
212
+            imapc = None
213
+            sleep_after_connection = True
199 214
             try:
200
-                self._connect()
201
-                with self.lock.acquire(
202
-                        timeout=MAIL_FETCHER_FILELOCK_TIMEOUT
203
-                ):
204
-                    messages = self._fetch()
205
-                cleaned_mails = [DecodedMail(m.message, m.uid)
206
-                                 for m in messages]
207
-                self._notify_tracim(cleaned_mails)
208
-                self._disconnect()
215
+                imapc = imapclient.IMAPClient(
216
+                    self.host,
217
+                    self.port,
218
+                    ssl=self.use_ssl,
219
+                    timeout=MAIL_FETCHER_CONNECTION_TIMEOUT
220
+                )
221
+                imapc.login(self.user, self.password)
222
+
223
+                logger.debug(self, 'Select folder {}'.format(
224
+                    self.folder,
225
+                ))
226
+                imapc.select_folder(self.folder)
227
+
228
+                # force renew connection when deadline is reached
229
+                deadline = time.time() + self.connection_max_lifetime
230
+                while True:
231
+                    if not self._is_active:
232
+                        logger.warning(self, 'Mail Fetcher process aborted')
233
+                        sleep_after_connection = False
234
+                        break
235
+
236
+                    if time.time() > deadline:
237
+                        logger.debug(
238
+                            self,
239
+                            "MailFetcher Connection Lifetime limit excess"
240
+                            ", Try Re-new connection")
241
+                        sleep_after_connection = False
242
+                        break
243
+
244
+                    # check for new mails
245
+                    self._check_mail(imapc)
246
+
247
+                    if self.use_idle and imapc.has_capability('IDLE'):
248
+                        # IDLE_mode wait until event from server
249
+                        logger.debug(self, 'wail for event(IDLE)')
250
+                        imapc.idle()
251
+                        imapc.idle_check(
252
+                            timeout=MAIL_FETCHER_IDLE_RESPONSE_TIMEOUT
253
+                        )
254
+                        imapc.idle_done()
255
+                    else:
256
+                        if self.use_idle and not imapc.has_capability('IDLE'):
257
+                            log = 'IDLE mode activated but server do not' \
258
+                                  'support it, use polling instead.'
259
+                            logger.warning(self, log)
260
+                        # normal polling mode : sleep a define duration
261
+                        logger.debug(self,
262
+                                     'sleep for {}'.format(self.heartbeat))
263
+                        time.sleep(self.heartbeat)
264
+
265
+            # Socket
266
+            except (socket.error,
267
+                    socket.gaierror,
268
+                    socket.herror) as e:
269
+                log = 'Socket fail with IMAP connection {}'
270
+                logger.error(self, log.format(e.__str__()))
271
+
272
+            except socket.timeout as e:
273
+                log = 'Socket timeout on IMAP connection {}'
274
+                logger.error(self, log.format(e.__str__()))
275
+
276
+            # SSL
277
+            except ssl.SSLError as e:
278
+                log = 'SSL error on IMAP connection'
279
+                logger.error(self, log.format(e.__str__()))
280
+
281
+            except ssl.CertificateError as e:
282
+                log = 'SSL Certificate verification failed on IMAP connection'
283
+                logger.error(self, log.format(e.__str__()))
284
+
285
+            # Filelock
209 286
             except filelock.Timeout as e:
210 287
                 log = 'Mail Fetcher Lock Timeout {}'
211 288
                 logger.warning(self, log.format(e.__str__()))
289
+
290
+            # IMAP
291
+            # TODO - G.M - 10-01-2017 - Support imapclient exceptions
292
+            # when Imapclient stable will be 2.0+
293
+
294
+            # Others
212 295
             except Exception as e:
213
-                # TODO - G.M - 2017-11-23 - Identify possible exceptions
214
-                log = 'IMAP error: {}'
215
-                logger.warning(self, log.format(e.__str__()))
296
+                log = 'Mail Fetcher error {}'
297
+                logger.error(self, log.format(e.__str__()))
298
+
299
+            finally:
300
+                # INFO - G.M - 2018-01-09 - Connection closing
301
+                # Properly close connection according to
302
+                # https://github.com/mjs/imapclient/pull/279/commits/043e4bd0c5c775c5a08cb5f1baa93876a46732ee
303
+                # TODO : Use __exit__ method instead when imapclient stable will
304
+                # be 2.0+ .
305
+                if imapc:
306
+                    logger.debug(self, 'Try logout')
307
+                    try:
308
+                        imapc.logout()
309
+                    except Exception:
310
+                        try:
311
+                            imapc.shutdown()
312
+                        except Exception as e:
313
+                            log = "Can't logout, connection broken ? {}"
314
+                            logger.error(self, log.format(e.__str__()))
315
+
316
+            if sleep_after_connection:
317
+                logger.debug(self, 'sleep for {}'.format(self.heartbeat))
318
+                time.sleep(self.heartbeat)
319
+
320
+        log = 'Mail Fetcher stopped'
321
+        logger.debug(self, log)
322
+
323
+    def _check_mail(self, imapc: imapclient.IMAPClient) -> None:
324
+        with self.lock.acquire(
325
+                timeout=MAIL_FETCHER_FILELOCK_TIMEOUT
326
+        ):
327
+            messages = self._fetch(imapc)
328
+            cleaned_mails = [DecodedMail(m.message, m.uid)
329
+                             for m in messages]
330
+            self._notify_tracim(cleaned_mails, imapc)
216 331
 
217 332
     def stop(self) -> None:
218 333
         self._is_active = False
219 334
 
220
-    def _connect(self) -> None:
221
-        # TODO - G.M - 2017-11-15 Verify connection/disconnection
222
-        # Are old connexion properly close this way ?
223
-        if self._connection:
224
-            logger.debug(self, 'Disconnect from IMAP')
225
-            self._disconnect()
226
-        # TODO - G.M - 2017-11-23 Support for predefined SSLContext ?
227
-        # without ssl_context param, tracim use default security configuration
228
-        # which is great in most case.
229
-        if self.use_ssl:
230
-            logger.debug(self, 'Connect IMAP {}:{} using SSL'.format(
231
-                self.host,
232
-                self.port,
233
-            ))
234
-            self._connection = imaplib.IMAP4_SSL(self.host, self.port)
235
-        else:
236
-            logger.debug(self, 'Connect IMAP {}:{}'.format(
237
-                self.host,
238
-                self.port,
239
-            ))
240
-            self._connection = imaplib.IMAP4(self.host, self.port)
241
-
242
-        try:
243
-            logger.debug(self, 'Login IMAP with login {}'.format(
244
-                self.user,
245
-            ))
246
-            self._connection.login(self.user, self.password)
247
-        except Exception as e:
248
-            log = 'Error during execution: {}'
249
-            logger.error(self, log.format(e.__str__()), exc_info=1)
250
-
251
-    def _disconnect(self) -> None:
252
-        if self._connection:
253
-            self._connection.close()
254
-            self._connection.logout()
255
-            self._connection = None
256
-
257
-    def _fetch(self) -> typing.List[MessageContainer]:
335
+    def _fetch(
336
+        self, 
337
+        imapc: imapclient.IMAPClient,
338
+    ) -> typing.List[MessageContainer]:
258 339
         """
259 340
         Get news message from mailbox
260 341
         :return: list of new mails
261 342
         """
262 343
         messages = []
263
-        # select mailbox
264
-        logger.debug(self, 'Fetch messages from folder {}'.format(
265
-            self.folder,
344
+
345
+        logger.debug(self, 'Fetch unseen messages')
346
+        uids = imapc.search(['UNSEEN'])
347
+        logger.debug(self, 'Found {} unseen mails'.format(
348
+            len(uids),
266 349
         ))
267
-        rv, data = self._connection.select(self.folder)
268
-        logger.debug(self, 'Response status {}'.format(
269
-            rv,
350
+        imapc.add_flags(uids, IMAP_SEEN_FLAG)
351
+        logger.debug(self, 'Temporary Flag {} mails as seen'.format(
352
+            len(uids),
270 353
         ))
271
-        if rv == 'OK':
272
-            # get mails
273
-            # TODO - G.M -  2017-11-15 Which files to select as new file ?
274
-            # Unseen file or All file from a directory (old one should be
275
-            #  moved/ deleted from mailbox during this process) ?
276
-            logger.debug(self, 'Fetch unseen messages')
277
-
278
-            rv, data = self._connection.search(None, "(UNSEEN)")
279
-            logger.debug(self, 'Response status {}'.format(
280
-                rv,
354
+        for msgid, data in imapc.fetch(uids, ['BODY.PEEK[]']).items():
355
+            # INFO - G.M - 2017-12-08 - Fetch BODY.PEEK[]
356
+            # Retrieve all mail(body and header) but don't set mail
357
+            # as seen because of PEEK
358
+            # see rfc3501
359
+            logger.debug(self, 'Fetch mail "{}"'.format(
360
+                msgid,
281 361
             ))
282
-            if rv == 'OK':
283
-                # get mail content
284
-                logger.debug(self, 'Found {} unseen mails'.format(
285
-                    len(data[0].split()),
286
-                ))
287
-                for uid in data[0].split():
288
-                    # INFO - G.M - 2017-12-08 - Fetch BODY.PEEK[]
289
-                    # Retrieve all mail(body and header) but don't set mail
290
-                    # as seen because of PEEK
291
-                    # see rfc3501
292
-                    logger.debug(self, 'Fetch mail "{}"'.format(
293
-                        uid,
294
-                    ))
295
-                    rv, data = self._connection.fetch(uid, 'BODY.PEEK[]')
296
-                    logger.debug(self, 'Response status {}'.format(
297
-                        rv,
298
-                    ))
299
-                    if rv == 'OK':
300
-                        msg = message_from_bytes(data[0][1])
301
-                        msg_container = MessageContainer(msg, uid)
302
-                        messages.append(msg_container)
303
-                        self._set_flag(uid, IMAP_SEEN_FLAG)
304
-                    else:
305
-                        log = 'IMAP : Unable to get mail : {}'
306
-                        logger.error(self, log.format(str(rv)))
307
-            else:
308
-                log = 'IMAP : Unable to get unseen mail : {}'
309
-                logger.error(self, log.format(str(rv)))
310
-        else:
311
-            log = 'IMAP : Unable to open mailbox : {}'
312
-            logger.error(self, log.format(str(rv)))
362
+            msg = message_from_bytes(data[b'BODY[]'])
363
+            msg_container = MessageContainer(msg, msgid)
364
+            messages.append(msg_container)
313 365
         return messages
314 366
 
315 367
     def _notify_tracim(
316 368
         self,
317 369
         mails: typing.List[DecodedMail],
370
+        imapc: imapclient.IMAPClient
318 371
     ) -> None:
319 372
         """
320 373
         Send http request to tracim endpoint
321 374
         :param mails: list of mails to send
322
-        :return: unsended mails
375
+        :return: none
323 376
         """
324 377
         logger.debug(self, 'Notify tracim about {} new responses'.format(
325 378
             len(mails),
326 379
         ))
327
-        unsended_mails = []
328 380
         # TODO BS 20171124: Look around mail.get_from_address(), mail.get_key()
329 381
         # , mail.get_body() etc ... for raise InvalidEmailError if missing
330 382
         #  required informations (actually get_from_address raise IndexError
@@ -371,64 +423,15 @@ class MailFetcher(object):
371 423
                     ))
372 424
                 # Flag all correctly checked mail, unseen the others
373 425
                 if r.status_code in [200, 204, 400]:
374
-                    self._set_flag(mail.uid, IMAP_CHECKED_FLAG)
426
+                    imapc.add_flags((mail.uid,), IMAP_CHECKED_FLAG)
375 427
                 else:
376
-                    self._unset_flag(mail.uid, IMAP_SEEN_FLAG)
428
+                    imapc.remove_flags((mail.uid,), IMAP_SEEN_FLAG)
377 429
             # TODO - G.M - Verify exception correctly works
378 430
             except requests.exceptions.Timeout as e:
379 431
                 log = 'Timeout error to transmit fetched mail to tracim : {}'
380 432
                 logger.error(self, log.format(str(e)))
381
-                unsended_mails.append(mail)
382
-                self._unset_flag(mail.uid, IMAP_SEEN_FLAG)
433
+                imapc.remove_flags((mail.uid,), IMAP_SEEN_FLAG)
383 434
             except requests.exceptions.RequestException as e:
384 435
                 log = 'Fail to transmit fetched mail to tracim : {}'
385 436
                 logger.error(self, log.format(str(e)))
386
-                self._unset_flag(mail.uid, IMAP_SEEN_FLAG)
387
-
388
-    def _set_flag(
389
-            self,
390
-            uid: int,
391
-            flag: str,
392
-            ) -> None:
393
-        assert uid is not None
394
-
395
-        rv, data = self._connection.store(
396
-            uid,
397
-            '+FLAGS',
398
-            flag,
399
-        )
400
-        if rv == 'OK':
401
-            log = 'Message {uid} set as {flag}.'.format(
402
-                uid=uid,
403
-                flag=flag)
404
-            logger.debug(self, log)
405
-        else:
406
-            log = 'Can not set Message {uid} as {flag} : {rv}'.format(
407
-                uid=uid,
408
-                flag=flag,
409
-                rv=rv)
410
-            logger.error(self, log)
411
-
412
-    def _unset_flag(
413
-            self,
414
-            uid: int,
415
-            flag: str,
416
-            ) -> None:
417
-        assert uid is not None
418
-
419
-        rv, data = self._connection.store(
420
-            uid,
421
-            '-FLAGS',
422
-            flag,
423
-        )
424
-        if rv == 'OK':
425
-            log = 'Message {uid} unset as {flag}.'.format(
426
-                uid=uid,
427
-                flag=flag)
428
-            logger.debug(self, log)
429
-        else:
430
-            log = 'Can not unset Message {uid} as {flag} : {rv}'.format(
431
-                uid=uid,
432
-                flag=flag,
433
-                rv=rv)
434
-            logger.error(self, log)
437
+                imapc.remove_flags((mail.uid,), IMAP_SEEN_FLAG)