Przeglądaj źródła

Merge pull request #514 from inkhey/fix/512/Better_email_fetching

Bastien Sevajol 6 lat temu
rodzic
commit
fad01888eb
No account linked to committer's email

+ 1 - 0
.gitignore Wyświetl plik

@@ -70,6 +70,7 @@ wsgidav.conf
70 70
 # Temporary files
71 71
 *~
72 72
 *.sqlite
73
+*.lock
73 74
 
74 75
 # npm packages
75 76
 /node_modules/

+ 1 - 1
install/requirements.postgresql.txt Wyświetl plik

@@ -1 +1 @@
1
-psycopg2==2.5.4
1
+psycopg2==2.7.3.2

+ 1 - 0
install/requirements.txt Wyświetl plik

@@ -67,3 +67,4 @@ rq==0.7.1
67 67
 click==6.7
68 68
 markdown==2.6.9
69 69
 email_reply_parser==0.5.9
70
+filelock==2.0.13

+ 3 - 0
tracim/development.ini.base Wyświetl plik

@@ -228,6 +228,9 @@ email.reply.token = mysecuretoken
228 228
 email.reply.check.heartbeat = 60
229 229
 email.reply.use_html_parsing = true
230 230
 email.reply.use_txt_parsing = true
231
+# Lockfile path is required for email_reply feature,
232
+# it's just an empty file use to prevent concurrent access to imap unseen mail
233
+email.reply.lockfile_path = %(here)s/email_fetcher.lock
231 234
 
232 235
 ## Radical (CalDav server) configuration
233 236
 # radicale.server.host = 0.0.0.0

+ 8 - 0
tracim/tracim/config/app_cfg.py Wyświetl plik

@@ -392,6 +392,14 @@ class CFG(object):
392 392
             'email.reply.use_txt_parsing',
393 393
             True,
394 394
         ))
395
+        self.EMAIL_REPLY_LOCKFILE_PATH = tg.config.get(
396
+            'email.reply.lockfile_path',
397
+            ''
398
+        )
399
+        if not self.EMAIL_REPLY_LOCKFILE_PATH and self.EMAIL_REPLY_ACTIVATED:
400
+            raise Exception(
401
+                mandatory_msg.format('email.reply.lockfile_path')
402
+            )
395 403
 
396 404
         self.TRACKER_JS_PATH = tg.config.get(
397 405
             'js_tracker_path',

+ 1 - 0
tracim/tracim/lib/daemons.py Wyświetl plik

@@ -179,6 +179,7 @@ class MailFetcherDaemon(Daemon):
179 179
             token=cfg.EMAIL_REPLY_TOKEN,
180 180
             use_html_parsing=cfg.EMAIL_REPLY_USE_HTML_PARSING,
181 181
             use_txt_parsing=cfg.EMAIL_REPLY_USE_TXT_PARSING,
182
+            lockfile_path=cfg.EMAIL_REPLY_LOCKFILE_PATH,
182 183
         )
183 184
         self._fetcher.run()
184 185
 

+ 91 - 18
tracim/tracim/lib/email_fetcher.py Wyświetl plik

@@ -10,6 +10,7 @@ from email.header import make_header
10 10
 from email.message import Message
11 11
 from email.utils import parseaddr
12 12
 
13
+import filelock
13 14
 import markdown
14 15
 import requests
15 16
 from email_reply_parser import EmailReplyParser
@@ -21,10 +22,21 @@ TRACIM_SPECIAL_KEY_HEADER = 'X-Tracim-Key'
21 22
 CONTENT_TYPE_TEXT_PLAIN = 'text/plain'
22 23
 CONTENT_TYPE_TEXT_HTML = 'text/html'
23 24
 
25
+IMAP_SEEN_FLAG = '\\Seen'
26
+IMAP_CHECKED_FLAG = '\\Flagged'
27
+MAIL_FETCHER_FILELOCK_TIMEOUT = 10
28
+
29
+
30
+class MessageContainer(object):
31
+    def __init__(self, message: Message, uid: int) -> None:
32
+        self.message = message
33
+        self.uid = uid
34
+
24 35
 
25 36
 class DecodedMail(object):
26
-    def __init__(self, message: Message) -> None:
37
+    def __init__(self, message: Message, uid: int=None) -> None:
27 38
         self._message = message
39
+        self.uid = uid
28 40
 
29 41
     def _decode_header(self, header_title: str) -> typing.Optional[str]:
30 42
         # FIXME : Handle exception
@@ -146,6 +158,7 @@ class MailFetcher(object):
146 158
         token: str,
147 159
         use_html_parsing: bool,
148 160
         use_txt_parsing: bool,
161
+        lockfile_path: str,
149 162
     ) -> None:
150 163
         """
151 164
         Fetch mail from a mailbox folder through IMAP and add their content to
@@ -175,7 +188,7 @@ class MailFetcher(object):
175 188
         self.token = token
176 189
         self.use_html_parsing = use_html_parsing
177 190
         self.use_txt_parsing = use_txt_parsing
178
-
191
+        self.lock = filelock.FileLock(lockfile_path)
179 192
         self._is_active = True
180 193
 
181 194
     def run(self) -> None:
@@ -185,13 +198,17 @@ class MailFetcher(object):
185 198
             time.sleep(self.delay)
186 199
             try:
187 200
                 self._connect()
188
-                messages = self._fetch()
189
-                # TODO - G.M -  2017-11-22 retry sending unsended mail
190
-                # These mails are return by _notify_tracim, flag them with "unseen" # nopep8
191
-                # or store them until new _notify_tracim call
192
-                cleaned_mails = [DecodedMail(msg) for msg in messages]
201
+                with self.lock.acquire(
202
+                        timeout=MAIL_FETCHER_FILELOCK_TIMEOUT
203
+                ):
204
+                    messages = self._fetch()
205
+                cleaned_mails = [DecodedMail(m.message, m.uid)
206
+                                 for m in messages]
193 207
                 self._notify_tracim(cleaned_mails)
194 208
                 self._disconnect()
209
+            except filelock.Timeout as e:
210
+                log = 'Mail Fetcher Lock Timeout {}'
211
+                logger.warning(self, log.format(e.__str__()))
195 212
             except Exception as e:
196 213
                 # TODO - G.M - 2017-11-23 - Identify possible exceptions
197 214
                 log = 'IMAP error: {}'
@@ -237,7 +254,7 @@ class MailFetcher(object):
237 254
             self._connection.logout()
238 255
             self._connection = None
239 256
 
240
-    def _fetch(self) -> typing.List[Message]:
257
+    def _fetch(self) -> typing.List[MessageContainer]:
241 258
         """
242 259
         Get news message from mailbox
243 260
         :return: list of new mails
@@ -257,6 +274,7 @@ class MailFetcher(object):
257 274
             # Unseen file or All file from a directory (old one should be
258 275
             #  moved/ deleted from mailbox during this process) ?
259 276
             logger.debug(self, 'Fetch unseen messages')
277
+
260 278
             rv, data = self._connection.search(None, "(UNSEEN)")
261 279
             logger.debug(self, 'Response status {}'.format(
262 280
                 rv,
@@ -266,21 +284,23 @@ class MailFetcher(object):
266 284
                 logger.debug(self, 'Found {} unseen mails'.format(
267 285
                     len(data[0].split()),
268 286
                 ))
269
-                for num in data[0].split():
270
-                    # INFO - G.M - 2017-11-23 - Fetch (RFC288) to retrieve all
271
-                    # complete mails see example : https://docs.python.org/fr/3.5/library/imaplib.html#imap4-example .  # nopep8
272
-                    # Be careful, This method remove also mails from Unseen
273
-                    # mails
287
+                for uid in data[0].split():
288
+                    # INFO - G.M - 2017-12-08 - Fetch BODY.PEEK[]
289
+                    # Retrieve all mail(body and header) but don't set mail
290
+                    # as seen because of PEEK
291
+                    # see rfc3501
274 292
                     logger.debug(self, 'Fetch mail "{}"'.format(
275
-                        num,
293
+                        uid,
276 294
                     ))
277
-                    rv, data = self._connection.fetch(num, '(RFC822)')
295
+                    rv, data = self._connection.fetch(uid, 'BODY.PEEK[]')
278 296
                     logger.debug(self, 'Response status {}'.format(
279 297
                         rv,
280 298
                     ))
281 299
                     if rv == 'OK':
282 300
                         msg = message_from_bytes(data[0][1])
283
-                        messages.append(msg)
301
+                        msg_container = MessageContainer(msg, uid)
302
+                        messages.append(msg_container)
303
+                        self._set_flag(uid, IMAP_SEEN_FLAG)
284 304
                     else:
285 305
                         log = 'IMAP : Unable to get mail : {}'
286 306
                         logger.error(self, log.format(str(rv)))
@@ -295,7 +315,7 @@ class MailFetcher(object):
295 315
     def _notify_tracim(
296 316
         self,
297 317
         mails: typing.List[DecodedMail],
298
-    ) -> typing.List[DecodedMail]:
318
+    ) -> None:
299 319
         """
300 320
         Send http request to tracim endpoint
301 321
         :param mails: list of mails to send
@@ -335,13 +355,66 @@ class MailFetcher(object):
335 355
                         str(r.status_code),
336 356
                         details,
337 357
                     ))
358
+                # Flag all correctly checked mail, unseen the others
359
+                if r.status_code in [200, 204, 400]:
360
+                    self._set_flag(mail.uid, IMAP_CHECKED_FLAG)
361
+                else:
362
+                    self._unset_flag(mail.uid, IMAP_SEEN_FLAG)
338 363
             # TODO - G.M - Verify exception correctly works
339 364
             except requests.exceptions.Timeout as e:
340 365
                 log = 'Timeout error to transmit fetched mail to tracim : {}'
341 366
                 logger.error(self, log.format(str(e)))
342 367
                 unsended_mails.append(mail)
368
+                self._unset_flag(mail.uid, IMAP_SEEN_FLAG)
343 369
             except requests.exceptions.RequestException as e:
344 370
                 log = 'Fail to transmit fetched mail to tracim : {}'
345 371
                 logger.error(self, log.format(str(e)))
372
+                self._unset_flag(mail.uid, IMAP_SEEN_FLAG)
346 373
 
347
-        return unsended_mails
374
+    def _set_flag(
375
+            self,
376
+            uid: int,
377
+            flag: str,
378
+            ) -> None:
379
+        assert uid is not None
380
+
381
+        rv, data = self._connection.store(
382
+            uid,
383
+            '+FLAGS',
384
+            flag,
385
+        )
386
+        if rv == 'OK':
387
+            log = 'Message {uid} set as {flag}.'.format(
388
+                uid=uid,
389
+                flag=flag)
390
+            logger.debug(self, log)
391
+        else:
392
+            log = 'Can not set Message {uid} as {flag} : {rv}'.format(
393
+                uid=uid,
394
+                flag=flag,
395
+                rv=rv)
396
+            logger.error(self, log)
397
+
398
+    def _unset_flag(
399
+            self,
400
+            uid: int,
401
+            flag: str,
402
+            ) -> None:
403
+        assert uid is not None
404
+
405
+        rv, data = self._connection.store(
406
+            uid,
407
+            '-FLAGS',
408
+            flag,
409
+        )
410
+        if rv == 'OK':
411
+            log = 'Message {uid} unset as {flag}.'.format(
412
+                uid=uid,
413
+                flag=flag)
414
+            logger.debug(self, log)
415
+        else:
416
+            log = 'Can not unset Message {uid} as {flag} : {rv}'.format(
417
+                uid=uid,
418
+                flag=flag,
419
+                rv=rv)
420
+            logger.error(self, log)