Merge pull request #498 from inkhey/feature/replywithmail

install/requirements.txt

 typing==3.5.3.0
 rq==0.7.1
 click==6.7
+markdown==2.6.9
+email_reply_parser==0.5.9

tracim/development.ini.base

 # notifications generated by a user or another one
 email.notification.from.email = noreply+{user_id}@trac.im
 email.notification.from.default_label = Tracim Notifications
+email.notification.reply_to.email = reply+{content_id}@trac.im
+email.notification.references.email = thread+{content_id}@trac.im
 email.notification.content_update.template.html = %(here)s/tracim/templates/mail/content_update_body_html.mak
 email.notification.content_update.template.text = %(here)s/tracim/templates/mail/content_update_body_text.mak
 email.notification.created_account.template.html = %(here)s/tracim/templates/mail/created_account_body_html.mak
 # email.async.redis.port = 6379
 # email.async.redis.db = 0
 
+# Email reply configuration
+email.reply.activated = False
+email.reply.imap.server = your_imap_server
+email.reply.imap.port = 993
+email.reply.imap.user = your_imap_user
+email.reply.imap.password = your_imap_password
+email.reply.imap.folder = INBOX
+email.reply.imap.use_ssl = true
+# Token for communication between mail fetcher and tracim controller
+email.reply.token = mysecuretoken
+# Delay in seconds between each check
+email.reply.check.heartbeat = 60
+
 ## Radical (CalDav server) configuration
 # radicale.server.host = 0.0.0.0
 # radicale.server.port = 5232

tracim/tracim/config/app_cfg.py

 from tracim.lib.base import logger
 from tracim.lib.daemons import DaemonsManager
 from tracim.lib.daemons import MailSenderDaemon
+from tracim.lib.daemons import MailFetcherDaemon
 from tracim.lib.daemons import RadicaleDaemon
 from tracim.lib.daemons import WsgiDavDaemon
 from tracim.lib.system import InterruptManager
     if cfg.EMAIL_PROCESSING_MODE == CFG.CST.ASYNC:
         manager.run('mail_sender', MailSenderDaemon)
 
+    if cfg.EMAIL_REPLY_ACTIVATED:
+        manager.run('mail_fetcher',MailFetcherDaemon)
+
 
 def configure_depot():
     """Configure Depot."""
         self.EMAIL_NOTIFICATION_FROM_DEFAULT_LABEL = tg.config.get(
             'email.notification.from.default_label'
         )
+        self.EMAIL_NOTIFICATION_REPLY_TO_EMAIL = tg.config.get(
+            'email.notification.reply_to.email',
+        )
+        self.EMAIL_NOTIFICATION_REFERENCES_EMAIL = tg.config.get(
+            'email.notification.references.email'
+        )
         self.EMAIL_NOTIFICATION_CONTENT_UPDATE_TEMPLATE_HTML = tg.config.get(
             'email.notification.content_update.template.html',
         )
             None,
         )
 
+        self.EMAIL_REPLY_ACTIVATED = asbool(tg.config.get(
+            'email.reply.activated',
+            False,
+        ))
+
+        self.EMAIL_REPLY_IMAP_SERVER = tg.config.get(
+            'email.reply.imap.server',
+        )
+        self.EMAIL_REPLY_IMAP_PORT = tg.config.get(
+            'email.reply.imap.port',
+        )
+        self.EMAIL_REPLY_IMAP_USER = tg.config.get(
+            'email.reply.imap.user',
+        )
+        self.EMAIL_REPLY_IMAP_PASSWORD = tg.config.get(
+            'email.reply.imap.password',
+        )
+        self.EMAIL_REPLY_IMAP_FOLDER = tg.config.get(
+            'email.reply.imap.folder',
+        )
+        self.EMAIL_REPLY_CHECK_HEARTBEAT = int(tg.config.get(
+            'email.reply.check.heartbeat',
+            60,
+        ))
+        self.EMAIL_REPLY_TOKEN = tg.config.get(
+            'email.reply.token',
+        )
+        self.EMAIL_REPLY_IMAP_USE_SSL = asbool(tg.config.get(
+            'email.reply.imap.use_ssl',
+        ))
+
         self.TRACKER_JS_PATH = tg.config.get(
             'js_tracker_path',
         )

tracim/tracim/controllers/events.py

+import tg
+import typing
+from tg import request
+from tg import Response
+from tg import abort
+from tg import RestController
+from sqlalchemy.orm.exc import NoResultFound
+
+from tracim.lib.content import ContentApi
+from tracim.lib.user import UserApi
+from tracim.model.data import ContentType
+from tracim.config.app_cfg import CFG
+
+
+class EventRestController(RestController):
+
+    @tg.expose('json')
+    def post(self) -> Response:
+        cfg = CFG.get_instance()
+
+        try:
+            json = request.json_body
+        except ValueError:
+            return Response(
+                status=400,
+                json_body={'msg': 'Bad json'},
+            )
+
+        if json.get('token', None) != cfg.EMAIL_REPLY_TOKEN:
+            # TODO - G.M - 2017-11-23 - Switch to status 403 ?
+            # 403 is a better status code in this case.
+            # 403 status response can't now return clean json, because they are
+            # handled somewhere else to return html.
+            return Response(
+                status=400,
+                json_body={'msg': 'Invalid token'}
+            )
+
+        if 'user_mail' not in json:
+            return Response(
+                status=400,
+                json_body={'msg': 'Bad json: user_mail is required'}
+            )
+
+        if 'content_id' not in json:
+            return Response(
+                status=400,
+                json_body={'msg': 'Bad json: content_id is required'}
+            )
+
+        if 'payload' not in json:
+            return Response(
+                status=400,
+                json_body={'msg': 'Bad json: payload is required'}
+            )
+
+        uapi = UserApi(None)
+        try:
+            user = uapi.get_one_by_email(json['user_mail'])
+        except NoResultFound:
+            return Response(
+                status=400,
+                json_body={'msg': 'Unknown user email'},
+            )
+        api = ContentApi(user)
+
+        try:
+            thread = api.get_one(json['content_id'],
+                                 content_type=ContentType.Any)
+        except NoResultFound:
+            return Response(
+                status=400,
+                json_body={'msg': 'Unknown content_id'},
+            )
+
+        # INFO - G.M - 2017-11-17
+        # When content_id is a sub-elem of a main content like Comment,
+        # Attach the thread to the main content.
+        if thread.type == ContentType.Comment:
+            thread = thread.parent
+        if thread.type == ContentType.Folder:
+            return Response(
+                status=400,
+                json_body={'msg': 'comment for folder not allowed'},
+            )
+        if 'content' in json['payload']:
+            api.create_comment(
+                workspace=thread.workspace,
+                parent=thread,
+                content=json['payload']['content'],
+                do_save=True,
+            )
+            return Response(
+                status=204,
+            )
+        else:
+            return Response(
+                status=400,
+                json_body={'msg': 'No content to add new comment'},
+            )

tracim/tracim/controllers/root.py

 from tracim.controllers.previews import PreviewsController
 from tracim.controllers.user import UserRestController
 from tracim.controllers.workspace import UserWorkspaceRestController
+from tracim.controllers.events import EventRestController
 from tracim.lib import CST
 from tracim.lib.base import logger
 from tracim.lib.content import ContentApi
     previews = PreviewsController()
 
     content = ContentController()
-
+    events = EventRestController()
     # api
     api = APIController()
 

tracim/tracim/lib/daemons.py

 from tracim.lib.exceptions import AlreadyRunningDaemon
 
 from tracim.lib.utils import get_rq_queue
+from tracim.lib.email_fetcher import MailFetcher
 
 
 class DaemonsManager(object):
         raise NotImplementedError()
 
 
+class MailFetcherDaemon(Daemon):
+    """
+    Thread containing a daemon who fetch new mail from a mailbox and
+    send http request to a tracim endpoint to handle them.
+    """
+
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self._fetcher = None  # type: MailFetcher
+        self.ok = True
+
+    def run(self) -> None:
+        from tracim.config.app_cfg import CFG
+        cfg = CFG.get_instance()
+        self._fetcher = MailFetcher(
+            host=cfg.EMAIL_REPLY_IMAP_SERVER,
+            port=cfg.EMAIL_REPLY_IMAP_PORT,
+            user=cfg.EMAIL_REPLY_IMAP_USER,
+            password=cfg.EMAIL_REPLY_IMAP_PASSWORD,
+            use_ssl=cfg.EMAIL_REPLY_IMAP_USE_SSL,
+            folder=cfg.EMAIL_REPLY_IMAP_FOLDER,
+            delay=cfg.EMAIL_REPLY_CHECK_HEARTBEAT,
+            # FIXME - G.M - 2017-11-15 - proper tracim url formatting
+            endpoint=cfg.WEBSITE_BASE_URL + "/events",
+            token=cfg.EMAIL_REPLY_TOKEN,
+        )
+        self._fetcher.run()
+
+    def stop(self) -> None:
+        if self._fetcher:
+            self._fetcher.stop()
+
+    def append_thread_callback(self, callback: collections.Callable) -> None:
+        logger.warning('MailFetcherDaemon not implement append_thread_calback')
+        pass
+
+
 class MailSenderDaemon(Daemon):
     # NOTE: use *args and **kwargs because parent __init__ use strange
     # * parameter

tracim/tracim/lib/email_fetcher.py

+# -*- coding: utf-8 -*-
+
+import sys
+import time
+import imaplib
+import datetime
+import json
+import typing
+from email.message import Message
+from email.header import Header, decode_header, make_header
+from email.utils import parseaddr, parsedate_tz, mktime_tz
+from email import message_from_bytes
+
+import markdown
+import requests
+from bs4 import BeautifulSoup, Tag
+from email_reply_parser import EmailReplyParser
+
+from tracim.lib.base import logger
+
+TRACIM_SPECIAL_KEY_HEADER = 'X-Tracim-Key'
+# TODO BS 20171124: Think about replace thin dict config by object
+BEAUTIFULSOUP_HTML_BODY_PARSE_CONFIG = {
+    'tag_blacklist': ['script', 'style', 'blockquote'],
+    'class_blacklist': ['moz-cite-prefix', 'gmail_extra', 'gmail_quote',
+                        'yahoo_quoted'],
+    'id_blacklist': ['reply-intro'],
+    'tag_whitelist': ['a', 'b', 'strong', 'i', 'br', 'ul', 'li', 'ol',
+                      'em', 'i', 'u',
+                      'thead', 'tr', 'td', 'tbody', 'table', 'p', 'pre'],
+    'attrs_whitelist': ['href'],
+}
+CONTENT_TYPE_TEXT_PLAIN = 'text/plain'
+CONTENT_TYPE_TEXT_HTML = 'text/html'
+
+
+class DecodedMail(object):
+    def __init__(self, message: Message) -> None:
+        self._message = message
+
+    def _decode_header(self, header_title: str) -> typing.Optional[str]:
+        # FIXME : Handle exception
+        if header_title in self._message:
+            return str(make_header(decode_header(self._message[header_title])))
+        else:
+            return None
+
+    def get_subject(self) -> typing.Optional[str]:
+        return self._decode_header('subject')
+
+    def get_from_address(self) -> str:
+        return parseaddr(self._message['From'])[1]
+
+    def get_to_address(self) -> str:
+        return parseaddr(self._message['To'])[1]
+
+    def get_first_ref(self) -> str:
+        return parseaddr(self._message['References'])[1]
+
+    def get_special_key(self) -> typing.Optional[str]:
+        return self._decode_header(TRACIM_SPECIAL_KEY_HEADER)
+
+    def get_body(self) -> typing.Optional[str]:
+        body_part = self._get_mime_body_message()
+        body = None
+        if body_part:
+            charset = body_part.get_content_charset('iso-8859-1')
+            content_type = body_part.get_content_type()
+            if content_type == CONTENT_TYPE_TEXT_PLAIN:
+                txt_body = body_part.get_payload(decode=True).decode(
+                    charset)
+                body = DecodedMail._parse_txt_body(txt_body)
+
+            elif content_type == CONTENT_TYPE_TEXT_HTML:
+                html_body = body_part.get_payload(decode=True).decode(
+                    charset)
+                body = DecodedMail._parse_html_body(html_body)
+
+        return body
+
+    @classmethod
+    def _parse_txt_body(cls, txt_body: str) -> str:
+        txt_body = EmailReplyParser.parse_reply(txt_body)
+        html_body = markdown.markdown(txt_body)
+        body = DecodedMail._parse_html_body(html_body)
+        return body
+
+    @classmethod
+    def _parse_html_body(cls, html_body: str) -> str:
+        soup = BeautifulSoup(html_body, 'html.parser')
+        config = BEAUTIFULSOUP_HTML_BODY_PARSE_CONFIG
+        for tag in soup.findAll():
+            if DecodedMail._tag_to_extract(tag):
+                tag.extract()
+            elif tag.name.lower() in config['tag_whitelist']:
+                attrs = dict(tag.attrs)
+                for attr in attrs:
+                    if attr not in config['attrs_whitelist']:
+                        del tag.attrs[attr]
+            else:
+                tag.unwrap()
+        return str(soup)
+
+    @classmethod
+    def _tag_to_extract(cls, tag: Tag) -> bool:
+        config = BEAUTIFULSOUP_HTML_BODY_PARSE_CONFIG
+        if tag.name.lower() in config['tag_blacklist']:
+            return True
+        if 'class' in tag.attrs:
+            for elem in config['class_blacklist']:
+                if elem in tag.attrs['class']:
+                    return True
+        if 'id' in tag.attrs:
+            for elem in config['id_blacklist']:
+                if elem in tag.attrs['id']:
+                    return True
+        return False
+
+    def _get_mime_body_message(self) -> typing.Optional[Message]:
+        # TODO - G.M - 2017-11-16 - Use stdlib msg.get_body feature for py3.6+
+        part = None
+        # Check for html
+        for part in self._message.walk():
+            content_type = part.get_content_type()
+            content_dispo = str(part.get('Content-Disposition'))
+            if content_type == CONTENT_TYPE_TEXT_HTML \
+                    and 'attachment' not in content_dispo:
+                return part
+        # check for plain text
+        for part in self._message.walk():
+            content_type = part.get_content_type()
+            content_dispo = str(part.get('Content-Disposition'))
+            if content_type == CONTENT_TYPE_TEXT_PLAIN \
+                    and 'attachment' not in content_dispo:
+                return part
+        return part
+
+    def get_key(self) -> typing.Optional[str]:
+
+        """
+        key is the string contain in some mail header we need to retrieve.
+        First try checking special header, them check 'to' header
+        and finally check first(oldest) mail-id of 'references' header
+        """
+        first_ref = self.get_first_ref()
+        to_address = self.get_to_address()
+        special_key = self.get_special_key()
+
+        if special_key:
+            return special_key
+        if to_address:
+            return DecodedMail.find_key_from_mail_address(to_address)
+        if first_ref:
+            return DecodedMail.find_key_from_mail_address(first_ref)
+
+        return None
+
+    @classmethod
+    def find_key_from_mail_address(
+        cls,
+        mail_address: str,
+    ) -> typing.Optional[str]:
+        """ Parse mail_adress-like string
+        to retrieve key.
+
+        :param mail_address: user+key@something like string
+        :return: key
+        """
+        username = mail_address.split('@')[0]
+        username_data = username.split('+')
+        if len(username_data) == 2:
+            return username_data[1]
+        return None
+
+
+class MailFetcher(object):
+    def __init__(
+        self,
+        host: str,
+        port: str,
+        user: str,
+        password: str,
+        use_ssl: bool,
+        folder: str,
+        delay: int,
+        endpoint: str,
+        token: str,
+    ) -> None:
+        """
+        Fetch mail from a mailbox folder through IMAP and add their content to
+        Tracim through http according to mail Headers.
+        Fetch is regular.
+        :param host: imap server hostname
+        :param port: imap connection port
+        :param user: user login of mailbox
+        :param password: user password of mailbox
+        :param use_ssl: use imap over ssl connection
+        :param folder: mail folder where new mail are fetched
+        :param delay: seconds to wait before fetching new mail again
+        :param endpoint: tracim http endpoint where decoded mail are send.
+        :param token: token to authenticate http connexion
+        """
+        self._connection = None
+        self.host = host
+        self.port = port
+        self.user = user
+        self.password = password
+        self.use_ssl = use_ssl
+        self.folder = folder
+        self.delay = delay
+        self.endpoint = endpoint
+        self.token = token
+
+        self._is_active = True
+
+    def run(self) -> None:
+        while self._is_active:
+            time.sleep(self.delay)
+            try:
+                self._connect()
+                messages = self._fetch()
+                # TODO - G.M -  2017-11-22 retry sending unsended mail
+                # These mails are return by _notify_tracim, flag them with "unseen"
+                # or store them until new _notify_tracim call
+                cleaned_mails = [DecodedMail(msg) for msg in messages]
+                self._notify_tracim(cleaned_mails)
+                self._disconnect()
+            except Exception as e:
+                # TODO - G.M - 2017-11-23 - Identify possible exceptions
+                log = 'IMAP error: {}'
+                logger.warning(self, log.format(e.__str__()))
+
+    def stop(self) -> None:
+        self._is_active = False
+
+    def _connect(self) -> None:
+        # TODO - G.M - 2017-11-15 Verify connection/disconnection
+        # Are old connexion properly close this way ?
+        if self._connection:
+            self._disconnect()
+        # TODO - G.M - 2017-11-23 Support for predefined SSLContext ?
+        # without ssl_context param, tracim use default security configuration
+        # which is great in most case.
+        if self.use_ssl:
+            self._connection = imaplib.IMAP4_SSL(self.host, self.port)
+        else:
+            self._connection = imaplib.IMAP4(self.host, self.port)
+
+        try:
+            self._connection.login(self.user, self.password)
+        except Exception as e:
+            log = 'IMAP login error: {}'
+            logger.warning(self, log.format(e.__str__()))
+
+    def _disconnect(self) -> None:
+        if self._connection:
+            self._connection.close()
+            self._connection.logout()
+            self._connection = None
+
+    def _fetch(self) -> typing.List[Message]:
+        """
+        Get news message from mailbox
+        :return: list of new mails
+        """
+        messages = []
+        # select mailbox
+        rv, data = self._connection.select(self.folder)
+        if rv == 'OK':
+            # get mails
+            # TODO - G.M -  2017-11-15 Which files to select as new file ?
+            # Unseen file or All file from a directory (old one should be
+            #  moved/ deleted from mailbox during this process) ?
+            rv, data = self._connection.search(None, "(UNSEEN)")
+            if rv == 'OK':
+                # get mail content
+                for num in data[0].split():
+                    # INFO - G.M - 2017-11-23 - Fetch (RFC288) to retrieve all
+                    # complete mails see example : https://docs.python.org/fr/3.5/library/imaplib.html#imap4-example .  # nopep8
+                    # Be careful, This method remove also mails from Unseen
+                    # mails
+                    rv, data = self._connection.fetch(num, '(RFC822)')
+                    if rv == 'OK':
+                        msg = message_from_bytes(data[0][1])
+                        messages.append(msg)
+                    else:
+                        log = 'IMAP : Unable to get mail : {}'
+                        logger.debug(self, log.format(str(rv)))
+            else:
+                # FIXME : Distinct error from empty mailbox ?
+                pass
+        else:
+            log = 'IMAP : Unable to open mailbox : {}'
+            logger.debug(self, log.format(str(rv)))
+        return messages
+
+    def _notify_tracim(
+        self,
+        mails: typing.List[DecodedMail],
+    ) -> typing.List[DecodedMail]:
+        """
+        Send http request to tracim endpoint
+        :param mails: list of mails to send
+        :return: unsended mails
+        """
+        unsended_mails = []
+        # TODO BS 20171124: Look around mail.get_from_address(), mail.get_key()
+        # , mail.get_body() etc ... for raise InvalidEmailError if missing
+        #  required informations (actually get_from_address raise IndexError
+        #  if no from address for example) and catch it here
+        while mails:
+            mail = mails.pop()
+            msg = {'token': self.token,
+                   'user_mail': mail.get_from_address(),
+                   'content_id': mail.get_key(),
+                   'payload': {
+                       'content': mail.get_body(),
+                   }}
+            try:
+                r = requests.post(self.endpoint, json=msg)
+                if r.status_code not in [200, 204]:
+                    log = 'bad status code response when sending mail to tracim: {}'  # nopep8
+                    logger.error(self, log.format(str(r.status_code)))
+            # TODO - G.M - Verify exception correctly works
+            except requests.exceptions.Timeout as e:
+                log = 'Timeout error to transmit fetched mail to tracim : {}'
+                logger.error(self, log.format(str(e)))
+                unsended_mails.append(mail)
+                break
+            except requests.exceptions.RequestException as e:
+                log = 'Fail to transmit fetched mail to tracim : {}'
+                logger.error(self, log.format(str(e)))
+                break
+
+        return unsended_mails

tracim/tracim/lib/notifications.py

         for role in notifiable_roles:
             logger.info(self, 'Sending email to {}'.format(role.user.email))
             to_addr = formataddr((role.user.display_name, role.user.email))
+            #
+            # INFO - G.M - 2017-11-15 - set content_id in header to permit reply
+            # references can have multiple values, but only one in this case.
+            replyto_addr = self._global_config.EMAIL_NOTIFICATION_REPLY_TO_EMAIL.replace( # nopep8
+                '{content_id}',str(content.content_id)
+            )
 
+            reference_addr = self._global_config.EMAIL_NOTIFICATION_REFERENCES_EMAIL.replace( #nopep8
+                '{content_id}',str(content.content_id)
+             )
             #
             #  INFO - D.A. - 2014-11-06
             # We do not use .format() here because the subject defined in the .ini file
             subject = subject.replace(EST.WORKSPACE_LABEL, main_content.workspace.label.__str__())
             subject = subject.replace(EST.CONTENT_LABEL, main_content.label.__str__())
             subject = subject.replace(EST.CONTENT_STATUS_LABEL, main_content.get_status().label.__str__())
+            reply_to_label = l_('{username} & all members of {workspace}').format(
+                username=user.display_name,
+                workspace=main_content.workspace.label)
 
             message = MIMEMultipart('alternative')
             message['Subject'] = subject
             message['From'] = self._get_sender(user)
             message['To'] = to_addr
-
+            message['Reply-to'] = formataddr((reply_to_label, replyto_addr))
+            # INFO - G.M - 2017-11-15
+            # References can theorically have label, but in pratice, references
+            # contains only message_id from parents post in thread.
+            # To link this email to a content we create a virtual parent
+            # in reference who contain the content_id.
+            message['References'] = formataddr(('',reference_addr))
             body_text = self._build_email_body(self._global_config.EMAIL_NOTIFICATION_CONTENT_UPDATE_TEMPLATE_TEXT, role, content, user)
 
 

tracim/tracim/tests/library/test_email_fetcher.py

+from tracim.lib.email_fetcher import DecodedMail
+from tracim.tests import TestStandard
+
+class TestDecodedMail(TestStandard):
+    def test_unit__find_key_from_mail_address_no_key(self):
+        mail_address = "a@b"
+        assert DecodedMail.find_key_from_mail_address(mail_address) is None
+
+    def test_unit__find_key_from_mail_adress_key(self):
+        mail_address = "a+key@b"
+        assert DecodedMail.find_key_from_mail_address(mail_address) == 'key'