Merge pull request #506 from inkhey/feature/better_html_email_parsing

tracim/development.ini.base

@@ -226,6 +226,8 @@ email.reply.imap.use_ssl = true
 email.reply.token = mysecuretoken
 # Delay in seconds between each check
 email.reply.check.heartbeat = 60
+email.reply.use_html_parsing = true
+email.reply.use_txt_parsing = true
 
 ## Radical (CalDav server) configuration
 # radicale.server.host = 0.0.0.0

tracim/tracim/config/app_cfg.py

@@ -128,7 +128,7 @@ def start_daemons(manager: DaemonsManager):
         manager.run('mail_sender', MailSenderDaemon)
 
     if cfg.EMAIL_REPLY_ACTIVATED:
-        manager.run('mail_fetcher',MailFetcherDaemon)
+        manager.run('mail_fetcher', MailFetcherDaemon)
 
 
 def configure_depot():
@@ -384,6 +384,14 @@ class CFG(object):
         self.EMAIL_REPLY_IMAP_USE_SSL = asbool(tg.config.get(
             'email.reply.imap.use_ssl',
         ))
+        self.EMAIL_REPLY_USE_HTML_PARSING = asbool(tg.config.get(
+            'email.reply.use_html_parsing',
+            True,
+        ))
+        self.EMAIL_REPLY_USE_TXT_PARSING = asbool(tg.config.get(
+            'email.reply.use_txt_parsing',
+            True,
+        ))
 
         self.TRACKER_JS_PATH = tg.config.get(
             'js_tracker_path',

tracim/tracim/lib/daemons.py

@@ -177,6 +177,8 @@ class MailFetcherDaemon(Daemon):
             # FIXME - G.M - 2017-11-15 - proper tracim url formatting
             endpoint=cfg.WEBSITE_BASE_URL + "/events",
             token=cfg.EMAIL_REPLY_TOKEN,
+            use_html_parsing=cfg.EMAIL_REPLY_USE_HTML_PARSING,
+            use_txt_parsing=cfg.EMAIL_REPLY_USE_TXT_PARSING,
         )
         self._fetcher.run()
 

tracim/tracim/lib/email_fetcher.py

@@ -4,32 +4,20 @@ import time
 import imaplib
 import json
 import typing
-from email.message import Message
+from email import message_from_bytes
 from email.header import decode_header
 from email.header import make_header
+from email.message import Message
 from email.utils import parseaddr
-from email import message_from_bytes
 
 import markdown
 import requests
-from bs4 import BeautifulSoup
-from bs4 import Tag
 from email_reply_parser import EmailReplyParser
-
 from tracim.lib.base import logger
+from tracim.lib.email_processing.parser import ParsedHTMLMail
+from tracim.lib.email_processing.sanitizer import HtmlSanitizer
 
 TRACIM_SPECIAL_KEY_HEADER = 'X-Tracim-Key'
-# TODO BS 20171124: Think about replace thin dict config by object
-BEAUTIFULSOUP_HTML_BODY_PARSE_CONFIG = {
-    'tag_blacklist': ['script', 'style', 'blockquote'],
-    'class_blacklist': ['moz-cite-prefix', 'gmail_extra', 'gmail_quote',
-                        'yahoo_quoted'],
-    'id_blacklist': ['reply-intro'],
-    'tag_whitelist': ['a', 'b', 'strong', 'i', 'br', 'ul', 'li', 'ol',
-                      'em', 'i', 'u',
-                      'thead', 'tr', 'td', 'tbody', 'table', 'p', 'pre'],
-    'attrs_whitelist': ['href'],
-}
 CONTENT_TYPE_TEXT_PLAIN = 'text/plain'
 CONTENT_TYPE_TEXT_HTML = 'text/html'
 
@@ -60,7 +48,11 @@ class DecodedMail(object):
     def get_special_key(self) -> typing.Optional[str]:
         return self._decode_header(TRACIM_SPECIAL_KEY_HEADER)
 
-    def get_body(self) -> typing.Optional[str]:
+    def get_body(
+            self,
+            use_html_parsing=True,
+            use_txt_parsing=True,
+    ) -> typing.Optional[str]:
         body_part = self._get_mime_body_message()
         body = None
         if body_part:
@@ -69,53 +61,20 @@ class DecodedMail(object):
             if content_type == CONTENT_TYPE_TEXT_PLAIN:
                 txt_body = body_part.get_payload(decode=True).decode(
                     charset)
-                body = DecodedMail._parse_txt_body(txt_body)
+                if use_txt_parsing:
+                    txt_body = EmailReplyParser.parse_reply(txt_body)
+                html_body = markdown.markdown(txt_body)
+                body = HtmlSanitizer.sanitize(html_body)
 
             elif content_type == CONTENT_TYPE_TEXT_HTML:
                 html_body = body_part.get_payload(decode=True).decode(
                     charset)
-                body = DecodedMail._parse_html_body(html_body)
-
-        return body
+                if use_html_parsing:
+                    html_body = str(ParsedHTMLMail(html_body))
+                body = HtmlSanitizer.sanitize(html_body)
 
-    @classmethod
-    def _parse_txt_body(cls, txt_body: str) -> str:
-        txt_body = EmailReplyParser.parse_reply(txt_body)
-        html_body = markdown.markdown(txt_body)
-        body = DecodedMail._parse_html_body(html_body)
         return body
 
-    @classmethod
-    def _parse_html_body(cls, html_body: str) -> str:
-        soup = BeautifulSoup(html_body, 'html.parser')
-        config = BEAUTIFULSOUP_HTML_BODY_PARSE_CONFIG
-        for tag in soup.findAll():
-            if DecodedMail._tag_to_extract(tag):
-                tag.extract()
-            elif tag.name.lower() in config['tag_whitelist']:
-                attrs = dict(tag.attrs)
-                for attr in attrs:
-                    if attr not in config['attrs_whitelist']:
-                        del tag.attrs[attr]
-            else:
-                tag.unwrap()
-        return str(soup)
-
-    @classmethod
-    def _tag_to_extract(cls, tag: Tag) -> bool:
-        config = BEAUTIFULSOUP_HTML_BODY_PARSE_CONFIG
-        if tag.name.lower() in config['tag_blacklist']:
-            return True
-        if 'class' in tag.attrs:
-            for elem in config['class_blacklist']:
-                if elem in tag.attrs['class']:
-                    return True
-        if 'id' in tag.attrs:
-            for elem in config['id_blacklist']:
-                if elem in tag.attrs['id']:
-                    return True
-        return False
-
     def _get_mime_body_message(self) -> typing.Optional[Message]:
         # TODO - G.M - 2017-11-16 - Use stdlib msg.get_body feature for py3.6+
         part = None
@@ -185,6 +144,8 @@ class MailFetcher(object):
         delay: int,
         endpoint: str,
         token: str,
+        use_html_parsing: bool,
+        use_txt_parsing: bool,
     ) -> None:
         """
         Fetch mail from a mailbox folder through IMAP and add their content to
@@ -199,6 +160,8 @@ class MailFetcher(object):
         :param delay: seconds to wait before fetching new mail again
         :param endpoint: tracim http endpoint where decoded mail are send.
         :param token: token to authenticate http connexion
+        :param use_html_parsing: parse html mail
+        :param use_txt_parsing: parse txt mail
         """
         self._connection = None
         self.host = host
@@ -210,6 +173,8 @@ class MailFetcher(object):
         self.delay = delay
         self.endpoint = endpoint
         self.token = token
+        self.use_html_parsing = use_html_parsing
+        self.use_txt_parsing = use_txt_parsing
 
         self._is_active = True
 
@@ -222,7 +187,7 @@ class MailFetcher(object):
                 self._connect()
                 messages = self._fetch()
                 # TODO - G.M -  2017-11-22 retry sending unsended mail
-                # These mails are return by _notify_tracim, flag them with "unseen"
+                # These mails are return by _notify_tracim, flag them with "unseen" # nopep8
                 # or store them until new _notify_tracim call
                 cleaned_mails = [DecodedMail(msg) for msg in messages]
                 self._notify_tracim(cleaned_mails)
@@ -350,7 +315,9 @@ class MailFetcher(object):
                    'user_mail': mail.get_from_address(),
                    'content_id': mail.get_key(),
                    'payload': {
-                       'content': mail.get_body(),
+                       'content': mail.get_body(
+                           use_html_parsing=self.use_html_parsing,
+                           use_txt_parsing=self.use_txt_parsing),
                    }}
             try:
                 logger.debug(

tracim/tracim/lib/email_processing/checkers.py

@@ -0,0 +1,211 @@
+# -*- coding: utf-8 -*-
+import typing
+
+from bs4 import Tag, NavigableString
+
+
+class ProprietaryHTMLAttrValues(object):
+    """
+    This are all Proprietary (mail client specific) html attr value we need to
+    check Html Elements
+    """
+    # Gmail
+    Gmail_extras_class = 'gmail_extra'
+    Gmail_quote_class = 'gmail_quote'
+    Gmail_signature_class = 'gmail_signature'
+    # Thunderbird
+    Thunderbird_quote_prefix_class = 'moz-cite-prefix'
+    Thunderbird_signature_class = 'moz-signature'
+    # Outlook.com
+    Outlook_com_quote_id = 'divRplyFwdMsg'
+    Outlook_com_signature_id = 'Signature'
+    Outlook_com_wrapper_id = 'divtagdefaultwrapper'
+    # Yahoo
+    Yahoo_quote_class = 'yahoo_quoted'
+    # Roundcube
+    # INFO - G.M - 2017-11-29 - New tag
+    # see : https://github.com/roundcube/roundcubemail/issues/6049
+    Roundcube_quote_prefix_class = 'reply-intro'
+
+
+class HtmlChecker(object):
+
+    @classmethod
+    def _has_attr_value(
+            cls,
+            elem: typing.Union[Tag, NavigableString],
+            attribute_name: str,
+            attribute_value: str,
+    )-> bool:
+        """
+        Check if elem contains attribute named attribute_name with
+        attribute_value : example <a id="ident"> elem contain attribute
+        with id as attribute_name and ident as attribute_value.
+        Checking is not case_sensitive.
+
+        :param elem: Tag or String Html Element
+        :param attribute_name: Html attribute name
+        :param attribute_value: Html attribute value
+        :return: True only if Element contain this attribute.
+        """
+        if isinstance(elem, Tag) and attribute_name in elem.attrs:
+            # INFO - G.M - 2017-12-01 - attrs[value}] can be string or list
+            # use get_attribute_list to always check in a list
+            # see https://www.crummy.com/software/BeautifulSoup/bs4/doc/#multi-valued-attributes # nopep8
+            values_lower = [value.lower()
+                            for value
+                            in elem.get_attribute_list(attribute_name)]
+            return attribute_value.lower() in values_lower
+        return False
+
+
+class HtmlMailQuoteChecker(HtmlChecker):
+    """
+    Check if one HTML Element from Body Mail look-like a quote or not.
+    """
+    @classmethod
+    def is_quote(
+            cls,
+            elem: typing.Union[Tag, NavigableString]
+    ) -> bool:
+        return cls._is_standard_quote(elem) \
+               or cls._is_thunderbird_quote(elem) \
+               or cls._is_gmail_quote(elem) \
+               or cls._is_outlook_com_quote(elem) \
+               or cls._is_yahoo_quote(elem) \
+               or cls._is_roundcube_quote(elem)
+
+    @classmethod
+    def _is_standard_quote(
+            cls,
+            elem: typing.Union[Tag, NavigableString]
+    ) -> bool:
+        if isinstance(elem, Tag) \
+                and elem.name.lower() == 'blockquote':
+            return True
+        return False
+
+    @classmethod
+    def _is_thunderbird_quote(
+            cls,
+            elem: typing.Union[Tag, NavigableString]
+    ) -> bool:
+        return cls._has_attr_value(
+            elem,
+            'class',
+            ProprietaryHTMLAttrValues.Thunderbird_quote_prefix_class)
+
+    @classmethod
+    def _is_gmail_quote(
+            cls,
+            elem: typing.Union[Tag, NavigableString]
+    ) -> bool:
+        if cls._has_attr_value(
+                elem,
+                'class',
+                ProprietaryHTMLAttrValues.Gmail_extras_class):
+            for child in elem.children:
+                if cls._has_attr_value(
+                        child,
+                        'class',
+                        ProprietaryHTMLAttrValues.Gmail_quote_class):
+                    return True
+        return False
+
+    @classmethod
+    def _is_outlook_com_quote(
+        cls,
+        elem: typing.Union[Tag, NavigableString]
+    ) -> bool:
+        if cls._has_attr_value(
+                elem,
+                'id',
+                ProprietaryHTMLAttrValues.Outlook_com_quote_id):
+            return True
+        return False
+
+    @classmethod
+    def _is_yahoo_quote(
+            cls,
+            elem: typing.Union[Tag, NavigableString]
+    ) -> bool:
+        return cls._has_attr_value(
+            elem,
+            'class',
+            ProprietaryHTMLAttrValues.Yahoo_quote_class)
+
+    @classmethod
+    def _is_roundcube_quote(
+            cls,
+            elem: typing.Union[Tag, NavigableString]
+    ) -> bool:
+        return cls._has_attr_value(
+            elem,
+            'id',
+            ProprietaryHTMLAttrValues.Roundcube_quote_prefix_class)
+
+
+class HtmlMailSignatureChecker(HtmlChecker):
+    """
+    Check if one HTML Element from Body Mail look-like a signature or not.
+    """
+
+    @classmethod
+    def is_signature(
+            cls,
+            elem: typing.Union[Tag, NavigableString]
+    ) -> bool:
+        return cls._is_thunderbird_signature(elem) \
+               or cls._is_gmail_signature(elem) \
+               or cls._is_outlook_com_signature(elem)
+
+    @classmethod
+    def _is_thunderbird_signature(
+            cls,
+            elem: typing.Union[Tag, NavigableString]
+    ) -> bool:
+        return cls._has_attr_value(
+            elem,
+            'class',
+            ProprietaryHTMLAttrValues.Thunderbird_signature_class)
+
+    @classmethod
+    def _is_gmail_signature(
+            cls,
+            elem: typing.Union[Tag, NavigableString]
+    ) -> bool:
+        if cls._has_attr_value(
+                elem,
+                'class',
+                ProprietaryHTMLAttrValues.Gmail_signature_class):
+            return True
+        if cls._has_attr_value(
+                elem,
+                'class',
+                ProprietaryHTMLAttrValues.Gmail_extras_class):
+            for child in elem.children:
+                if cls._has_attr_value(
+                        child,
+                        'class',
+                        ProprietaryHTMLAttrValues.Gmail_signature_class):
+                    return True
+        if isinstance(elem, Tag) and elem.name.lower() == 'div':
+            for child in elem.children:
+                if cls._has_attr_value(
+                        child,
+                        'class',
+                        ProprietaryHTMLAttrValues.Gmail_signature_class):
+                    return True
+        return False
+
+    @classmethod
+    def _is_outlook_com_signature(
+            cls,
+            elem: typing.Union[Tag, NavigableString]
+    ) -> bool:
+        if cls._has_attr_value(
+                elem,
+                'id',
+                ProprietaryHTMLAttrValues.Outlook_com_signature_id):
+            return True
+        return False

tracim/tracim/lib/email_processing/models.py

@@ -0,0 +1,115 @@
+from bs4 import BeautifulSoup
+
+# -*- coding: utf-8 -*-
+
+
+class BodyMailPartType(object):
+    Signature = 'sign'
+    Main = 'main'
+    Quote = 'quote'
+
+
+class BodyMailPart(object):
+    def __init__(
+            self,
+            text: str,
+            part_type: str
+    )-> None:
+        self.text = text
+        self.part_type = part_type
+
+
+class BodyMailParts(object):
+    """
+    Data Structure to Distinct part of a Mail body into a "list" of BodyMailPart
+    When 2 similar BodyMailPart (same part_type) are added one after the other,
+    it doesn't create a new Part, it just merge those elements into one.
+    It should always have only one Signature type part, normally
+    at the end of the body.
+    This object doesn't provide other set method than append() in order to
+    preserve object coherence.
+    """
+    def __init__(self) -> None:
+        self._list = []  # type; List[BodyMailPart]
+        # INFO - G.M -
+        # automatically merge new value with last item if true, without any
+        # part_type check, same type as the older one, useful when some tag
+        # say "all elem after me is Signature"
+        self.follow = False
+
+    def __len__(self) -> int:
+        return len(self._list)
+
+    def __getitem__(self, index) -> BodyMailPart:
+        return self._list[index]
+
+    def __delitem__(self, index) -> None:
+        del self._list[index]
+        # FIXME - G.M - 2017-11-27 - Preserve BodyMailParts consistence
+        # check elem after and before index and merge them if necessary.
+
+    def append(self, value) -> None:
+        BodyMailParts._check_value(value)
+        self._append(value)
+
+    def _append(self, value, follow=None) -> None:
+        if follow is None:
+            follow = self.follow
+
+        if len(self._list) < 1:
+            self._list.append(value)
+        else:
+            if self._list[-1].part_type == value.part_type or follow:
+                self._list[-1].text += value.text
+            else:
+                self._list.append(value)
+
+    @classmethod
+    def _check_value(cls, value) -> None:
+        if not isinstance(value, BodyMailPart):
+            raise TypeError()
+
+    def drop_part_type(self, part_type: str) -> None:
+        """
+        Drop all elem of one part_type
+        :param part_type: part_type to completely remove
+        :return: None
+        """
+        new_list = [x for x in self._list if x.part_type != part_type]
+        self._list = []
+
+        # INFO - G.M - 2017-11-27 - use append() to have a consistent list
+        for elem in new_list:
+            self._append(elem, follow=False)
+
+    def get_nb_part_type(self, part_type: str) -> int:
+        """
+        Get number of elements of one part_type
+        :param part_type: part_type to check
+        :return: number of part_type elements
+        """
+        count = 0
+        for elem in self._list:
+            if elem.part_type == part_type:
+                count += 1
+        return count
+
+    def __str__(self) -> str:
+        s_mail = ''
+        for elem in self._list:
+            s_mail += elem.text
+        return str(s_mail)
+
+
+class HtmlBodyMailParts(BodyMailParts):
+
+    def append(self, value):
+        # INFO - G.M - 2017-12-01 - Override part_type is elem has no content.
+        # Choose last elem part_type instead of the proposed one.
+        if len(self._list) > 0:
+            txt = BeautifulSoup(value.text, 'html.parser').get_text()
+            txt = txt.replace('\n', '').strip()
+            if not txt:
+                value.part_type = self._list[-1].part_type
+        BodyMailParts._check_value(value)
+        BodyMailParts._append(self, value)

tracim/tracim/lib/email_processing/parser.py

@@ -0,0 +1,128 @@
+# -*- coding: utf-8 -*-
+from bs4 import BeautifulSoup
+
+from tracim.lib.email_processing.checkers import ProprietaryHTMLAttrValues
+from tracim.lib.email_processing.checkers import HtmlMailQuoteChecker
+from tracim.lib.email_processing.checkers import HtmlMailSignatureChecker
+from tracim.lib.email_processing.models import BodyMailPartType
+from tracim.lib.email_processing.models import BodyMailPart
+from tracim.lib.email_processing.models import HtmlBodyMailParts
+
+
+class PreSanitizeConfig(object):
+    """
+    To avoid problems, html need to be sanitize a bit during parsing to distinct
+    Main,Quote and Signature elements
+    """
+    meta_tag = ['body', 'div']
+
+
+class ParsedHTMLMail(object):
+    """
+    Parse HTML Mail depending of some rules.
+    Distinct part of html mail body using BodyMailParts object and
+    process differents rules using HtmlChecker(s)
+    """
+
+    def __init__(self, html_body: str):
+        self.src_html_body = html_body
+
+    def __str__(self):
+        return str(self._parse_mail())
+
+    def get_elements(self) -> HtmlBodyMailParts:
+        tree = self._get_proper_main_body_tree()
+        return self._distinct_elements(tree)
+
+    def _parse_mail(self) -> HtmlBodyMailParts:
+        elements = self.get_elements()
+        elements = self._process_elements(elements)
+        return elements
+
+    def _get_proper_main_body_tree(self) -> BeautifulSoup:
+        """
+        Get html body tree without some kind of wrapper.
+        We need to have text, quote and signature parts at the same tree level
+        """
+        tree = BeautifulSoup(self.src_html_body, 'html.parser')
+
+        # Only parse body part of html if available
+        subtree = tree.find('body')
+        if subtree:
+            tree = BeautifulSoup(str(subtree), 'html.parser')
+
+        # if some kind of "meta_div", unwrap it
+        while len(tree.findAll(recursive=None)) == 1 and \
+                tree.find().name.lower() in PreSanitizeConfig.meta_tag:
+            tree.find().unwrap()
+
+        for tag in tree.findAll():
+            # HACK - G.M - 2017-11-28 - Unwrap outlook.com mail
+            # if Text -> Signature -> Quote Mail
+            # Text and signature are wrapped into divtagdefaultwrapper
+            if tag.attrs.get('id'):
+                if ProprietaryHTMLAttrValues.Outlook_com_wrapper_id\
+                        in tag.attrs['id']:
+                    tag.unwrap()
+        return tree
+
+    @classmethod
+    def _distinct_elements(cls, tree: BeautifulSoup) -> HtmlBodyMailParts:
+        parts = HtmlBodyMailParts()
+        for elem in list(tree):
+            part_txt = str(elem)
+            part_type = BodyMailPartType.Main
+
+            if HtmlMailQuoteChecker.is_quote(elem):
+                part_type = BodyMailPartType.Quote
+            elif HtmlMailSignatureChecker.is_signature(elem):
+                part_type = BodyMailPartType.Signature
+
+            part = BodyMailPart(part_txt, part_type)
+            parts.append(part)
+            # INFO - G.M - 2017-11-28 - Outlook.com special case
+            # all after quote tag is quote
+            if HtmlMailQuoteChecker._is_outlook_com_quote(elem):
+                parts.follow = True
+        return parts
+
+    @classmethod
+    def _process_elements(
+            cls,
+            elements: HtmlBodyMailParts,
+    ) -> HtmlBodyMailParts:
+        if len(elements) >= 2:
+            # Case 1 and 2, only one main and one quote
+            if elements.get_nb_part_type('main') == 1 and \
+                            elements.get_nb_part_type('quote') == 1:
+                # Case 1 : Main first
+                if elements[0].part_type == BodyMailPartType.Main:
+                    cls._process_main_first_case(elements)
+                # Case 2 : Quote first
+                if elements[0].part_type == BodyMailPartType.Quote:
+                    cls._process_quote_first_case(elements)
+            else:
+                # Case 3 : Multiple quotes and/or main
+                cls._process_multiples_elems_case(elements)
+        else:
+            cls._process_default_case(elements)
+            # default case (only one element or empty list)
+        return elements
+
+    @classmethod
+    def _process_quote_first_case(cls, elements: HtmlBodyMailParts) -> None:
+        elements.drop_part_type(BodyMailPartType.Signature)
+
+    @classmethod
+    def _process_main_first_case(cls, elements: HtmlBodyMailParts) -> None:
+        elements.drop_part_type(BodyMailPartType.Quote)
+        elements.drop_part_type(BodyMailPartType.Signature)
+
+    @classmethod
+    def _process_multiples_elems_case(cls, elements: HtmlBodyMailParts) -> None:
+        elements.drop_part_type(BodyMailPartType.Signature)
+
+    @classmethod
+    def _process_default_case(cls, elements: HtmlBodyMailParts) -> None:
+        elements.drop_part_type(BodyMailPartType.Quote)
+        elements.drop_part_type(BodyMailPartType.Signature)

tracim/tracim/lib/email_processing/sanitizer.py

@@ -0,0 +1,79 @@
+from bs4 import BeautifulSoup, Tag
+
+
+class HtmlSanitizerConfig(object):
+    # some Default_html_tags type
+    HTML_Heading_tag = ['h1', 'h2', 'h3', 'h4', 'h5', 'h6']
+    HTML_Text_parts_tag = ['p',
+                           'br', 'hr',
+                           'pre', 'code', 'samp',  # preformatted content
+                           'q', 'blockquote',  # quotes
+                           ]
+    HTML_Text_format_tag = ['b', 'i', 'u', 'small', 'sub', 'sup', ]
+    HTML_Text_semantic_tag = ['strong', 'em',
+                              'mark', 'cite', 'dfn',
+                              'del', 'ins', ]
+    HTML_Table_tag = ['table',
+                      'thead', 'tfoot', 'tbody',
+                      'tr', 'td', 'caption', ]
+
+    HTML_List_tag = ['ul', 'li', 'ol',  # simple list
+                     'dl', 'dt', 'dd', ]  # definition list
+
+    # Rules
+    Tag_whitelist = HTML_Heading_tag \
+                    + HTML_Text_parts_tag \
+                    + HTML_Text_format_tag \
+                    + HTML_Text_semantic_tag \
+                    + HTML_Table_tag \
+                    + HTML_List_tag
+
+    Tag_blacklist = ['script', 'style']
+
+    # TODO - G.M - 2017-12-01 - Think about removing class/id Blacklist
+    # These elements are no longer required.
+    Class_blacklist = []
+    Id_blacklist = []
+
+    Attrs_whitelist = ['href']
+
+
+class HtmlSanitizer(object):
+    """
+    Sanitize Html Rules :
+    - Tag :
+      - Remove Tag_blacklist tag
+      - Keep Tag_whitelist tag
+      - Unwrap others tags
+    - Attrs :
+      - Remove non-whitelisted attributes
+    """
+
+    @classmethod
+    def sanitize(cls, html_body: str) -> str:
+        soup = BeautifulSoup(html_body, 'html.parser')
+        for tag in soup.findAll():
+            if cls._tag_to_extract(tag):
+                tag.extract()
+            elif tag.name.lower() in HtmlSanitizerConfig.Tag_whitelist:
+                attrs = dict(tag.attrs)
+                for attr in attrs:
+                    if attr not in HtmlSanitizerConfig.Attrs_whitelist:
+                        del tag.attrs[attr]
+            else:
+                tag.unwrap()
+        return str(soup)
+
+    @classmethod
+    def _tag_to_extract(cls, tag: Tag) -> bool:
+        if tag.name.lower() in HtmlSanitizerConfig.Tag_blacklist:
+            return True
+        if 'class' in tag.attrs:
+            for elem in HtmlSanitizerConfig.Class_blacklist:
+                if elem in tag.attrs['class']:
+                    return True
+        if 'id' in tag.attrs:
+            for elem in HtmlSanitizerConfig.Id_blacklist:
+                if elem in tag.attrs['id']:
+                    return True
+        return False

tracim/tracim/tests/library/test_email_body_parser.py

@@ -0,0 +1,759 @@
+from bs4 import BeautifulSoup
+from nose.tools import raises
+
+from tracim.lib.email_processing.checkers import HtmlMailQuoteChecker
+from tracim.lib.email_processing.checkers import HtmlMailSignatureChecker
+from tracim.lib.email_processing.parser import ParsedHTMLMail
+from tracim.lib.email_processing.models import BodyMailPartType
+from tracim.lib.email_processing.models import BodyMailPart
+from tracim.lib.email_processing.models import BodyMailParts
+from tracim.tests import TestStandard
+
+
+class TestHtmlMailQuoteChecker(TestStandard):
+    def test_unit__is_standard_quote_ok(self):
+        soup = BeautifulSoup('<blockquote></blockquote>', 'html.parser')
+        main_elem = soup.find()
+        assert HtmlMailQuoteChecker._is_standard_quote(main_elem) is True
+
+    def test_unit__is_standard_quote_no(self):
+        soup = BeautifulSoup('<a></a>', 'html.parser')
+        main_elem = soup.find()
+        assert HtmlMailQuoteChecker._is_standard_quote(main_elem) is False
+
+    def test_unit__is_thunderbird_quote_ok(self):
+        soup = BeautifulSoup('<div class="moz-cite-prefix"></div>',
+                             'html.parser')
+        main_elem = soup.find()
+        assert HtmlMailQuoteChecker._is_thunderbird_quote(main_elem) is True
+
+    def test_unit__is_thunderbird_quote_no(self):
+        soup = BeautifulSoup('<div class="nothing"></div>', 'html.parser')
+        main_elem = soup.find()
+        assert HtmlMailQuoteChecker._is_thunderbird_quote(main_elem) is False
+
+    def test_unit__is_gmail_quote_ok(self):
+        html = '<div class="gmail_extra">' + \
+              '<a></a><div class="gmail_quote"></div>' + \
+              '</div>'
+        soup = BeautifulSoup(html, 'html.parser')
+        main_elem = soup.find()
+        assert HtmlMailQuoteChecker._is_gmail_quote(main_elem) is True
+
+    def test_unit__is_gmail_quote_no(self):
+        soup = BeautifulSoup('<div class="nothing"></div>', 'html.parser')
+        main_elem = soup.find()
+        assert HtmlMailQuoteChecker._is_gmail_quote(main_elem) is False
+
+    def test_unit__is_gmail_quote_no_2(self):
+        html = '<div class="gmail_extra">' + \
+              '<a></a><div class="gmail_signature"></div>' + \
+              '</div>'
+        soup = BeautifulSoup(html, 'html.parser')
+        main_elem = soup.find()
+        assert HtmlMailQuoteChecker._is_gmail_quote(main_elem) is False
+
+    def test_unit__is_outlook_com_quote_ok(self):
+        soup = BeautifulSoup('<div id="divRplyFwdMsg"></div>', 'html.parser')
+        main_elem = soup.find()
+        assert HtmlMailQuoteChecker._is_outlook_com_quote(main_elem) is True
+
+    def test_unit__is_outlook_com_quote_no(self):
+        soup = BeautifulSoup('<div id="Signature"></div>', 'html.parser')
+        main_elem = soup.find()
+        assert HtmlMailQuoteChecker._is_outlook_com_quote(main_elem) is False
+
+    # TODO - G.M - 2017-11-24 - Check Yahoo and New roundcube html mail with
+    # correct mail example
+
+
+class TestHtmlMailSignatureChecker(TestStandard):
+    def test_unit__is_thunderbird_signature_ok(self):
+        soup = BeautifulSoup('<div class="moz-signature"></div>', 'html.parser')
+        main_elem = soup.find()
+        assert HtmlMailSignatureChecker._is_thunderbird_signature(main_elem) is True  # nopep8
+
+    def test_unit__is_thunderbird_signature_no(self):
+        soup = BeautifulSoup('<div class="other"></div>', 'html.parser')
+        main_elem = soup.find()
+        assert HtmlMailSignatureChecker._is_thunderbird_signature(main_elem) is False  # nopep8
+
+    def test_unit__is_gmail_signature_ok(self):
+        html = '<div class="gmail_extra">' + \
+               '<a></a><div class="gmail_quote"></div>' + \
+               '</div>'
+        soup = BeautifulSoup(html, 'html.parser')
+        main_elem = soup.find()
+        assert HtmlMailSignatureChecker._is_gmail_signature(main_elem) is False
+
+    def test_unit__is_gmail_signature_no(self):
+        soup = BeautifulSoup('<div class="nothing"></div>', 'html.parser')
+        main_elem = soup.find()
+        assert HtmlMailSignatureChecker._is_gmail_signature(main_elem) is False
+
+    def test_unit__is_gmail_signature_yes(self):
+        html = '<div class="gmail_extra">' + \
+               '<a></a><div class="gmail_signature"></div>' + \
+               '</div>'
+        soup = BeautifulSoup(html, 'html.parser')
+        main_elem = soup.find()
+        assert HtmlMailSignatureChecker._is_gmail_signature(main_elem) is True
+
+    def test_unit__is_gmail_signature_yes_2(self):
+        html = '<div class="gmail_signature">' + \
+               '</div>'
+        soup = BeautifulSoup(html, 'html.parser')
+        main_elem = soup.find()
+        assert HtmlMailSignatureChecker._is_gmail_signature(main_elem) is True
+
+    def test_unit__is_outlook_com_signature_no(self):
+        soup = BeautifulSoup('<div id="divRplyFwdMsg"></div>', 'html.parser')
+        main_elem = soup.find()
+        assert HtmlMailSignatureChecker._is_outlook_com_signature(main_elem) \
+               is False
+
+    def test_unit__is_outlook_com_signature_ok(self):
+        soup = BeautifulSoup('<div id="Signature"></div>', 'html.parser')
+        main_elem = soup.find()
+        assert HtmlMailSignatureChecker._is_outlook_com_signature(main_elem) \
+               is True
+
+
+class TestBodyMailsParts(TestStandard):
+
+    def test_unit__std_list_methods(self):
+        mail_parts = BodyMailParts()
+        assert len(mail_parts) == 0
+        a = BodyMailPart('a', BodyMailPartType.Main)
+        mail_parts._list.append(a)
+        assert len(mail_parts) == 1
+        assert mail_parts[0] == a
+        del mail_parts[0]
+        assert len(mail_parts) == 0
+
+    def test_unit__append_same_type(self):
+        mail_parts = BodyMailParts()
+        a = BodyMailPart('a', BodyMailPartType.Main)
+        mail_parts._append(a)
+        b = BodyMailPart('b', BodyMailPartType.Main)
+        mail_parts._append(b)
+        assert len(mail_parts) == 1
+        assert mail_parts[0].part_type == BodyMailPartType.Main
+        assert mail_parts[0].text == 'ab'
+
+    def test_unit__append_different_type(self):
+        mail_parts = BodyMailParts()
+        a = BodyMailPart('a', BodyMailPartType.Main)
+        mail_parts.append(a)
+        b = BodyMailPart('b', BodyMailPartType.Quote)
+        mail_parts._append(b)
+        assert len(mail_parts) == 2
+        assert mail_parts[0] == a
+        assert mail_parts[1] == b
+
+    def test_unit__append_follow(self):
+        mail_parts = BodyMailParts()
+        mail_parts.follow = True
+        a = BodyMailPart('a', BodyMailPartType.Main)
+        mail_parts._append(a)
+        b = BodyMailPart('b', BodyMailPartType.Quote)
+        mail_parts._append(b)
+        assert len(mail_parts) == 1
+        assert mail_parts[0].part_type == BodyMailPartType.Main
+        assert mail_parts[0].text == 'ab'
+
+    def test_unit__append_dont_follow_when_first(self):
+        mail_parts = BodyMailParts()
+        a = BodyMailPart('a', BodyMailPartType.Main)
+        mail_parts._append(a, follow=True)
+        assert len(mail_parts) == 1
+        assert mail_parts[0].part_type == BodyMailPartType.Main
+        assert mail_parts[0].text == 'a'
+
+    @raises(TypeError)
+    def test_unit__check_value__type_error(self):
+        mail_parts = BodyMailParts()
+        mail_parts._check_value('a')
+
+    def test_unit__check_value__ok(self):
+        mail_parts = BodyMailParts()
+        a = BodyMailPart('a', BodyMailPartType.Main)
+        mail_parts._check_value(a)
+
+    def test_unit__drop_part_type(self):
+        mail_parts = BodyMailParts()
+        a = BodyMailPart('a', BodyMailPartType.Main)
+        mail_parts._list.append(a)
+        b = BodyMailPart('b', BodyMailPartType.Quote)
+        mail_parts._list.append(b)
+        c = BodyMailPart('c', BodyMailPartType.Signature)
+        mail_parts._list.append(c)
+        mail_parts.drop_part_type(BodyMailPartType.Quote)
+        assert len(mail_parts) == 2
+        assert mail_parts[0].text == 'a'
+        assert mail_parts[0].part_type == BodyMailPartType.Main
+        assert len(mail_parts) == 2
+        assert mail_parts[1].text == 'c'
+        assert mail_parts[1].part_type == BodyMailPartType.Signature
+
+    def test_unit__drop_part_type_verify_no_follow_incidence(self):
+        mail_parts = BodyMailParts()
+        a = BodyMailPart('a', BodyMailPartType.Main)
+        mail_parts._list.append(a)
+        b = BodyMailPart('b', BodyMailPartType.Quote)
+        mail_parts._list.append(b)
+        c = BodyMailPart('c', BodyMailPartType.Signature)
+        mail_parts._list.append(c)
+        mail_parts.follow = True
+        mail_parts.drop_part_type(BodyMailPartType.Quote)
+        assert len(mail_parts) == 2
+        assert mail_parts[0].text == 'a'
+        assert mail_parts[0].part_type == BodyMailPartType.Main
+        assert len(mail_parts) == 2
+        assert mail_parts[1].text == 'c'
+        assert mail_parts[1].part_type == BodyMailPartType.Signature
+
+    def test_unit__drop_part_type_consistence(self):
+        mail_parts = BodyMailParts()
+        a = BodyMailPart('a', BodyMailPartType.Main)
+        mail_parts._list.append(a)
+        b = BodyMailPart('b', BodyMailPartType.Quote)
+        mail_parts._list.append(b)
+        c = BodyMailPart('c', BodyMailPartType.Main)
+        mail_parts._list.append(c)
+        mail_parts.drop_part_type(BodyMailPartType.Quote)
+        assert len(mail_parts) == 1
+        assert mail_parts[0].text == 'ac'
+        assert mail_parts[0].part_type == BodyMailPartType.Main
+
+    def test_unit__get_nb_part_type(self):
+        mail_parts = BodyMailParts()
+        assert mail_parts.get_nb_part_type(BodyMailPartType.Main) == 0
+        assert mail_parts.get_nb_part_type(BodyMailPartType.Quote) == 0
+        assert mail_parts.get_nb_part_type(BodyMailPartType.Signature) == 0
+        a = BodyMailPart('a', BodyMailPartType.Main)
+        mail_parts._list.append(a)
+        assert mail_parts.get_nb_part_type(BodyMailPartType.Main) == 1
+        b = BodyMailPart('b', BodyMailPartType.Quote)
+        mail_parts._list.append(b)
+        assert mail_parts.get_nb_part_type(BodyMailPartType.Quote) == 1
+        c = BodyMailPart('c', BodyMailPartType.Signature)
+        mail_parts._list.append(c)
+        assert mail_parts.get_nb_part_type(BodyMailPartType.Main) == 1
+        assert mail_parts.get_nb_part_type(BodyMailPartType.Quote) == 1
+        assert mail_parts.get_nb_part_type(BodyMailPartType.Signature) == 1
+
+    def test_unit__str(self):
+        mail_parts = BodyMailParts()
+        a = BodyMailPart('a', BodyMailPartType.Main)
+        mail_parts._list.append(a)
+        b = BodyMailPart('b', BodyMailPartType.Quote)
+        mail_parts._list.append(b)
+        c = BodyMailPart('c', BodyMailPartType.Signature)
+        mail_parts._list.append(c)
+        assert str(mail_parts) == 'abc'
+
+
+class TestParsedMail(TestStandard):
+
+    def test_other__check_gmail_mail_text_only(self):
+        text_only = '''<div dir="ltr">Voici le texte<br></div>'''
+        mail = ParsedHTMLMail(text_only)
+        elements = mail.get_elements()
+        assert len(elements) == 1
+        assert elements[0].part_type == BodyMailPartType.Main
+
+    def test_other__check_gmail_mail_text_signature(self):
+        text_and_signature = '''
+        <div dir="ltr">POF<br clear="all"><div><br>-- <br>
+        <div class="gmail_signature" data-smartmail="gmail_signature">
+        <div dir="ltr">Voici Ma signature. En HTML <br><ol>
+        <li>Plop</li>
+        <li>Plip</li>
+        <li>Plop<br>
+        </li></ol></div></div></div></div>
+        '''
+        mail = ParsedHTMLMail(text_and_signature)
+        elements = mail.get_elements()
+        assert len(elements) == 2
+        assert elements[0].part_type == BodyMailPartType.Main
+        assert elements[1].part_type == BodyMailPartType.Signature
+
+    def test_other__check_gmail_mail_text_quote(self):
+        text_and_quote = '''
+        <div dir="ltr">Réponse<br>
+        <div class="gmail_extra"><br>
+        <div class="gmail_quote">Le 28 novembre 2017 à 10:29, John Doe <span
+        dir="ltr">&lt;<a href="mailto:bidule@localhost.fr"
+        target="_blank">bidule@localhost.fr</a>&gt;</span>
+        a écrit :<br>
+        <blockquote class="gmail_quote" style="margin:0 0 0
+        .8ex;border-left:1px #ccc solid;padding-left:1ex">Voici ma réponse<br>
+        <br><br>
+        Le 28/11/2017 à 10:05, Foo Bar a écrit&nbsp;:<br>
+        <blockquote class="gmail_quote" style="margin:0 0 0
+        .8ex;border-left:1px #ccc solid;padding-left:1ex">
+        Voici le texte<span class="HOEnZb"><font color="#888888"><br>
+        </font></span></blockquote>
+        <span class="HOEnZb"><font color="#888888">
+        <br>
+        -- <br>
+        TEST DE signature<br>
+        </font></span></blockquote>
+        </div><br></div></div>
+        '''
+        mail = ParsedHTMLMail(text_and_quote)
+        elements = mail.get_elements()
+        assert len(elements) == 2
+        assert elements[0].part_type == BodyMailPartType.Main
+        assert elements[1].part_type == BodyMailPartType.Quote
+
+    def test_other__check_gmail_mail_text_quote_text(self):
+        text_quote_text = '''
+              <div dir="ltr">Avant<br>
+              <div class="gmail_extra"><br>
+              <div class="gmail_quote">Le 28 novembre 2017 à 10:29, John Doe 
+              <span dir="ltr">&lt;<a href="mailto:bidule@localhost.fr"
+              target="_blank">bidule@localhost.fr</a>&gt;</span>
+              a écrit :<br>
+              <blockquote class="gmail_quote" style="margin:0 0 0
+              .8ex;border-left:1px #ccc solid;padding-left:1ex">Voici ma
+              réponse<br>
+              <br>
+              <br>
+              Le 28/11/2017 à 10:05, Foo Bar a écrit&nbsp;:<br>
+              <blockquote class="gmail_quote" style="margin:0 0 0
+              .8ex;border-left:1px #ccc solid;padding-left:1ex">
+              Voici le texte<span class="HOEnZb"><font color="#888888"><br>
+              </font></span></blockquote>
+              <span class="HOEnZb"><font color="#888888">
+              <br>
+              -- <br>
+              TEST DE signature<br>
+              </font></span></blockquote>
+              </div>
+              <br>
+              </div>
+              <div class="gmail_extra">Aprés<br>
+              </div>
+              </div>
+              '''
+
+        mail = ParsedHTMLMail(text_quote_text)
+        elements = mail.get_elements()
+        assert len(elements) == 3
+        assert elements[0].part_type == BodyMailPartType.Main
+        assert elements[1].part_type == BodyMailPartType.Quote
+        assert elements[2].part_type == BodyMailPartType.Main
+
+    def test_other__check_gmail_mail_text_quote_signature(self):
+        text_quote_signature = '''
+        <div dir="ltr">Hey !<br>
+                 </div>
+                 <div class="gmail_extra"><br>
+                 <div class="gmail_quote">Le 28 novembre 2017 à 10:29,
+                  John Doe <span
+                 dir="ltr">&lt;<a href="mailto:bidule@localhost.fr"
+                 target="_blank">bidule@localhost.fr</a>&gt;</span>
+                 a écrit :<br>
+                 <blockquote class="gmail_quote" style="margin:0 0 0
+                 .8ex;border-left:1px #ccc solid;padding-left:1ex">Voici ma
+                 réponse<br>
+                 <br>
+                 <br>
+                  Le 28/11/2017 à 10:05, Foo Bar a écrit&nbsp;:<br>
+                  <blockquote class="gmail_quote" style="margin:0 0 0
+                  .8ex;border-left:1px #ccc solid;padding-left:1ex">
+                  Voici le texte<span class="HOEnZb"><font color="#888888"><br>
+                  </font></span></blockquote>
+                  <span class="HOEnZb"><font color="#888888">
+                  <br>
+                  -- <br>
+                  TEST DE signature<br>
+                  </font></span></blockquote>
+                  </div>
+                  <br>
+                  <br clear="all">
+                  <br>
+                  -- <br>
+                  <div class="gmail_signature" data-smartmail="gmail_signature">
+                  <div dir="ltr">Voici Ma signature. En HTML <br>
+                  <ol>
+                  <li>Plop</li>
+                  <li>Plip</li>
+                  <li>Plop<br>
+                  </li>
+                  </ol>
+                  </div>
+                  </div>
+                  </div>
+                 '''
+
+        # INFO - G.M - 2017-11-28 -
+        # Now Quote + Signature block in Gmail is considered as one Quote
+        # Block.
+        mail = ParsedHTMLMail(text_quote_signature)
+        elements = mail.get_elements()
+        assert len(elements) == 2
+        assert elements[0].part_type == BodyMailPartType.Main
+        assert elements[1].part_type == BodyMailPartType.Quote
+
+    def test_other__check_gmail_mail_text_quote_text_signature(self):
+        text_quote_text_sign = '''
+        <div dir="ltr">Test<br>
+        <div class="gmail_extra"><br>
+        <div class="gmail_quote">Le 28 novembre 2017 à 10:29, John Doe <span
+        dir="ltr">&lt;<a href="mailto:bidule@localhost.fr"
+        target="_blank">bidule@localhost.fr</a>&gt;</span>
+        a écrit :<br>
+        <blockquote class="gmail_quote" style="margin:0 0 0
+        .8ex;border-left:1px #ccc solid;padding-left:1ex">Voici ma
+        réponse<br>
+        <br>
+        <br>
+        Le 28/11/2017 à 10:05, Foo Bar a écrit&nbsp;:<br>
+        <blockquote class="gmail_quote" style="margin:0 0 0
+        .8ex;border-left:1px #ccc solid;padding-left:1ex">
+        Voici le texte<span class="HOEnZb"><font color="#888888"><br>
+        </font></span></blockquote>
+        <span class="HOEnZb"><font color="#888888">
+        <br>
+        -- <br>
+        TEST DE signature<br>
+        </font></span></blockquote>
+        </div>
+        <br>
+        <br>
+        </div>
+        <div class="gmail_extra">RE test<br clear="all">
+        </div>
+        <div class="gmail_extra"><br>
+        -- <br>
+        <div class="gmail_signature" data-smartmail="gmail_signature">
+        <div dir="ltr">Voici Ma signature. En HTML <br>
+        <ol>
+        <li>Plop</li>
+        <li>Plip</li>
+        <li>Plop<br>
+        </li>
+        </ol>
+        </div>
+        </div>
+        </div>
+        </div>
+        '''
+
+        mail = ParsedHTMLMail(text_quote_text_sign)
+        elements = mail.get_elements()
+        assert len(elements) == 4
+        assert elements[0].part_type == BodyMailPartType.Main
+        assert elements[1].part_type == BodyMailPartType.Quote
+        assert elements[2].part_type == BodyMailPartType.Main
+        assert elements[3].part_type == BodyMailPartType.Signature
+
+    def test_other__check_thunderbird_mail_text_only(self):
+
+        text_only = '''Coucou<br><br><br>'''
+        mail = ParsedHTMLMail(text_only)
+        elements = mail.get_elements()
+        assert len(elements) == 1
+        assert elements[0].part_type == BodyMailPartType.Main
+
+    def test_other__check_thunderbird_mail_text_signature(self):
+        text_and_signature = '''
+        <p>Test<br>
+        </p>
+        <div class="moz-signature">-- <br>
+          TEST DE signature</div>
+        '''
+        mail = ParsedHTMLMail(text_and_signature)
+        elements = mail.get_elements()
+        assert len(elements) == 2
+        assert elements[0].part_type == BodyMailPartType.Main
+        assert elements[1].part_type == BodyMailPartType.Signature
+
+    def test_other__check_thunderbird_mail_text_quote(self):
+        text_and_quote = '''
+            <p>Pof<br>
+            </p>
+            <br>
+            <div class="moz-cite-prefix">Le 28/11/2017 à 11:21, John Doe a
+              écrit&nbsp;:<br>
+            </div>
+            <blockquote type="cite"
+              cite="mid:658592c1-14de-2958-5187-3571edea0aac@localhost.fr">
+              <meta http-equiv="Context-Type" 
+              content="text/html; charset=utf-8">
+              <p>Test<br>
+              </p>
+              <div class="moz-signature">-- <br>
+                TEST DE signature</div>
+            </blockquote>
+            <br>
+        '''
+        mail = ParsedHTMLMail(text_and_quote)
+        elements = mail.get_elements()
+        assert len(elements) == 2
+        assert elements[0].part_type == BodyMailPartType.Main
+        assert elements[1].part_type == BodyMailPartType.Quote
+
+    def test_other__check_thunderbird_mail_text_quote_text(self):
+        text_quote_text = '''
+        <p>Pof<br>
+        </p>
+        <br>
+        <div class="moz-cite-prefix">Le 28/11/2017 à 11:54, 
+         Bidule a
+          écrit&nbsp;:<br>
+        </div>
+        <blockquote type="cite"
+          cite="mid:b541b451-bb31-77a4-45b9-ad89969d7962@localhost.fr">
+          <meta http-equiv="Context-Type" 
+          content="text/html; charset=utf-8">
+          <p>Pof<br>
+          </p>
+          <br>
+          <div class="moz-cite-prefix">Le 28/11/2017 à 11:21, John Doe a
+            écrit&nbsp;:<br>
+          </div>
+          <blockquote type="cite"
+            cite="mid:658592c1-14de-2958-5187-3571edea0aac@localhost.fr">
+            <p>Test<br>
+            </p>
+            <div class="moz-signature">-- <br>
+              TEST DE signature</div>
+          </blockquote>
+          <br>
+        </blockquote>
+        Pif<br>
+        '''
+
+        mail = ParsedHTMLMail(text_quote_text)
+        elements = mail.get_elements()
+        assert len(elements) == 3
+        assert elements[0].part_type == BodyMailPartType.Main
+        assert elements[1].part_type == BodyMailPartType.Quote
+        assert elements[2].part_type == BodyMailPartType.Main
+
+    def test_other__check_thunderbird_mail_text_quote_signature(self):
+        text_quote_signature = '''
+        <p>Coucou<br>
+        </p>
+        <br>
+        <div class="moz-cite-prefix">Le 28/11/2017 à 11:22, Bidule a
+        écrit&nbsp;:<br>
+        </div>
+        <blockquote type="cite"
+        cite="mid:4e6923e2-796d-eccf-84b7-6824da4151ee@localhost.fr">Réponse<br>
+        <br>
+        Le 28/11/2017 à 11:21, John Doe a écrit&nbsp;: <br>
+        <blockquote type="cite"> <br>
+        Test <br>
+        <br>
+        --&nbsp;<br>
+        TEST DE signature <br>
+        </blockquote>
+        <br>
+        </blockquote>
+        <br>
+        <div class="moz-signature">-- <br>
+        TEST DE signature</div>
+        '''
+
+        mail = ParsedHTMLMail(text_quote_signature)
+        elements = mail.get_elements()
+        assert len(elements) == 3
+        assert elements[0].part_type == BodyMailPartType.Main
+        assert elements[1].part_type == BodyMailPartType.Quote
+        assert elements[2].part_type == BodyMailPartType.Signature
+
+    def test_other__check_thunderbird_mail_text_quote_text_signature(self):
+        text_quote_text_sign = '''
+        <p>Avant<br>
+        </p>
+        <br>
+        <div class="moz-cite-prefix">Le 28/11/2017 à 11:19, Bidule a
+          écrit&nbsp;:<br>
+        </div>
+        <blockquote type="cite"
+          cite="mid:635df73c-d3c9-f2e9-2304-24ff536bfa16@localhost.fr">Coucou 
+          <br><br>
+        </blockquote>
+        Aprés<br>
+        <br>
+        <div class="moz-signature">-- <br>
+          TEST DE signature</div>
+        '''
+
+        mail = ParsedHTMLMail(text_quote_text_sign)
+        elements = mail.get_elements()
+        assert len(elements) == 4
+        assert elements[0].part_type == BodyMailPartType.Main
+        assert elements[1].part_type == BodyMailPartType.Quote
+        assert elements[2].part_type == BodyMailPartType.Main
+        assert elements[3].part_type == BodyMailPartType.Signature
+
+    # INFO - G.M - 2017-11-28 - Test for outlook.com webapp html mail
+    # outlook.com ui doesn't seems to allow complex reply, new message
+    # and signature are always before quoted one.
+
+    def test_other__check_outlook_com_mail_text_only(self):
+
+        text_only = '''
+        <div id="divtagdefaultwrapper"
+        style="font-size:12pt;color:#000000;
+        font-family:Calibri,Helvetica,sans-serif;"
+        dir="ltr">
+        <p style="margin-top:0;margin-bottom:0">message<br>
+        </p>
+        </div>
+        '''
+        mail = ParsedHTMLMail(text_only)
+        elements = mail.get_elements()
+        assert len(elements) == 1
+        assert elements[0].part_type == BodyMailPartType.Main
+
+    def test_other__check_outlook_com_mail_text_signature(self):
+        text_and_signature = '''
+        <div id="divtagdefaultwrapper"
+        style="font-size:12pt;color:#000000;
+        font-family:Calibri,Helvetica,sans-serif;"
+          dir="ltr">
+          <p style="margin-top:0;margin-bottom:0">Test<br>
+          </p>
+          <p style="margin-top:0;margin-bottom:0"><br>
+          </p>
+          <div id="Signature">
+            <div id="divtagdefaultwrapper" style="font-size: 12pt; color:
+              rgb(0, 0, 0); background-color: rgb(255, 255, 255);
+              font-family:
+              Calibri,Arial,Helvetica,sans-serif,&quot;EmojiFont&quot;,&quot;Apple
+              Color Emoji&quot;,&quot;Segoe UI
+              Emoji&quot;,NotoColorEmoji,&quot;Segoe UI
+              Symbol&quot;,&quot;Android Emoji&quot;,EmojiSymbols;">
+              Envoyé à partir de <a href="http://aka.ms/weboutlook"
+                id="LPNoLP">Outlook</a></div>
+          </div>
+        </div>
+        '''
+        mail = ParsedHTMLMail(text_and_signature)
+        elements = mail.get_elements()
+        assert len(elements) == 2
+        assert elements[0].part_type == BodyMailPartType.Main
+        assert elements[1].part_type == BodyMailPartType.Signature
+
+    def test_other__check_outlook_com_mail_text_quote(self):
+        text_and_quote = '''
+        <div id="divtagdefaultwrapper"
+        style="font-size:12pt;color:#000000;font-family:Calibri,Helvetica,sans-serif;"
+        dir="ltr">
+        <p style="margin-top:0;margin-bottom:0">Salut !<br>
+        </p>
+        </div>
+        <hr style="display:inline-block;width:98%" tabindex="-1">
+        <div id="divRplyFwdMsg" dir="ltr"><font style="font-size:11pt"
+        color="#000000" face="Calibri, sans-serif"><b>De :</b> John Doe<br>
+        <b>Envoyé :</b> mardi 28 novembre 2017 12:44:59<br>
+        <b>À :</b> dev.bidule@localhost.fr<br>
+        <b>Objet :</b> voila</font>
+        <div>&nbsp;</div>
+        </div>
+        <style type="text/css" style="display:none">
+        <!--
+        p
+        &#x09;{margin-top:0;
+        &#x09;margin-bottom:0}
+        -->
+        </style>
+        <div dir="ltr">
+        <div id="x_divtagdefaultwrapper" dir="ltr" style="font-size:12pt;
+        color:#000000; font-family:Calibri,Helvetica,sans-serif">
+        Contenu
+        <p style="margin-top:0; margin-bottom:0"><br>
+        </p>
+        <div id="x_Signature">
+          <div id="x_divtagdefaultwrapper" dir="ltr"
+            style="font-size:12pt; color:rgb(0,0,0);
+            background-color:rgb(255,255,255);
+        font-family:Calibri,Arial,Helvetica,sans-serif,&quot;EmojiFont&quot;,&quot;Apple
+            Color Emoji&quot;,&quot;Segoe UI
+            Emoji&quot;,NotoColorEmoji,&quot;Segoe UI
+            Symbol&quot;,&quot;Android Emoji&quot;,EmojiSymbols">
+            DLMQDNLQNDMLQS<br>
+            qs<br>
+            dqsd<br>
+            d<br>
+            qsd<br>
+          </div>
+        </div>
+        </div>
+        </div>
+        '''
+        mail = ParsedHTMLMail(text_and_quote)
+        elements = mail.get_elements()
+        assert len(elements) == 2
+        assert elements[0].part_type == BodyMailPartType.Main
+        assert elements[1].part_type == BodyMailPartType.Quote
+
+    def test_other__check_outlook_com_mail_text_signature_quote(self):
+        text_signature_quote = '''
+        <div id="divtagdefaultwrapper"
+        style="font-size:12pt;color:#000000;font-family:Calibri,Helvetica,sans-serif;"
+        dir="ltr">
+        <p style="margin-top:0;margin-bottom:0">Salut !<br>
+        </p>
+        <p style="margin-top:0;margin-bottom:0"><br>
+        </p>
+        <div id="Signature">
+        <div id="divtagdefaultwrapper" dir="ltr" style="font-size: 12pt;
+        color: rgb(0, 0, 0); background-color: rgb(255, 255, 255);
+        font-family:
+        Calibri,Arial,Helvetica,sans-serif,&quot;EmojiFont&quot;,&quot;Apple
+        Color Emoji&quot;,&quot;Segoe UI
+        Emoji&quot;,NotoColorEmoji,&quot;Segoe UI
+        Symbol&quot;,&quot;Android Emoji&quot;,EmojiSymbols;">
+        Envoyée depuis Outlook<br>
+        </div>
+        </div>
+        </div>
+        <hr style="display:inline-block;width:98%" tabindex="-1">
+        <div id="divRplyFwdMsg" dir="ltr"><font style="font-size:11pt"
+        color="#000000" face="Calibri, sans-serif"><b>De :</b> John Doe
+        &lt;dev.bidule@localhost.fr&gt;<br>
+        <b>Envoyé :</b> mardi 28 novembre 2017 12:51:42<br>
+        <b>À :</b> John Doe<br>
+        <b>Objet :</b> Re: Test</font>
+        <div>&nbsp;</div>
+        </div>
+        <div style="background-color:#FFFFFF">
+        <p>Coucou<br>
+        </p>
+        <br>
+        <div class="x_moz-cite-prefix">Le 28/11/2017 à 12:39, John Doe a
+        écrit&nbsp;:<br>
+        </div>
+        <blockquote type="cite">
+        <div id="x_divtagdefaultwrapper" dir="ltr">
+        <p>Test<br>
+        </p>
+        <p><br>
+        </p>
+        <div id="x_Signature">
+        <div id="x_divtagdefaultwrapper">Envoyé à partir de <a
+        href="http://aka.ms/weboutlook" id="LPNoLP">
+        Outlook</a></div>
+        </div>
+        </div>
+        </blockquote>
+        <br>
+        <div class="x_moz-signature">-- <br>
+        TEST DE signature</div>
+        </div>
+        '''
+
+        mail = ParsedHTMLMail(text_signature_quote)
+        elements = mail.get_elements()
+        assert len(elements) == 3
+        assert elements[0].part_type == BodyMailPartType.Main
+        assert elements[1].part_type == BodyMailPartType.Signature
+        assert elements[2].part_type == BodyMailPartType.Quote