add search engine based on a ilike sql query and a filterable result page

tracim/tracim/controllers/__init__.py

@@ -126,40 +126,13 @@ class TIMRestController(RestController, BaseController):
 
 class TIMRestControllerWithBreadcrumb(TIMRestController):
 
-    def get_breadcrumb(self, folder_id=None) -> [BreadcrumbItem]:
+    def get_breadcrumb(self, item_id=None) -> [BreadcrumbItem]:
         """
         TODO - Remove this and factorize it with other get_breadcrumb_xxx methods
-        :param folder_id:
+        :param item_id: an item id (item may be normal content or folder
         :return:
         """
-        workspace = tmpl_context.workspace
-        workspace_id = tmpl_context.workspace_id
-        breadcrumb = []
-
-        breadcrumb.append(BreadcrumbItem(ContentType.icon(ContentType.FAKE_Dashboard), _('Workspaces'), tg.url('/workspaces')))
-        breadcrumb.append(BreadcrumbItem(ContentType.icon(ContentType.FAKE_Workspace), workspace.label, tg.url('/workspaces/{}'.format(workspace.workspace_id))))
-
-        content_api = ContentApi(tmpl_context.current_user)
-        if folder_id:
-            breadcrumb_folder_items = []
-            current_item = content_api.get_one(folder_id, ContentType.Any, workspace)
-            is_active = True
-
-            while current_item:
-                breadcrumb_item = BreadcrumbItem(ContentType.icon(current_item.type),
-                                                 current_item.label,
-                                                 tg.url('/workspaces/{}/folders/{}'.format(workspace_id, current_item.content_id)),
-                                                 is_active)
-                is_active = False # the first item is True, then all other are False => in the breadcrumb, only the last item is "active"
-                breadcrumb_folder_items.append(breadcrumb_item)
-                current_item = current_item.parent
-
-            for item in reversed(breadcrumb_folder_items):
-                breadcrumb.append(item)
-
-
-        return breadcrumb
-
+        return ContentApi(tmpl_context.current_user).build_breadcrumb(tmpl_context.workspace, item_id)
 
 
 class TIMWorkspaceContentRestController(TIMRestControllerWithBreadcrumb):

tracim/tracim/controllers/content.py

@@ -762,7 +762,6 @@ class UserWorkspaceFolderRestController(TIMRestControllerWithBreadcrumb):
     @tg.require(current_user_is_content_manager())
     @tg.expose()
     def put_archive_undo(self, item_id):
-        print('AGAGA')
         # TODO - CHECK RIGHTS
         item_id = int(item_id)
         content_api = ContentApi(tmpl_context.current_user, True, True) # Here we do not filter deleted items

tracim/tracim/controllers/root.py

@@ -123,3 +123,28 @@ class RootController(StandardController):
         fake_api = Context(CTX.CURRENT_USER).toDict({'current_user': current_user_content})
 
         return DictLikeClass(fake_api=fake_api)
+
+    @require(predicates.not_anonymous())
+    @expose('tracim.templates.search')
+    def search(self, keywords = ''):
+        from tracim.lib.content import ContentApi
+
+        user = tmpl_context.current_user
+        api = ContentApi(user)
+
+        items = []
+        keyword_list = api.get_keywords(keywords)
+
+        result = api.search(keyword_list)
+        if result:
+            items = result.limit(ContentApi.SEARCH_DEFAULT_RESULT_NB).all()
+
+        current_user_content = Context(CTX.CURRENT_USER).toDict(user)
+        fake_api = Context(CTX.CURRENT_USER).toDict({'current_user': current_user_content})
+
+        search_results = Context(CTX.SEARCH).toDict(items, 'results', 'result_nb')
+        search_results.keywords = keyword_list
+
+        return DictLikeClass(fake_api=fake_api, search=search_results)
+
+

tracim/tracim/controllers/workspace.py

@@ -80,7 +80,6 @@ class UserWorkspaceRestController(TIMRestController):
             dictified_workspaces = Context(CTX.MENU_API).toDict(workspaces, 'd')
             return dictified_workspaces
 
-
         allowed_content_types = ContentType.allowed_types_from_str(folder_allowed_content_types)
         ignored_item_ids = [int(ignore_id)] if ignore_id else []
 

tracim/tracim/lib/base.py

@@ -113,7 +113,7 @@ class Logger(object):
         self._logger = logging.getLogger(self._name)
 
     def _txt(self, instance_or_class):
-        if instance_or_class.__class__.__name__=='type':
+        if instance_or_class.__class__.__name__ in ('function', 'type'):
             return instance_or_class.__name__
         else:
             return instance_or_class.__class__.__name__

tracim/tracim/lib/content.py

@@ -2,15 +2,25 @@
 
 __author__ = 'damien'
 
+import re
+
 import tg
+from tg.i18n import ugettext as _
 
+import sqlalchemy
+from sqlalchemy.orm import aliased
+from sqlalchemy.orm import joinedload
 from sqlalchemy.orm.attributes import get_history
 from sqlalchemy import not_
+from sqlalchemy import or_
 from tracim.lib import cmp_to_key
 from tracim.lib.notifications import NotifierFactory
 from tracim.model import DBSession
 from tracim.model.auth import User
-from tracim.model.data import ContentStatus, ContentRevisionRO, ActionDescription
+from tracim.model.data import ActionDescription
+from tracim.model.data import BreadcrumbItem
+from tracim.model.data import ContentStatus
+from tracim.model.data import ContentRevisionRO
 from tracim.model.data import Content
 from tracim.model.data import ContentType
 from tracim.model.data import NodeTreeItem
@@ -48,6 +58,10 @@ def compare_tree_items_for_sorting_by_type_and_name(item1: NodeTreeItem, item2:
 
 class ContentApi(object):
 
+    SEARCH_SEPARATORS = ',| '
+    SEARCH_DEFAULT_RESULT_NB = 50
+
+
     def __init__(self, current_user: User, show_archived=False, show_deleted=False, all_content_in_treeview=True):
         self._user = current_user
         self._show_archived = show_archived
@@ -71,6 +85,45 @@ class ContentApi(object):
         content_list.sort(key=cmp_to_key(compare_content_for_sorting_by_type_and_name))
         return content_list
 
+    def build_breadcrumb(self, workspace, item_id=None, skip_root=False) -> [BreadcrumbItem]:
+        """
+        TODO - Remove this and factorize it with other get_breadcrumb_xxx methods
+        :param item_id: an item id (item may be normal content or folder
+        :return:
+        """
+        workspace_id = workspace.workspace_id
+        breadcrumb = []
+
+        if not skip_root:
+            breadcrumb.append(BreadcrumbItem(ContentType.icon(ContentType.FAKE_Dashboard), _('Workspaces'), tg.url('/workspaces')))
+        breadcrumb.append(BreadcrumbItem(ContentType.icon(ContentType.FAKE_Workspace), workspace.label, tg.url('/workspaces/{}'.format(workspace.workspace_id))))
+
+        if item_id:
+            breadcrumb_folder_items = []
+            current_item = self.get_one(item_id, ContentType.Any, workspace)
+            is_active = True
+            if current_item.type==ContentType.Folder:
+                next_url = tg.url('/workspaces/{}/folders/{}'.format(workspace_id, current_item.content_id))
+            else:
+                next_url = tg.url('/workspaces/{}/folders/{}/{}s/{}'.format(workspace_id, current_item.parent_id, current_item.type, current_item.content_id))
+
+            while current_item:
+                breadcrumb_item = BreadcrumbItem(ContentType.icon(current_item.type),
+                                                 current_item.label,
+                                                 next_url,
+                                                 is_active)
+                is_active = False # the first item is True, then all other are False => in the breadcrumb, only the last item is "active"
+                breadcrumb_folder_items.append(breadcrumb_item)
+                current_item = current_item.parent
+                if current_item:
+                    # In last iteration, the parent is None, and there is no more breadcrumb item to build
+                    next_url = tg.url('/workspaces/{}/folders/{}'.format(workspace_id, current_item.content_id))
+
+            for item in reversed(breadcrumb_folder_items):
+                breadcrumb.append(item)
+
+
+        return breadcrumb
 
     def _base_query(self, workspace: Workspace=None):
         result = DBSession.query(Content)
@@ -79,10 +132,16 @@ class ContentApi(object):
             result = result.filter(Content.workspace_id==workspace.workspace_id)
 
         if not self._show_deleted:
-            result = result.filter(Content.is_deleted==False)
+            parent = aliased(Content)
+            result.join(parent, Content.parent).\
+                filter(Content.is_deleted==False).\
+                filter(parent.is_deleted==False)
 
         if not self._show_archived:
-            result = result.filter(Content.is_archived==False)
+            parent = aliased(Content)
+            result.join(parent, Content.parent).\
+                filter(Content.is_archived==False).\
+                filter(parent.is_archived==False)
 
         return result
 
@@ -110,7 +169,7 @@ class ContentApi(object):
             all()
 
         if not filter_by_allowed_content_types or len(filter_by_allowed_content_types)<=0:
-            return folders
+            filter_by_allowed_content_types = ContentType.allowed_types_for_folding()
 
         # Now, the case is to filter folders by the content that they are allowed to contain
         result = []
@@ -118,6 +177,7 @@ class ContentApi(object):
             for allowed_content_type in filter_by_allowed_content_types:
                 if folder.type==ContentType.Folder and folder.properties['allowed_content'][allowed_content_type]==True:
                     result.append(folder)
+                    break
 
         return result
 
@@ -295,3 +355,35 @@ class ContentApi(object):
         if do_notify:
             NotifierFactory.create(self._user).notify_content_update(content)
 
+
+    def get_keywords(self, search_string, search_string_separators=None) -> [str]:
+        """
+        :param search_string: a list of coma-separated keywords
+        :return: a list of str (each keyword = 1 entry
+        """
+
+        search_string_separators = search_string_separators or ContentApi.SEARCH_SEPARATORS
+
+        keywords = []
+        if search_string:
+            keywords = [keyword.strip() for keyword in re.split(search_string_separators, search_string)]
+
+        return keywords
+
+    def search(self, keywords: [str]) -> sqlalchemy.orm.query.Query:
+        """
+        :return: a sorted list of Content items
+        """
+
+        if len(keywords)<=0:
+            return None
+
+        filter_group_label = list(Content.label.ilike('%{}%'.format(keyword)) for keyword in keywords)
+        filter_group_desc = list(Content.description.ilike('%{}%'.format(keyword)) for keyword in keywords)
+        title_keyworded_items = self._base_query().\
+            filter(or_(*(filter_group_label+filter_group_desc))).\
+            options(joinedload('children')).\
+            options(joinedload('parent'))
+
+        return title_keyworded_items
+

tracim/tracim/lib/helpers.py

@@ -180,5 +180,16 @@ def is_item_still_editable(item):
             return True
     return False
 
+def shorten(text: str, maxlength: int, add_three_points=True) -> str:
+
+    result = text
+    if len(text)>maxlength:
+        result = text[:maxlength]
+
+        if add_three_points:
+            result += '…'
+
+    return result
+
 from tracim.config.app_cfg import CFG as CFG_ORI
 CFG = CFG_ORI.get_instance() # local CFG var is an instance of CFG class found in app_cfg

tracim/tracim/lib/utils.py

@@ -0,0 +1,17 @@
+# -*- coding: utf-8 -*-
+
+import time
+
+from tracim.lib.base import logger
+
+def exec_time_monitor():
+    def decorator_func(func):
+        def wrapper_func(*args, **kwargs):
+            start = time.time()
+            retval = func(*args, **kwargs)
+            end = time.time()
+            logger.debug(func, 'exec time: {} seconds'.format(end-start))
+            return retval
+        return wrapper_func
+    return decorator_func
+

tracim/tracim/model/data.py

@@ -311,11 +311,16 @@ class ContentType(object):
         return [cls.Folder, cls.File, cls.Comment, cls.Thread, cls.Page]
 
     @classmethod
+    def allowed_types_for_folding(cls):
+        # This method is used for showing only "main" types in the left-side treeview
+        return [cls.Folder, cls.File, cls.Thread, cls.Page]
+
+    @classmethod
     def allowed_types_from_str(cls, allowed_types_as_string: str):
         allowed_types = []
         # HACK - THIS
         for item in allowed_types_as_string.split(ContentType._STRING_LIST_SEPARATOR):
-            if item and item in ContentType.allowed_types():
+            if item and item in ContentType.allowed_types_for_folding():
                 allowed_types.append(item)
         return allowed_types
 

tracim/tracim/model/serializers.py

@@ -1,10 +1,12 @@
 # -*- coding: utf-8 -*-
 import types
 
+from bs4 import BeautifulSoup
 import tg
 from tg.util import LazyString
 from tracim.lib.base import logger
-from tracim.model.auth import Group, Profile
+from tracim.lib.utils import exec_time_monitor
+from tracim.model.auth import Profile
 from tracim.model.auth import User
 from tracim.model.data import BreadcrumbItem, ActionDescription
 from tracim.model.data import ContentStatus
@@ -47,7 +49,6 @@ class pod_serializer(object):
         Context.register_converter(self.context, self.model_class, func)
         return func
 
-
 class ContextConverterNotFoundException(Exception):
     def __init__(self, context_string, model_class):
         message = 'converter not found (context: {0} - model: {1})'.format(context_string, model_class.__name__)
@@ -68,6 +69,7 @@ class CTX(object):
     MENU_API_BUILD_FROM_TREE_ITEM = 'MENU_API_BUILD_FROM_TREE_ITEM'
     PAGE = 'PAGE'
     PAGES = 'PAGES'
+    SEARCH = 'SEARCH'
     THREAD = 'THREAD'
     THREADS = 'THREADS'
     USER = 'USER'
@@ -412,17 +414,7 @@ def serialize_node_for_page(item: Content, context: Context):
 
     if item.type==ContentType.Folder:
         return Context(CTX.DEFAULT).toDict(item)
-    ### CODE BELOW IS REPLACED BY THE TWO LINES UP ^^
-    # 2014-10-08 - IF YOU FIND THIS COMMENT, YOU CAn REMOVE THE CODE
-    #
-    #if item.type==ContentType.Folder:
-    #    value = DictLikeClass(
-    #        id = item.content_id,
-    #        label = item.label,
-    #    )
-    #    return value
 
-    raise NotImplementedError
 
 
 @pod_serializer(Content, CTX.THREADS)
@@ -556,6 +548,65 @@ def serialize_content_for_workspace_and_folder(content: Content, context: Contex
     return result
 
 
+from tg import cache
+
+@pod_serializer(Content, CTX.SEARCH)
+def serialize_content_for_search_result(content: Content, context: Context):
+
+    def serialize_it():
+        nonlocal content
+
+        if content.type == ContentType.Comment:
+            logger.info('serialize_content_for_search_result', 'Serializing parent class {} instead of {} [content #{}]'.format(content.parent.type, content.type, content.content_id))
+            content = content.parent
+
+        data_container = content
+
+        if content.revision_to_serialize>0:
+            for revision in content.revisions:
+                if revision.revision_id==content.revision_to_serialize:
+                    data_container = revision
+                    break
+
+        # FIXME - D.A. - 2015-02-23 - This import should not be there...
+        from tracim.lib.content import ContentApi
+        breadcrumbs = ContentApi(None).build_breadcrumb(data_container.workspace, data_container.content_id, skip_root=True)
+
+        last_comment_datetime = data_container.updated
+        comments = data_container.get_comments()
+        if comments:
+            last_comment_datetime = max(last_comment_datetime, max(comment.updated for comment in comments))
+
+        result = DictLikeClass(
+            id = content.content_id,
+            parent = context.toDict(content.parent),
+            workspace = context.toDict(content.workspace),
+            type = content.type,
+
+            content = data_container.description,
+            content_raw = BeautifulSoup(data_container.description).text,
+
+            created = data_container.created,
+            label = data_container.label,
+            icon = ContentType.icon(content.type),
+            owner = context.toDict(data_container.owner),
+            status = context.toDict(data_container.get_status()),
+            breadcrumb = context.toDict(breadcrumbs),
+            last_activity = last_comment_datetime
+        )
+
+        if content.type==ContentType.File:
+            result.label = content.label.__str__() if content.label else content.file_name.__str__()
+
+        if not result.label or ''==result.label:
+            result.label = 'No title'
+
+        return result
+
+    return serialize_it()
+
+
+
 ########################################################################################################################
 # ContentStatus
 

tracim/tracim/public/assets/css/dashboard.css

@@ -178,12 +178,26 @@ table.user_content, table.user_content td  {
 
 table.first_row_headers tr:first-child td {
     font-weight: bold;
-    background-color: #EEE;
+    background-color: #EEE;
     align: center;
 }
 
 table.first_column_headers tr td:first-child {
     font-weight: bold;
     background-color: #EEE;
-    align: center;
-}
+    align: center;
+}
+
+.search-result-item-breadcrumb {
+    margin-top: -1em;
+}
+
+.search-result-item-breadcrumb a {
+    color: #090;
+}
+
+#search-result-dynamic-resume {
+    color: #090;
+    margin-top: -1em;
+    margin-bottom: 0;
+}

tracim/tracim/templates/master_authenticated.mak

@@ -113,6 +113,13 @@
                                 </ul>
                             </li>
                         % endif
+                        
+                        <form class="navbar-form navbar-left" role="search" action="${tg.url('/search?')}">
+                            <div class="form-group">
+                                <input type="text" class="form-control" placeholder="${_('Search for...')}" name="keywords" value="${','.join(search.keywords) if search else ''}">
+                            </div>
+                            <button type="submit" class="btn btn-default">${_('Search')}</button>
+                        </form>
                     </ul>
                 % endif
 

tracim/tracim/templates/search.mak

@@ -0,0 +1,77 @@
+<%inherit file="local:templates.master_authenticated_left_treeview_right_toolbar"/>
+
+<%namespace name="TIM" file="tracim.templates.pod"/>
+<%namespace name="TOOLBAR" file="tracim.templates.search_toolbars"/>
+<%namespace name="FORMS" file="tracim.templates.user_workspace_forms"/>
+<%namespace name="WIDGETS" file="tracim.templates.user_workspace_widgets"/>
+
+<%def name="title()">${_('My workspaces')}</%def>
+
+<%def name="SIDEBAR_LEFT_CONTENT()">
+    <h4>${_('Workspaces')}</h4>
+    ${WIDGETS.TREEVIEW('sidebar-left-menu', '__')}
+    <hr/>
+</%def>
+
+<%def name="SIDEBAR_RIGHT_CONTENT()">
+    ${TOOLBAR.SECURED_SEARCH(fake_api.current_user)}
+</%def>
+
+<%def name="REQUIRED_DIALOGS()">
+</%def>
+
+############################################################################
+##
+## PAGE CONTENT BELOW
+##
+############################################################################
+
+<div class="row">
+    <h1 class="page-header">
+        ${TIM.ICO(32, 'actions/system-search')} ${_('Search results')}
+        <small>
+            ${_('<span class="badge">{}</span> results for keywords: '.format(search.result_nb))|n}
+            % for keyword in search.keywords:
+                <span class="label label-default">${keyword}</span>
+            % endfor
+        </small>
+    </h1>
+</div>
+<div class="row">
+    <p id="search-result-dynamic-resume">${_('loading...')}</p>
+</div>
+<div class="row">
+    <div id='application-document-panel'>
+        <ol class="search-results">
+            % for item in search.results:
+                <li class="search-result-type-${item.type} search-result-status-${item.status.id}">
+                    <h4>
+                        <a href="${item.breadcrumb[-1].url}">${TIM.ICO(16, item.icon)} ${item.label}</a>
+                         &nbsp;&nbsp;<span style="color: #AAA;">—</span>
+                         <button type="button" class="btn btn-default btn-disabled btn-link ">
+                             ${TIM.ICO(16, item['status']['icon'])}&nbsp;
+                             <span class="${item.status.css}">${item.status.label}</span>
+                         </button>
+                    </h4>
+                    <p style="margin-bottom: 2em;" class="search-result-item-breadcrumb">
+                        <i style="color: #AAA;" class="fa fa-fw fa-map-marker"></i>
+                        % for bread in item.breadcrumb:
+                            / <a href="${bread.url}">${bread.label}</a>
+                        % endfor
+                        <br/>
+                        <span
+                            style="color: #AAA;"
+##                            rel="tooltip"
+##                            data-toggle="tooltip"
+##                            data-placement="top"
+                            title="${_('Last known activty')}" ><i class="fa fa-fw fa-calendar"></i> ${h.date_time_in_long_format(item.last_activity, '%d %B %Y at %I:%M')}</span> &mdash;
+                            ${h.shorten(item.content_raw, 300)}
+                    </p>
+
+##                    <hr style="width: 33%; margin-left: 0;"/>
+                </li>
+            % endfor
+        </ol>
+    </div>
+</div>
+

tracim/tracim/templates/search_toolbars.mak

@@ -0,0 +1,83 @@
+<%namespace name="TIM" file="tracim.templates.pod"/>
+
+<%def name="SECURED_SEARCH(user)">
+    <%
+        content_types = [
+            ('page', 'mimetypes/text-html'),
+            ('file', 'status/mail-attachment'),
+            ('thread', 'apps/internet-group-chat'),
+            ('folder', 'places/jstree-folder')
+        ]
+
+        statuses = [
+            ('open', 'status/status-open'),
+            ('closed-validated', 'emblems/emblem-checked'),
+            ('closed-cancelled', 'emblems/emblem-unreadable'),
+            ('closed-deprecated', 'status/status-outdated')
+        ]
+    %>
+    <div class="btn-group btn-group-vertical" data-toggle="buttons">
+        % for content_type, icon in content_types:
+            <label class="btn btn-default active search-result-filter-button" id='show-hide-search-result-of-type-${content_type}'>
+                <input type="checkbox" autocomplete="off" checked> ${TIM.ICO(32, icon)}
+            </label>
+        % endfor
+    </div>
+    <p></p>
+    <div class="btn-group btn-group-vertical" data-toggle="buttons">
+        % for status, icon in statuses:
+            <label class="btn btn-default active search-result-filter-button" id='show-hide-search-result-with-status-${status}'>
+                <input type="checkbox" autocomplete="off" checked> ${TIM.ICO(32, icon)}
+            </label>
+        % endfor
+    </div>
+    <p></p>
+    
+    <script>
+        $(document).ready(function() {
+            % for content_type, icon in content_types: # python code
+                $('#show-hide-search-result-of-type-${content_type}').click(function() {
+                    if ($('#show-hide-search-result-of-type-${content_type}').hasClass('active')) {
+                        $('.search-result-type-${content_type}').hide();
+                    } else {
+                        $('.search-result-type-${content_type}').show();
+                    }
+                });
+            % endfor # python code
+
+            % for status, icon in statuses: # python code
+                $('#show-hide-search-result-with-status-${status}').click(function() {
+                    console.log('clieck')
+                    if ($('#show-hide-search-result-with-status-${status}').hasClass('active')) {
+                        $('.search-result-status-${status}').hide();
+                    } else {
+                        $('.search-result-status-${status}').show();
+                    }
+                });
+            % endfor # python code
+            
+            function refresh_search_result_count() {
+                var itemNb = $('ol.search-results > li').length;
+                var visibleItemNb = $('ol.search-results > li:visible').length;
+                var message = "${_('Showing {0} filtered items of {1}')}"
+                if(visibleItemNb<=1) {
+                    message = "${_('Showing {0} filtered item of a total of {1}')}"
+                } else if (visibleItemNb==itemNb) {
+                    message = "${_('Showing all items. You can filter by clicking right toolbar buttons.')}"
+                }
+
+                message = message.replace('{0}', visibleItemNb)
+                message = message.replace('{1}', itemNb)
+                $('#search-result-dynamic-resume').html(message);
+            }
+
+            $('.search-result-filter-button').click(function() {
+                refresh_search_result_count();
+            });
+            
+            refresh_search_result_count();
+        });
+    </script>
+
+</%def>
+