From 8d7a216f2f0e1385c64a44ae8e508fb5325d481e Mon Sep 17 00:00:00 2001
From: Shaily <shaily15297@yahoo.com>
Date: Mar 20 2018 00:50:14 +0000
Subject: [PATCH 1/7] Use whoosh for hubs search


---

diff --git a/hubs/models/hub.py b/hubs/models/hub.py
index 6e7b77b..02937e1 100644
--- a/hubs/models/hub.py
+++ b/hubs/models/hub.py
@@ -37,6 +37,7 @@ from hubs.defaults import (
     add_group_widgets, add_user_widgets, add_stream_widgets,
 )
 from hubs.utils import username2avatar
+from hubs.utils.search import hub_index
 from hubs.utils.fedmsg import publish
 from hubs.signals import hub_created
 from hubs.widgets import clean_input
@@ -132,6 +133,12 @@ class Hub(ObjectAuthzMixin, BASE):
         return [assoc.user for assoc in self.associations
                 if assoc.role == 'stargazer']
 
+    def index(self):
+        writer = hub_index.writer()
+        writer.update_document(id=str(self.id), name=self.name,
+                               summary=self.config['summary'])
+        writer.commit()
+
     def publish(self, topic, extra_msg=None):
         msg = {
             "hub_name": self.name,
@@ -257,9 +264,11 @@ class Hub(ObjectAuthzMixin, BASE):
             add_user_widgets(self)
             user = User.query.get(self.name)
             self.subscribe(user, role='owner')
+            self.index()
         elif self.hub_type == "team":
             add_group_widgets(self)
             self.publish("hub.created")
+            self.index()
         elif self.hub_type == "stream":
             add_stream_widgets(self)
 
@@ -269,6 +278,7 @@ class Hub(ObjectAuthzMixin, BASE):
             key for key in new_config.keys()
             if new_config.get(key) != old_config.get(key)
         ]
+        self.index()
         # Notify but don't send the config values on the bus, there
         # may be private stuff there.
         self.publish("hub.updated", {
diff --git a/hubs/utils/search.py b/hubs/utils/search.py
new file mode 100644
index 0000000..07a32fc
--- /dev/null
+++ b/hubs/utils/search.py
@@ -0,0 +1,23 @@
+from __future__ import unicode_literals
+
+import os
+
+from whoosh.fields import Schema, ID, TEXT
+from whoosh.analysis import StandardAnalyzer, CharsetFilter
+from whoosh.index import open_dir, create_in
+from whoosh.support.charset import accent_map
+
+
+def _get_hub_index():
+    index_dir = '/var/tmp/search_index'
+    if not os.path.exists(index_dir):
+        os.mkdir(index_dir)
+        accent_folding = StandardAnalyzer() | CharsetFilter(accent_map)
+        schema = Schema(id=ID(unique=True, stored=True), name=TEXT(),
+                        summary=TEXT(analyzer=accent_folding))
+        create_in(index_dir, schema)
+    ix = open_dir(index_dir)
+    return ix
+
+
+hub_index = _get_hub_index()
diff --git a/hubs/utils/views.py b/hubs/utils/views.py
index 9a13cd6..3a9876b 100644
--- a/hubs/utils/views.py
+++ b/hubs/utils/views.py
@@ -12,9 +12,12 @@ import flask
 from six.moves.urllib import parse as urlparse
 from sqlalchemy import or_, and_
 from sqlalchemy.orm.exc import NoResultFound
+from whoosh.qparser import MultifieldParser
+from whoosh.query import Prefix
 
 from hubs.models import Hub, HubConfig, Widget
 from hubs.models.constants import HUB_TYPES
+from hubs.utils.search import hub_index
 
 
 log = logging.getLogger(__name__)
@@ -51,17 +54,12 @@ def get_hub_by_id(hub_id):
 
 
 def query_hubs(querystring):
-    query = Hub.query.join(HubConfig).filter(
-        Hub.hub_type.in_(["user", "team"]),
-        or_(
-            Hub.name.ilike('%{}%'.format(querystring)),
-            and_(
-                HubConfig.key == "summary",
-                HubConfig.value.ilike('%{}%'.format(querystring)),
-            )
-        )
-    )
-    return query.all()
+    with hub_index.searcher() as searcher:
+        query = MultifieldParser(["name", "summary"], hub_index.schema,
+                                 plugins=[], termclass=Prefix)
+        results = searcher.search(query.parse(querystring))
+        hubs = [get_hub_by_id(result['id']) for result in results]
+    return hubs
 
 
 def get_widget_instance(idx):
diff --git a/requirements.txt b/requirements.txt
index 91e102c..7eb1128 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -28,3 +28,4 @@ setuptools
 six
 sqlalchemy
 txredisapi
+whoosh

From bbc3112958db3b1ae2ec7218ed0ab712c83aca05 Mon Sep 17 00:00:00 2001
From: Shaily <shaily15297@yahoo.com>
Date: Mar 20 2018 00:50:14 +0000
Subject: [PATCH 2/7] Add docblocks to search code


---

diff --git a/hubs/models/hub.py b/hubs/models/hub.py
index 02937e1..baa0fa1 100644
--- a/hubs/models/hub.py
+++ b/hubs/models/hub.py
@@ -134,6 +134,7 @@ class Hub(ObjectAuthzMixin, BASE):
                 if assoc.role == 'stargazer']
 
     def index(self):
+        """ Add or update hub details in the search index. """
         writer = hub_index.writer()
         writer.update_document(id=str(self.id), name=self.name,
                                summary=self.config['summary'])
diff --git a/hubs/utils/search.py b/hubs/utils/search.py
index 07a32fc..dcfc7d0 100644
--- a/hubs/utils/search.py
+++ b/hubs/utils/search.py
@@ -9,6 +9,16 @@ from whoosh.support.charset import accent_map
 
 
 def _get_hub_index():
+    """
+    Returns a whoosh.index.Index object for hub search
+    Used for adding, updating and searching across the
+    indexed collection of documents.
+
+    If the index directory is not present, creates it.
+
+    Returns:
+        Index: Represents an indexed collection of documents.
+    """
     index_dir = '/var/tmp/search_index'
     if not os.path.exists(index_dir):
         os.mkdir(index_dir)

From 7cd6c672126d94ad22cc6f53c194b5226f1286c0 Mon Sep 17 00:00:00 2001
From: Shaily <shaily15297@yahoo.com>
Date: Mar 20 2018 00:50:14 +0000
Subject: [PATCH 3/7] Don't execute index function on import


---

diff --git a/hubs/app.py b/hubs/app.py
index f5cb044..2c6f93a 100644
--- a/hubs/app.py
+++ b/hubs/app.py
@@ -12,6 +12,7 @@ from flask_oidc import OpenIDConnect
 import hubs.models
 from hubs.utils import username2avatar, hub2groupavatar
 from hubs.utils.fedmsg import get_fedmsg_config
+from hubs.utils.search import get_hub_index
 
 app = flask.Flask(__name__)
 
@@ -36,6 +37,7 @@ elif os.path.exists("/etc/fedora-hubs/hubs.py"):
 
 fedmsg_config = get_fedmsg_config()
 
+hub_index = get_hub_index()
 
 # Database
 
diff --git a/hubs/models/hub.py b/hubs/models/hub.py
index baa0fa1..2cdf851 100644
--- a/hubs/models/hub.py
+++ b/hubs/models/hub.py
@@ -37,7 +37,7 @@ from hubs.defaults import (
     add_group_widgets, add_user_widgets, add_stream_widgets,
 )
 from hubs.utils import username2avatar
-from hubs.utils.search import hub_index
+from hubs.utils.search import get_hub_index
 from hubs.utils.fedmsg import publish
 from hubs.signals import hub_created
 from hubs.widgets import clean_input
@@ -135,6 +135,7 @@ class Hub(ObjectAuthzMixin, BASE):
 
     def index(self):
         """ Add or update hub details in the search index. """
+        hub_index = get_hub_index()
         writer = hub_index.writer()
         writer.update_document(id=str(self.id), name=self.name,
                                summary=self.config['summary'])
diff --git a/hubs/utils/search.py b/hubs/utils/search.py
index dcfc7d0..147eaff 100644
--- a/hubs/utils/search.py
+++ b/hubs/utils/search.py
@@ -8,7 +8,7 @@ from whoosh.index import open_dir, create_in
 from whoosh.support.charset import accent_map
 
 
-def _get_hub_index():
+def get_hub_index():
     """
     Returns a whoosh.index.Index object for hub search
     Used for adding, updating and searching across the
@@ -28,6 +28,3 @@ def _get_hub_index():
         create_in(index_dir, schema)
     ix = open_dir(index_dir)
     return ix
-
-
-hub_index = _get_hub_index()
diff --git a/hubs/utils/views.py b/hubs/utils/views.py
index 3a9876b..2732c0f 100644
--- a/hubs/utils/views.py
+++ b/hubs/utils/views.py
@@ -17,7 +17,7 @@ from whoosh.query import Prefix
 
 from hubs.models import Hub, HubConfig, Widget
 from hubs.models.constants import HUB_TYPES
-from hubs.utils.search import hub_index
+from hubs.utils.search import get_hub_index
 
 
 log = logging.getLogger(__name__)
@@ -54,6 +54,7 @@ def get_hub_by_id(hub_id):
 
 
 def query_hubs(querystring):
+    hub_index = get_hub_index()
     with hub_index.searcher() as searcher:
         query = MultifieldParser(["name", "summary"], hub_index.schema,
                                  plugins=[], termclass=Prefix)

From f5ee91890291a940b1a2ffee4ddc9b9af01193b6 Mon Sep 17 00:00:00 2001
From: Shaily <shaily15297@yahoo.com>
Date: Mar 20 2018 00:50:14 +0000
Subject: [PATCH 4/7] Changes after review


---

diff --git a/hubs/app.py b/hubs/app.py
index 2c6f93a..2ff9458 100644
--- a/hubs/app.py
+++ b/hubs/app.py
@@ -37,7 +37,8 @@ elif os.path.exists("/etc/fedora-hubs/hubs.py"):
 
 fedmsg_config = get_fedmsg_config()
 
-hub_index = get_hub_index()
+with app.app_context():
+    hub_index = get_hub_index()
 
 # Database
 
diff --git a/hubs/default_config.py b/hubs/default_config.py
index eb2ee8f..546557c 100644
--- a/hubs/default_config.py
+++ b/hubs/default_config.py
@@ -51,6 +51,8 @@ CHAT_NETWORKS = [
 DATAGREPPER_URI = 'https://apps.fedoraproject.org/datagrepper'
 DEFAULT_SERVER_NAME = "hubs.fedoraproject.org"
 
+SEARCH_INDEX_DIR = '/var/tmp/search_index'
+
 MANAGE_MEMBERSHIP_IN_FAS = True
 
 EMAIL_HOST = "localhost"
diff --git a/hubs/models/hub.py b/hubs/models/hub.py
index 2cdf851..8a142a6 100644
--- a/hubs/models/hub.py
+++ b/hubs/models/hub.py
@@ -133,7 +133,7 @@ class Hub(ObjectAuthzMixin, BASE):
         return [assoc.user for assoc in self.associations
                 if assoc.role == 'stargazer']
 
-    def index(self):
+    def update_index(self):
         """ Add or update hub details in the search index. """
         hub_index = get_hub_index()
         writer = hub_index.writer()
@@ -266,11 +266,11 @@ class Hub(ObjectAuthzMixin, BASE):
             add_user_widgets(self)
             user = User.query.get(self.name)
             self.subscribe(user, role='owner')
-            self.index()
+            self.update_index()
         elif self.hub_type == "team":
             add_group_widgets(self)
             self.publish("hub.created")
-            self.index()
+            self.update_index()
         elif self.hub_type == "stream":
             add_stream_widgets(self)
 
@@ -280,7 +280,7 @@ class Hub(ObjectAuthzMixin, BASE):
             key for key in new_config.keys()
             if new_config.get(key) != old_config.get(key)
         ]
-        self.index()
+        self.update_index()
         # Notify but don't send the config values on the bus, there
         # may be private stuff there.
         self.publish("hub.updated", {
diff --git a/hubs/utils/search.py b/hubs/utils/search.py
index 147eaff..737faac 100644
--- a/hubs/utils/search.py
+++ b/hubs/utils/search.py
@@ -2,6 +2,7 @@ from __future__ import unicode_literals
 
 import os
 
+from flask import current_app
 from whoosh.fields import Schema, ID, TEXT
 from whoosh.analysis import StandardAnalyzer, CharsetFilter
 from whoosh.index import open_dir, create_in
@@ -19,7 +20,7 @@ def get_hub_index():
     Returns:
         Index: Represents an indexed collection of documents.
     """
-    index_dir = '/var/tmp/search_index'
+    index_dir = current_app.config.get('SEARCH_INDEX_DIR')
     if not os.path.exists(index_dir):
         os.mkdir(index_dir)
         accent_folding = StandardAnalyzer() | CharsetFilter(accent_map)

From 7ded259aace61a086d7e98d16872487b9efaa97e Mon Sep 17 00:00:00 2001
From: Shaily <shaily15297@yahoo.com>
Date: Mar 20 2018 00:50:14 +0000
Subject: [PATCH 5/7] Change config key name


Widget schema will be stored in WIDGET_INDEX_DIR

---

diff --git a/hubs/default_config.py b/hubs/default_config.py
index 546557c..88e3e3b 100644
--- a/hubs/default_config.py
+++ b/hubs/default_config.py
@@ -51,7 +51,7 @@ CHAT_NETWORKS = [
 DATAGREPPER_URI = 'https://apps.fedoraproject.org/datagrepper'
 DEFAULT_SERVER_NAME = "hubs.fedoraproject.org"
 
-SEARCH_INDEX_DIR = '/var/tmp/search_index'
+HUB_INDEX_DIR = '/var/tmp/hub_index'
 
 MANAGE_MEMBERSHIP_IN_FAS = True
 
diff --git a/hubs/utils/search.py b/hubs/utils/search.py
index 737faac..ca6970c 100644
--- a/hubs/utils/search.py
+++ b/hubs/utils/search.py
@@ -20,7 +20,7 @@ def get_hub_index():
     Returns:
         Index: Represents an indexed collection of documents.
     """
-    index_dir = current_app.config.get('SEARCH_INDEX_DIR')
+    index_dir = current_app.config.get('HUB_INDEX_DIR')
     if not os.path.exists(index_dir):
         os.mkdir(index_dir)
         accent_folding = StandardAnalyzer() | CharsetFilter(accent_map)

From 934014596508394688fbc1e6f711557d22594e9b Mon Sep 17 00:00:00 2001
From: Shaily <shaily15297@yahoo.com>
Date: Mar 20 2018 00:50:14 +0000
Subject: [PATCH 6/7] Remove redundant variable assignment


---

diff --git a/hubs/app.py b/hubs/app.py
index 2ff9458..0f6fab0 100644
--- a/hubs/app.py
+++ b/hubs/app.py
@@ -38,7 +38,7 @@ elif os.path.exists("/etc/fedora-hubs/hubs.py"):
 fedmsg_config = get_fedmsg_config()
 
 with app.app_context():
-    hub_index = get_hub_index()
+    get_hub_index()
 
 # Database
 

From b4e9c0079750e2312e318246134a9e0e45cb8bd1 Mon Sep 17 00:00:00 2001
From: Shaily <shaily15297@yahoo.com>
Date: Mar 20 2018 01:58:55 +0000
Subject: [PATCH 7/7] Fix team hub creation in populate.py


---

diff --git a/hubs/models/hub.py b/hubs/models/hub.py
index 8a142a6..1bb2f23 100644
--- a/hubs/models/hub.py
+++ b/hubs/models/hub.py
@@ -240,10 +240,11 @@ class Hub(ObjectAuthzMixin, BASE):
         return hub
 
     @classmethod
-    def create_group_hub(cls, name):
+    def create_group_hub(cls, name, summary):
         session = Session()
         hub = cls(name=name, hub_type="team")
         session.add(hub)
+        hub.config["summary"] = summary
         # Commit before sending the signal or other processes won't see it
         # (the workers). Flushing isn't enough.
         session.commit()
diff --git a/populate.py b/populate.py
index efa8ce5..51142e6 100755
--- a/populate.py
+++ b/populate.py
@@ -92,10 +92,9 @@ def create_users():
 
 def create_teams():
     # Ambassadors
-    hub = hubs.models.Hub(name='ambassadors', hub_type="team")
+    hub = hubs.models.Hub.create_group_hub(name='ambassadors', summary='Fedora Ambassadors')
     db.add(hub)
     hub.config.update(dict(
-        summary='Fedora Ambassadors',
         avatar=placekitten,
         description=(
             'Ambassadors are the representatives of Fedora. Ambassadors '
@@ -122,9 +121,8 @@ def create_teams():
     db.commit()
 
 # ############# CommOps
-    hub = hubs.models.Hub(name='commops', hub_type="team")
+    hub = hubs.models.Hub.create_group_hub(name='commops', summary='The Fedora Community Operations Team')
     db.add(hub)
-    hub.config["summary"] = 'The Fedora Community Operations Team'
     hub.config["chat_domain"] = 'irc.freenode.net'
     hub.config["chat_channel"] = '#fedora-commops'
     hub.config["pagure"] = ['fedora-commops']
@@ -149,9 +147,8 @@ def create_teams():
     db.commit()
 
 # ############# Marketing team
-    hub = hubs.models.Hub(name='marketing', hub_type="team")
+    hub = hubs.models.Hub.create_group_hub(name='marketing', summary='The Fedora Marketing Team')
     db.add(hub)
-    hub.config["summary"] = 'The Fedora Marketing Team'
     hub.config["description"] = (
         'The Fedora Marketing Team develops and executes marketing strategies'
         ' to promote the usage and support of Fedora worldwide. Through the'
@@ -182,9 +179,8 @@ def create_teams():
     db.commit()
 
 # ############# Design team
-    hub = hubs.models.Hub(name='designteam', hub_type="team")
+    hub = hubs.models.Hub.create_group_hub(name='designteam', summary='The Fedora Design Team')
     db.add(hub)
-    hub.config["summary"] = 'The Fedora Design Team'
     hub.config["description"] = (
         'The Design Team is the design group of the Fedora project. Our'
         ' interests are not only in creating graphics for use by the'
@@ -216,9 +212,8 @@ def create_teams():
 
 
 # ############# Infra team -- commented out, as there is no infra FAS group yet
-    # hub = hubs.models.Hub(name='infrastructure', hub_type="team")
+    # hub = hubs.models.Hub.create_group_hub(name='infrastructure', summary='The Fedora Infra Team')
     # db.add(hub)
-    # hub.config["summary"] = 'The Fedora Infra Team'
     # hub.config["description"] = """
     # The Infrastructure Team consists of dedicated volunteers and professional
     # managing the servers, building the tools and utilities, and creating new