lowercased the email address import from ldap and added search term lookup to handle better queries

2 files changed, 50 insertions, 3 deletions

diff --git a/lib/d2/app/adapters/search.py b/lib/d2/app/adapters/search.py
index 14df59f..e507bc1 100644
--- a/lib/d2/app/adapters/search.py
+++ b/lib/d2/app/adapters/search.py
@@ -12,6 +12,7 @@ from whoosh.fields import Schema, TEXT, NUMERIC, ID
 from whoosh.index import create_in
 from whoosh.index import open_dir
 from whoosh.qparser import QueryParser
+from whoosh.spelling import SpellChecker
 import os
 import stat
 
@@ -83,6 +84,12 @@ class SearchAdapter(BaseAdapter):
         for row in results:
             writer.add_document(**row)
         writer.commit()
+        self._speller(results)
+
+    def _speller(self, data):
+        ix = open_dir(self.index_directory)
+        speller = SpellChecker(ix.storage)
+        speller.add_field(ix, "context")
 
     def search_index_data(self, date_start=None):
         hold = {u'plot':{}, u'occupant':{}}
@@ -128,11 +135,51 @@ class SearchAdapter(BaseAdapter):
             out['context'] = u' '.join(hold)
             return out
 
-    def search(self, search_):
+    def _all_terms(self, user_query):
+        out = []
+        terms = set()
+        user_query.all_terms(terms)
+        for key, val in terms:
+            if key == 'context':
+                out.append(val)
+        return out
+
+    def _missing_terms(self, user_query):
+        out = []
+        terms = set()
+        user_query.existing_terms(self.ix.reader(), terms, reverse=True)
+        for key, val in terms:
+            if key == 'context':
+                out.append(val)
+        return out
+
+    def _build_query(self, user_query_string):
+        parser = QueryParser("context", self.ix.schema)
+        user_query = parser.parse(user_query_string)
+        all_terms = self._all_terms(user_query)
+        missing_terms = self._missing_terms(user_query)
+        if missing_terms:
+            out = []
+            speller = SpellChecker(self.ix.storage)
+            for term in all_terms:
+                if term in missing_terms:
+                    suggestions = speller.suggest(term)
+                    if suggestions:
+                        out.append(suggestions[0])
+                    else:
+                        out.append(term)
+                else:
+                    out.append(term)
+            return u' '.join(out)
+        else:
+            return u' '.join(all_terms)
+
+    def search(self, user_query_string):
         out = []
+        user_query_string = self._build_query(user_query_string)
         with self.ix.searcher() as searcher:
             query = QueryParser("context", self.ix.schema)
-            myquery = query.parse(search_)
+            myquery = query.parse(user_query_string)
             results = searcher.search(myquery)
             for row in results[0:len(results)]:
                 if row[u'id_type'] == 'plot':
diff --git a/lib/d2/bin/d2_data_merge.py b/lib/d2/bin/d2_data_merge.py
index 01c1c5e..2270830 100644
--- a/lib/d2/bin/d2_data_merge.py
+++ b/lib/d2/bin/d2_data_merge.py
@@ -335,7 +335,7 @@ class DbEmail(BaseDb):
 
     def _grab_data(self, occupant):
         if 'mail' in occupant.ldap_data.keys():
-            return unicode(occupant.ldap_data['mail'])
+            return unicode(occupant.ldap_data['mail']).lower()
 
 
 class DbDepartment(BaseDb):