Merge pull request #291 from uktrade/feature/search-custom-analyzer

Improved search results.

Author: elcct

datahub/search/apps.py

@@ -13,6 +13,7 @@ class SearchConfig(AppConfig):
     def ready(self):
         """Configures Elasticsearch default connection."""
         elasticsearch.configure_connection()
+        elasticsearch.configure_index(index=settings.ES_INDEX)
 
         # Makes sure mappings exist in Elasticsearch.
         # Those calls are idempotent

datahub/search/elasticsearch.py

@@ -1,13 +1,18 @@
 from collections import defaultdict
-from urllib.parse import ParseResult, urlparse  # noqa: F401
 
 import dateutil.parser
 from django.conf import settings
 from elasticsearch.helpers import bulk as es_bulk
-from elasticsearch_dsl import Search
+from elasticsearch_dsl import analysis, Index, Search
 from elasticsearch_dsl.connections import connections
 from elasticsearch_dsl.query import Match, MatchPhrase, Q
 
+lowercase_keyword_analyzer = analysis.CustomAnalyzer(
+    'lowercase_keyword_analyzer',
+    tokenizer='keyword',
+    filter=['lowercase']
+)
+
 
 def configure_connection():
     """Configure Elasticsearch default connection."""
@@ -18,12 +23,20 @@ def configure_connection():
     )
 
 
+def configure_index(index_name, settings=None):
+    """Configures Elasticsearch index."""
+    index = Index(index_name)
+    index.analyzer(lowercase_keyword_analyzer)
+    if settings:
+        index.settings(**settings)
+    index.create()
+
+
 def get_search_term_query(term):
     """Returns search term query."""
     return Q('bool', should=[
-        MatchPhrase(name={'query': term, 'slop': 200}),
+        MatchPhrase(name_keyword={'query': term, 'boost': 2}),
         Match(name={'query': term}),
-        MatchPhrase(_all={'query': term}),
         Match(_all={'query': term}),
     ])
 

datahub/search/models.py

@@ -1,5 +1,6 @@
 from django.conf import settings
-from elasticsearch_dsl import Boolean, Date, DocType, Double, Integer, Nested, String
+from elasticsearch_dsl import (Boolean, Date, DocType, Double,
+                               Integer, Nested, String)
 
 
 def _id_name_dict(obj):
@@ -113,7 +114,8 @@ class Company(DocType, MapDBModelToDict):
     headquarter_type = Nested(properties={'id': String(index='not_analyzed'), 'name': String()})
     id = String(index='not_analyzed')
     modified_on = Date()
-    name = String()
+    name = String(copy_to='name_keyword')
+    name_keyword = String(analyzer='lowercase_keyword_analyzer')
     one_list_account_owner = Nested(properties={'id': String(index='not_analyzed'),
                                                 'first_name': String(copy_to='one_list_account_owner.name'),
                                                 'last_name': String(copy_to='one_list_account_owner.name'),
@@ -185,7 +187,8 @@ class Contact(DocType, MapDBModelToDict):
     created_on = Date()
     modified_on = Date()
     id = String(index='not_analyzed')
-    name = String()
+    name = String(copy_to='name_keyword')
+    name_keyword = String(analyzer='lowercase_keyword_analyzer')
     title = Nested(properties={'id': String(index='not_analyzed'), 'name': String(copy_to='name')})
     first_name = String(copy_to='name')
     last_name = String(copy_to='name')
@@ -307,8 +310,8 @@ class InvestmentProject(DocType, MapDBModelToDict):
         'id': String(index='not_analyzed'),
         'name': String()
     })  # InvestmentType
-    name = String()
-    description = String()
+    name = String(copy_to='name_keyword')
+    name_keyword = String(analyzer='lowercase_keyword_analyzer')
     r_and_d_budget = Boolean()
     non_fdi_r_and_d_budget = Boolean()
     new_tech_to_uk = Boolean()

datahub/search/test/conftest.py

@@ -4,15 +4,14 @@
 from django.core import management
 from django.db.models.signals import post_save
 from elasticsearch.helpers.test import get_test_client
-from elasticsearch_dsl import Index
 from pytest import fixture
 
 from datahub.company.models import Company, Contact
 from datahub.company.test.factories import CompanyFactory, ContactFactory
 from datahub.core import constants
 from datahub.investment.models import InvestmentProject
 from datahub.investment.test.factories import InvestmentProjectFactory
-from datahub.search import models
+from datahub.search import models, elasticsearch
 from datahub.search.management.commands import sync_es
 
 
@@ -74,9 +73,10 @@ def create_test_index(client, index):
     if client.indices.exists(index=index):
         client.indices.delete(index)
 
-    index = Index(index)
-    index.create()
-
+    elasticsearch.configure_index(index, {
+        'number_of_shards': 1,
+        'number_of_replicas': 0,
+    })
 
 @fixture
 def post_save_handlers():

datahub/search/test/test_elasticsearch.py

@@ -13,9 +13,9 @@ def test_get_search_term_query():
             'should': [
                 {
                     'match_phrase': {
-                        'name': {
+                        'name_keyword': {
                             'query': 'hello',
-                            'slop': 200
+                            'boost': 2
                         }
                     }
                 }, {
@@ -24,12 +24,6 @@ def test_get_search_term_query():
                             'query': 'hello',
                         }
                     }
-                }, {
-                    'match_phrase': {
-                        '_all': {
-                            'query': 'hello',
-                        }
-                    }
                 }, {
                     'match': {
                         '_all': {
@@ -52,9 +46,9 @@ def test_get_basic_search_query():
                 'should': [
                     {
                         'match_phrase': {
-                            'name': {
+                            'name_keyword': {
                                 'query': 'test',
-                                'slop': 200
+                                'boost': 2
                             }
                         }
                     }, {
@@ -63,12 +57,6 @@ def test_get_basic_search_query():
                                 'query': 'test',
                             }
                         }
-                    }, {
-                        'match_phrase': {
-                            '_all': {
-                                'query': 'test',
-                            }
-                        }
                     }, {
                         'match': {
                             '_all': {
@@ -137,9 +125,9 @@ def test_search_by_entity_query():
                             'should': [
                                 {
                                     'match_phrase': {
-                                        'name': {
+                                        'name_keyword': {
                                             'query': 'test',
-                                            'slop': 200
+                                            'boost': 2
                                         }
                                     }
                                 }, {
@@ -148,12 +136,6 @@ def test_search_by_entity_query():
                                             'query': 'test',
                                         }
                                     }
-                                }, {
-                                    'match_phrase': {
-                                        '_all': {
-                                            'query': 'test',
-                                        }
-                                    }
                                 }, {
                                     'match': {
                                         '_all': {

datahub/search/test/test_views.py

@@ -1,5 +1,5 @@
 import datetime
-from unittest import mock, skip
+from unittest import mock
 
 import pytest
 from elasticsearch_dsl.connections import connections
@@ -217,7 +217,6 @@ def test_search_investment_project_no_filters(self):
 
         assert response.status_code == status.HTTP_400_BAD_REQUEST
 
-    @skip('This test fails randomly. Skip until fixed.')
     @mock.patch('datahub.core.utils.executor.submit', synchronous_executor_submit)
     @mock.patch('django.db.transaction.on_commit', synchronous_transaction_on_commit)
     def test_search_results_quality(self):