Skip to content

Commit

Permalink
Update Library v2 Indexes to work with ElasticSearch 7. Repair blocks…
Browse files Browse the repository at this point in the history 
…tore integration tests. (#26230)
  • Loading branch information
@Kelketek
Kelketek authored Feb 8, 2021
1 parent 367d2a0 commit c840e6f
Show file tree
Hide file tree
Showing 3 changed files with 53 additions and 50 deletions.
53 changes: 18 additions & 35 deletions openedx/core/djangoapps/content_libraries/libraries_index.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@ class SearchIndexerBase(ABC):
Abstract Base Class for implementing library search indexers.
"""
INDEX_NAME = None
DOCUMENT_TYPE = None
ENABLE_INDEXING_KEY = None
SCHEMA_VERSION = 0
SEARCH_KWARGS = {
Expand All @@ -56,7 +55,7 @@ def index_items(cls, items):
"""
searcher = SearchEngine.get_search_engine(cls.INDEX_NAME)
items = [cls.get_item_definition(item) for item in items]
return searcher.index(cls.DOCUMENT_TYPE, items, **cls.SEARCH_KWARGS)
return searcher.index(items, **cls.SEARCH_KWARGS)

@classmethod
def get_items(cls, ids=None, filter_terms=None, text_search=None):
Expand All @@ -79,7 +78,7 @@ def get_items(cls, ids=None, filter_terms=None, text_search=None):
response = cls._perform_elastic_search(filter_terms, text_search)
else:
searcher = SearchEngine.get_search_engine(cls.INDEX_NAME)
response = searcher.search(doc_type=cls.DOCUMENT_TYPE, field_dictionary=filter_terms, size=MAX_SIZE)
response = searcher.search(field_dictionary=filter_terms, size=MAX_SIZE)

response = [result["data"] for result in response["results"]]
return sorted(response, key=lambda i: i["id"])
Expand All @@ -91,17 +90,17 @@ def remove_items(cls, ids):
"""
searcher = SearchEngine.get_search_engine(cls.INDEX_NAME)
ids_str = [str(i) for i in ids]
searcher.remove(cls.DOCUMENT_TYPE, ids_str, **cls.SEARCH_KWARGS)
searcher.remove(ids_str, **cls.SEARCH_KWARGS)

@classmethod
def remove_all_items(cls):
"""
Remove all items from the index
"""
searcher = SearchEngine.get_search_engine(cls.INDEX_NAME)
response = searcher.search(doc_type=cls.DOCUMENT_TYPE, filter_dictionary={}, size=MAX_SIZE)
response = searcher.search(filter_dictionary={}, size=MAX_SIZE)
ids = [result["data"]["id"] for result in response["results"]]
searcher.remove(cls.DOCUMENT_TYPE, ids, **cls.SEARCH_KWARGS)
searcher.remove(ids, **cls.SEARCH_KWARGS)

@classmethod
def indexing_is_enabled(cls):
Expand All @@ -117,7 +116,6 @@ def _perform_elastic_search(cls, filter_terms, text_search):
"""
searcher = SearchEngine.get_search_engine(cls.INDEX_NAME)
return _translate_hits(searcher._es.search( # pylint: disable=protected-access
doc_type=cls.DOCUMENT_TYPE,
index=searcher.index_name,
body=cls.build_elastic_query(filter_terms, text_search),
size=MAX_SIZE
Expand All @@ -130,7 +128,7 @@ def build_elastic_query(filter_terms, text_search):
"""
# Remove reserved characters (and ") from the text to prevent unexpected errors.
text_search_normalised = text_search.translate(text_search.maketrans('', '', RESERVED_CHARACTERS + '"'))
text_search_normalised = text_search.replace('-', ' ')
text_search_normalised = text_search_normalised.replace('-', ' ')
# Wrap with asterix to enable partial matches
text_search_normalised = "*{}*".format(text_search_normalised)
terms = [
Expand All @@ -143,32 +141,19 @@ def build_elastic_query(filter_terms, text_search):
]
return {
'query': {
'filtered': {
'query': {
'bool': {
'should': [
{
'query_string': {
'query': text_search_normalised,
"fields": ["content.*"],
"minimum_should_match": "100%",
},
},
# Add a special wildcard search for id, as it contains a ":" character which is
# filtered out in query_string
{
'wildcard': {
'id': {
'value': '*{}*'.format(text_search),
}
},
},
],
'bool': {
'must': [
{
'query_string': {
'query': text_search_normalised,
"fields": ["content.*"],
'minimum_should_match': '100%',
},
},
},
],
'filter': {
'bool': {
'must': terms
'must': terms,
}
}
},
Expand All @@ -183,7 +168,6 @@ class ContentLibraryIndexer(SearchIndexerBase):

INDEX_NAME = "content_library_index"
ENABLE_INDEXING_KEY = "ENABLE_CONTENT_LIBRARY_INDEX"
DOCUMENT_TYPE = "content_library"
SCHEMA_VERSION = 0

@classmethod
Expand Down Expand Up @@ -212,7 +196,7 @@ def get_item_definition(cls, item):
"last_published": last_published_str,
"has_unpublished_changes": has_unpublished_changes,
"has_unpublished_deletes": has_unpublished_deletes,
# only 'content' field is analyzed by elastisearch, and allows text-search
# only 'content' field is analyzed by elasticsearch, and allows text-search
"content": {
"id": str(item),
"title": bundle_metadata.title,
Expand All @@ -226,9 +210,8 @@ class LibraryBlockIndexer(SearchIndexerBase):
Class to perform indexing on the XBlocks in content libraries.
"""

INDEX_NAME = "content_library_index"
INDEX_NAME = "content_library_block_index"
ENABLE_INDEXING_KEY = "ENABLE_CONTENT_LIBRARY_INDEX"
DOCUMENT_TYPE = "content_library_block"
SCHEMA_VERSION = 0

@classmethod
Expand Down
1 change: 0 additions & 1 deletion openedx/core/djangoapps/content_libraries/tests/base.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,6 @@ def elasticsearch_test(func):
def mock_perform(cls, filter_terms, text_search):
# pylint: disable=no-member
return SearchEngine.get_search_engine(cls.INDEX_NAME).search(
doc_type=cls.DOCUMENT_TYPE,
field_dictionary=filter_terms,
query_string=text_search,
size=MAX_SIZE
Expand Down
49 changes: 35 additions & 14 deletions openedx/core/djangoapps/content_libraries/tests/test_content_libraries.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
from mock import patch
from organizations.models import Organization

from openedx.core.djangoapps.content_libraries.libraries_index import LibraryBlockIndexer, ContentLibraryIndexer
from openedx.core.djangoapps.content_libraries.tests.base import ContentLibrariesRestApiTest, elasticsearch_test
from openedx.core.djangoapps.content_libraries.constants import VIDEO, COMPLEX, PROBLEM, CC_4_BY, ALL_RIGHTS_RESERVED
from common.djangoapps.student.tests.factories import UserFactory
Expand Down Expand Up @@ -42,6 +43,12 @@ class ContentLibrariesTest(ContentLibrariesRestApiTest):
and cached forever.
"""

def setUp(self):
super().setUp()
if settings.ENABLE_ELASTICSEARCH_FOR_TESTS:
ContentLibraryIndexer.remove_all_items()
LibraryBlockIndexer.remove_all_items()

def test_library_crud(self):
"""
Test Create, Read, Update, and Delete of a Content Library
Expand Down Expand Up @@ -218,30 +225,44 @@ def test_library_filters(self, is_indexing_enabled):
"""
Test the filters in the list libraries API
"""
suffix = str(is_indexing_enabled)
with override_settings(FEATURES={**settings.FEATURES, 'ENABLE_CONTENT_LIBRARY_INDEX': is_indexing_enabled}):
self._create_library(slug="test-lib1", title="Foo", description="Bar", library_type=VIDEO)
self._create_library(slug="test-lib2", title="Library-Title-2", description="Bar2")
self._create_library(slug="l3", title="Library-Title-3", description="Description", library_type=VIDEO)
self._create_library(
slug=f"test-lib-filter-{suffix}-1", title="Fob", description=f"Bar-{suffix}", library_type=VIDEO,
)
self._create_library(
slug=f"test-lib-filter-{suffix}-2", title=f"Library-Title-{suffix}-2", description=f"Bar-{suffix}-2",
)
self._create_library(
slug=f"l3{suffix}", title=f"Library-Title-{suffix}-3", description="Description", library_type=VIDEO,
)

Organization.objects.get_or_create(
short_name="org-test",
short_name=f"org-test-{suffix}",
defaults={"name": "Content Libraries Tachyon Exploration & Survey Team"},
)
self._create_library(
slug="l4", title="Library-Title-4", description="Library-Description", org='org-test',
slug=f"l4-{suffix}", title=f"Library-Title-{suffix}-4",
description="Library-Description", org=f'org-test-{suffix}',
library_type=VIDEO,
)
self._create_library(slug="l5", title="Library-Title-5", description="Library-Description", org='org-test')
self._create_library(
slug="l5", title=f"Library-Title-{suffix}-5", description="Library-Description",
org=f'org-test-{suffix}',
)

self.assertEqual(len(self._list_libraries()), 5)
self.assertEqual(len(self._list_libraries({'org': 'org-test'})), 2)
self.assertEqual(len(self._list_libraries({'text_search': 'test-lib'})), 2)
self.assertEqual(len(self._list_libraries({'text_search': 'test-lib', 'type': VIDEO})), 1)
self.assertEqual(len(self._list_libraries({'text_search': 'library-title'})), 4)
self.assertEqual(len(self._list_libraries({'text_search': 'library-title', 'type': VIDEO})), 2)
self.assertEqual(len(self._list_libraries({'text_search': 'bar'})), 2)
self.assertEqual(len(self._list_libraries({'text_search': 'org-tes'})), 2)
self.assertEqual(len(self._list_libraries({'org': 'org-test', 'text_search': 'library-title-4'})), 1)
self.assertEqual(len(self._list_libraries({'org': f'org-test-{suffix}'})), 2)
self.assertEqual(len(self._list_libraries({'text_search': f'test-lib-filter-{suffix}'})), 2)
self.assertEqual(len(self._list_libraries({'text_search': f'test-lib-filter-{suffix}', 'type': VIDEO})), 1)
self.assertEqual(len(self._list_libraries({'text_search': f'library-title-{suffix}'})), 4)
self.assertEqual(len(self._list_libraries({'text_search': f'library-title-{suffix}', 'type': VIDEO})), 2)
self.assertEqual(len(self._list_libraries({'text_search': f'bar-{suffix}'})), 2)
self.assertEqual(len(self._list_libraries({'text_search': f'org-test-{suffix}'})), 2)
self.assertEqual(
len(self._list_libraries({'org': f'org-test-{suffix}', 'text_search': f'library-title-{suffix}-4'})),
1,
)
self.assertEqual(len(self._list_libraries({'type': VIDEO})), 3)

# General Content Library XBlock tests:
Expand Down

0 comments on commit c840e6f

Please sign in to comment.