Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[SE-3989] - Fixes for Blockstore integration tests. #26230

Merged
merged 1 commit into from
Feb 8, 2021
Merged

[SE-3989] - Fixes for Blockstore integration tests. #26230

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 18 additions & 35 deletions openedx/core/djangoapps/content_libraries/libraries_index.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@ class SearchIndexerBase(ABC):
Abstract Base Class for implementing library search indexers.
"""
INDEX_NAME = None
DOCUMENT_TYPE = None
ENABLE_INDEXING_KEY = None
SCHEMA_VERSION = 0
SEARCH_KWARGS = {
Expand All @@ -56,7 +55,7 @@ def index_items(cls, items):
"""
searcher = SearchEngine.get_search_engine(cls.INDEX_NAME)
items = [cls.get_item_definition(item) for item in items]
return searcher.index(cls.DOCUMENT_TYPE, items, **cls.SEARCH_KWARGS)
return searcher.index(items, **cls.SEARCH_KWARGS)

@classmethod
def get_items(cls, ids=None, filter_terms=None, text_search=None):
Expand All @@ -79,7 +78,7 @@ def get_items(cls, ids=None, filter_terms=None, text_search=None):
response = cls._perform_elastic_search(filter_terms, text_search)
else:
searcher = SearchEngine.get_search_engine(cls.INDEX_NAME)
response = searcher.search(doc_type=cls.DOCUMENT_TYPE, field_dictionary=filter_terms, size=MAX_SIZE)
response = searcher.search(field_dictionary=filter_terms, size=MAX_SIZE)

response = [result["data"] for result in response["results"]]
return sorted(response, key=lambda i: i["id"])
Expand All @@ -91,17 +90,17 @@ def remove_items(cls, ids):
"""
searcher = SearchEngine.get_search_engine(cls.INDEX_NAME)
ids_str = [str(i) for i in ids]
searcher.remove(cls.DOCUMENT_TYPE, ids_str, **cls.SEARCH_KWARGS)
searcher.remove(ids_str, **cls.SEARCH_KWARGS)

@classmethod
def remove_all_items(cls):
"""
Remove all items from the index
"""
searcher = SearchEngine.get_search_engine(cls.INDEX_NAME)
response = searcher.search(doc_type=cls.DOCUMENT_TYPE, filter_dictionary={}, size=MAX_SIZE)
response = searcher.search(filter_dictionary={}, size=MAX_SIZE)
ids = [result["data"]["id"] for result in response["results"]]
searcher.remove(cls.DOCUMENT_TYPE, ids, **cls.SEARCH_KWARGS)
searcher.remove(ids, **cls.SEARCH_KWARGS)

@classmethod
def indexing_is_enabled(cls):
Expand All @@ -117,7 +116,6 @@ def _perform_elastic_search(cls, filter_terms, text_search):
"""
searcher = SearchEngine.get_search_engine(cls.INDEX_NAME)
return _translate_hits(searcher._es.search( # pylint: disable=protected-access
doc_type=cls.DOCUMENT_TYPE,
index=searcher.index_name,
body=cls.build_elastic_query(filter_terms, text_search),
size=MAX_SIZE
Expand All @@ -130,7 +128,7 @@ def build_elastic_query(filter_terms, text_search):
"""
# Remove reserved characters (and ") from the text to prevent unexpected errors.
text_search_normalised = text_search.translate(text_search.maketrans('', '', RESERVED_CHARACTERS + '"'))
text_search_normalised = text_search.replace('-', ' ')
text_search_normalised = text_search_normalised.replace('-', ' ')
# Wrap with asterix to enable partial matches
text_search_normalised = "*{}*".format(text_search_normalised)
terms = [
Expand All @@ -143,32 +141,19 @@ def build_elastic_query(filter_terms, text_search):
]
return {
'query': {
'filtered': {
'query': {
'bool': {
'should': [
{
'query_string': {
'query': text_search_normalised,
"fields": ["content.*"],
"minimum_should_match": "100%",
},
},
# Add a special wildcard search for id, as it contains a ":" character which is
# filtered out in query_string
{
'wildcard': {
'id': {
'value': '*{}*'.format(text_search),
}
},
},
],
'bool': {
'must': [
{
'query_string': {
'query': text_search_normalised,
"fields": ["content.*"],
'minimum_should_match': '100%',
},
},
},
],
'filter': {
'bool': {
'must': terms
'must': terms,
}
}
},
Expand All @@ -183,7 +168,6 @@ class ContentLibraryIndexer(SearchIndexerBase):

INDEX_NAME = "content_library_index"
ENABLE_INDEXING_KEY = "ENABLE_CONTENT_LIBRARY_INDEX"
DOCUMENT_TYPE = "content_library"
SCHEMA_VERSION = 0

@classmethod
Expand Down Expand Up @@ -212,7 +196,7 @@ def get_item_definition(cls, item):
"last_published": last_published_str,
"has_unpublished_changes": has_unpublished_changes,
"has_unpublished_deletes": has_unpublished_deletes,
# only 'content' field is analyzed by elastisearch, and allows text-search
# only 'content' field is analyzed by elasticsearch, and allows text-search
"content": {
"id": str(item),
"title": bundle_metadata.title,
Expand All @@ -226,9 +210,8 @@ class LibraryBlockIndexer(SearchIndexerBase):
Class to perform indexing on the XBlocks in content libraries.
"""

INDEX_NAME = "content_library_index"
INDEX_NAME = "content_library_block_index"
ENABLE_INDEXING_KEY = "ENABLE_CONTENT_LIBRARY_INDEX"
DOCUMENT_TYPE = "content_library_block"
SCHEMA_VERSION = 0

@classmethod
Expand Down
1 change: 0 additions & 1 deletion openedx/core/djangoapps/content_libraries/tests/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,6 @@ def elasticsearch_test(func):
def mock_perform(cls, filter_terms, text_search):
# pylint: disable=no-member
return SearchEngine.get_search_engine(cls.INDEX_NAME).search(
doc_type=cls.DOCUMENT_TYPE,
field_dictionary=filter_terms,
query_string=text_search,
size=MAX_SIZE
Expand Down
49 changes: 35 additions & 14 deletions openedx/core/djangoapps/content_libraries/tests/test_content_libraries.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
from mock import patch
from organizations.models import Organization

from openedx.core.djangoapps.content_libraries.libraries_index import LibraryBlockIndexer, ContentLibraryIndexer
from openedx.core.djangoapps.content_libraries.tests.base import ContentLibrariesRestApiTest, elasticsearch_test
from openedx.core.djangoapps.content_libraries.constants import VIDEO, COMPLEX, PROBLEM, CC_4_BY, ALL_RIGHTS_RESERVED
from common.djangoapps.student.tests.factories import UserFactory
Expand Down Expand Up @@ -42,6 +43,12 @@ class ContentLibrariesTest(ContentLibrariesRestApiTest):
and cached forever.
"""

def setUp(self):
super().setUp()
if settings.ENABLE_ELASTICSEARCH_FOR_TESTS:
ContentLibraryIndexer.remove_all_items()
LibraryBlockIndexer.remove_all_items()

def test_library_crud(self):
"""
Test Create, Read, Update, and Delete of a Content Library
Expand Down Expand Up @@ -218,30 +225,44 @@ def test_library_filters(self, is_indexing_enabled):
"""
Test the filters in the list libraries API
"""
suffix = str(is_indexing_enabled)
with override_settings(FEATURES={**settings.FEATURES, 'ENABLE_CONTENT_LIBRARY_INDEX': is_indexing_enabled}):
self._create_library(slug="test-lib1", title="Foo", description="Bar", library_type=VIDEO)
self._create_library(slug="test-lib2", title="Library-Title-2", description="Bar2")
self._create_library(slug="l3", title="Library-Title-3", description="Description", library_type=VIDEO)
self._create_library(
slug=f"test-lib-filter-{suffix}-1", title="Fob", description=f"Bar-{suffix}", library_type=VIDEO,
)
self._create_library(
slug=f"test-lib-filter-{suffix}-2", title=f"Library-Title-{suffix}-2", description=f"Bar-{suffix}-2",
)
self._create_library(
slug=f"l3{suffix}", title=f"Library-Title-{suffix}-3", description="Description", library_type=VIDEO,
)

Organization.objects.get_or_create(
short_name="org-test",
short_name=f"org-test-{suffix}",
defaults={"name": "Content Libraries Tachyon Exploration & Survey Team"},
)
self._create_library(
slug="l4", title="Library-Title-4", description="Library-Description", org='org-test',
slug=f"l4-{suffix}", title=f"Library-Title-{suffix}-4",
description="Library-Description", org=f'org-test-{suffix}',
library_type=VIDEO,
)
self._create_library(slug="l5", title="Library-Title-5", description="Library-Description", org='org-test')
self._create_library(
slug="l5", title=f"Library-Title-{suffix}-5", description="Library-Description",
org=f'org-test-{suffix}',
)

self.assertEqual(len(self._list_libraries()), 5)
self.assertEqual(len(self._list_libraries({'org': 'org-test'})), 2)
self.assertEqual(len(self._list_libraries({'text_search': 'test-lib'})), 2)
self.assertEqual(len(self._list_libraries({'text_search': 'test-lib', 'type': VIDEO})), 1)
self.assertEqual(len(self._list_libraries({'text_search': 'library-title'})), 4)
self.assertEqual(len(self._list_libraries({'text_search': 'library-title', 'type': VIDEO})), 2)
self.assertEqual(len(self._list_libraries({'text_search': 'bar'})), 2)
self.assertEqual(len(self._list_libraries({'text_search': 'org-tes'})), 2)
self.assertEqual(len(self._list_libraries({'org': 'org-test', 'text_search': 'library-title-4'})), 1)
self.assertEqual(len(self._list_libraries({'org': f'org-test-{suffix}'})), 2)
self.assertEqual(len(self._list_libraries({'text_search': f'test-lib-filter-{suffix}'})), 2)
self.assertEqual(len(self._list_libraries({'text_search': f'test-lib-filter-{suffix}', 'type': VIDEO})), 1)
self.assertEqual(len(self._list_libraries({'text_search': f'library-title-{suffix}'})), 4)
self.assertEqual(len(self._list_libraries({'text_search': f'library-title-{suffix}', 'type': VIDEO})), 2)
self.assertEqual(len(self._list_libraries({'text_search': f'bar-{suffix}'})), 2)
self.assertEqual(len(self._list_libraries({'text_search': f'org-test-{suffix}'})), 2)
self.assertEqual(
len(self._list_libraries({'org': f'org-test-{suffix}', 'text_search': f'library-title-{suffix}-4'})),
1,
)
self.assertEqual(len(self._list_libraries({'type': VIDEO})), 3)

# General Content Library XBlock tests:
Expand Down