Skip to content

Commit 7d8e8fb

Browse files
CaptainDarioCaptainDario
committed
updated the searching
* searches for the search term, kana version and deconjugation at the same time * blanks at the beginning and end of searches terms are ignored
1 parent 1472969 commit 7d8e8fb

13 files changed

+247
-141
lines changed

‎CHANGELOG.md

+3
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,9 @@ Improvements:
1414

1515
- Dictionary
1616
- New deconjugation engine, handles more deconjugation patterns
17+
- Improved searching
18+
- Searches for multiple possible search terms at the same time
19+
- Ignore white space at the beginning and end of search terms
1720
- Image search and DeepL supported on MacOS and Windows
1821
- Back navigation clears search and resets current entry
1922
- Readability of kanji stroke order diagrams

‎lib/application/dictionary/dictionary_search.dart

+30-27
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,8 @@ import 'package:tuple/tuple.dart';
1313
/// 2. sort inside each category based on <br/>
1414
/// 1. word frequency
1515
List<List<JMdict>> sortJmdictList(
16-
List<JMdict> entries, String query, String? queryKana, List<String> languages,
17-
bool convertToHiragana
16+
List<JMdict> entries, String query, String? queryKana, String? queryDeconjugated,
17+
List<String> languages
1818
){
1919

2020
/// lists with three sub lists
@@ -33,11 +33,11 @@ List<List<JMdict>> sortJmdictList(
3333
// iterate over the entries and create a ranking for each
3434
for (JMdict entry in entries) {
3535
// KANJI matched (normal query)
36-
Tuple3 ranked = rankMatches([entry.kanjiIndexes], query, queryKana);
36+
Tuple3 ranked = rankMatches([entry.kanjiIndexes], query, queryKana, queryDeconjugated);
3737

3838
// READING matched
39-
if(ranked.item1 == -1 && queryKana != null){
40-
ranked = rankMatches([entry.hiraganas], query, queryKana);
39+
if(ranked.item1 == -1){
40+
ranked = rankMatches([entry.hiraganas], query, queryKana, queryDeconjugated);
4141
}
4242

4343
// MEANING matched
@@ -52,13 +52,13 @@ List<List<JMdict>> sortJmdictList(
5252
)
5353
.toList();
5454

55-
ranked = rankMatches(k, query, queryKana);
55+
ranked = rankMatches(k, query, queryKana, queryDeconjugated);
5656
}
57-
// the query was found in this entry
57+
// the query was found in this entry せんせ
5858
if(ranked.item1 != -1){
59-
matches[ranked.item1].add(entry);
60-
matchIndices[ranked.item1].add(ranked.item3);
61-
lenDifferences[ranked.item1].add(ranked.item2);
59+
matches[ranked.item1%3].add(entry);
60+
matchIndices[ranked.item1%3].add(ranked.item3);
61+
lenDifferences[ranked.item1%3].add(ranked.item2);
6262
}
6363
}
6464

@@ -82,39 +82,42 @@ List<List<JMdict>> sortJmdictList(
8282
/// 1 - if it was a full (0), start(1) or other(2) match <br/>
8383
/// 2 - how many characters are in the match but not in `queryText` <br/>
8484
/// 3 - the index where the search matched <br/>
85-
Tuple3<int, int, int> rankMatches(List<List<String>> matches, String queryText,
86-
String? queryKana) {
85+
Tuple3<int, int, int> rankMatches(List<List<String>> matches,
86+
String queryText, String? queryKana, String? queryDeconjugated) {
8787

8888
int result = -1, lenDiff = -1; List<int> matchIndeces = [-1, -1];
89+
90+
List<String> allSearches = [queryText, queryKana, queryDeconjugated].nonNulls.toList();
8991

9092
// convert query and matches to lower case; find where the query matched
9193
queryText = queryText.toLowerCase();
9294
for (var i = 0; i < matches.length; i++) {
9395
for (var j = 0; j < matches[i].length; j++) {
9496
matches[i][j] = matches[i][j].toLowerCase();
95-
if(matches[i][j].contains(queryText) ||
96-
queryKana != null && matches[i][j].contains(queryKana)){
97+
if(matches[i][j].contains(RegExp(allSearches.join("|")))){
9798
matchIndeces = [i, j];
9899
break;
99100
}
100101
}
101102
}
102103

103104
if(matchIndeces[0] != -1 && matchIndeces[1] != -1){
104-
// check for full match
105-
if(queryText == matches[matchIndeces[0]][matchIndeces[1]]){
106-
result = 0;
107-
}
108-
// does the found dict entry start with the search term
109-
else if(matches[matchIndeces[0]][matchIndeces[1]].startsWith(queryText)){
110-
result = 1;
111-
}
112-
// the query matches somwhere in the entry
113-
else {
114-
result = 2;
105+
for (var i = 0; i < allSearches.length; i++) {
106+
// check for full match
107+
if(allSearches[i] == matches[matchIndeces[0]][matchIndeces[1]]){
108+
result = 0 + i*allSearches.length;
109+
}
110+
// does the found dict entry start with the search term
111+
else if(matches[matchIndeces[0]][matchIndeces[1]].startsWith(allSearches[i])){
112+
result = 1 + i*allSearches.length;
113+
}
114+
// the query matches somwhere in the entry
115+
else {
116+
result = 2 + i*allSearches.length;
117+
}
118+
/// calculate the difference in length between the query and the result
119+
lenDiff = matches[matchIndeces[0]][matchIndeces[1]].length - allSearches[i].length;
115120
}
116-
/// calculate the difference in length between the query and the result
117-
lenDiff = matches[matchIndeces[0]][matchIndeces[1]].length - queryText.length;
118121
}
119122

120123
return Tuple3(result, lenDiff, matchIndeces[1]);

‎lib/entities/dictionary/dict_search_result.dart

+3-3
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,9 @@ class DictSearch with ChangeNotifier {
1212
String currentSearch = "";
1313

1414
/// a list of all search results
15-
List _searchResults = [];
16-
List get searchResults => _searchResults;
17-
set searchResults(List newResults){
15+
List<JMdict> _searchResults = [];
16+
List<JMdict> get searchResults => _searchResults;
17+
set searchResults(List<JMdict> newResults){
1818
_searchResults = newResults;
1919
notifyListeners();
2020
}

‎lib/entities/dictionary/dictionary_search.dart

+15-26
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import 'package:kana_kit/kana_kit.dart';
66
// Project imports:
77
import 'package:da_kanji_mobile/application/dictionary/dictionary_search.dart';
88
import 'package:da_kanji_mobile/entities/dictionary_filters/filter_options.dart';
9+
import 'package:tuple/tuple.dart';
910
import 'search_isolate.dart';
1011

1112
/// Class that spawns a number of isolates to search multi-processed in the
@@ -29,14 +30,12 @@ class DictionarySearch {
2930
/// Is a search currently running
3031
bool _isSearching = false;
3132
/// The last query that was blocked by a running search
32-
String? _lastBlockedQuery;
33+
/// Consists of <query, kana query, deconjugated query>
34+
Tuple3<String, String?, String?>? _lastBlockedQuery;
3335
/// Should the search be converted to hiragana
3436
bool convertToHiragana;
3537

36-
final KanaKit _kKitRomaji = const KanaKit();
37-
final KanaKit _kKitKanji = const KanaKit(
38-
config: KanaKitConfig(passRomaji: true, passKanji: true, upcaseKatakana: false)
39-
);
38+
4039

4140

4241
DictionarySearch(
@@ -58,18 +57,19 @@ class DictionarySearch {
5857
}
5958

6059
/// Queries the database and sorts the results using multiple isolates.
61-
Future<List<JMdict>?> query (String queryText) async {
60+
Future<List<JMdict>?> search(
61+
String query, String? queryKana, String? queryDeconjugated) async {
6262
_checkInitialized();
6363

6464
// do not search if a search is already running but remember the last blocked query
6565
if(_isSearching){
66-
_lastBlockedQuery = queryText;
66+
_lastBlockedQuery = Tuple3(query, queryKana, queryDeconjugated);
6767
return null;
6868
}
6969
_isSearching = true;
7070

7171
// check if the message contains wildcards and replace them appropriately
72-
String query = queryText.replaceAll(RegExp(r"\?|\﹖|\︖|\？"), "???");
72+
query = query.replaceAll(RegExp(r"\?|\﹖|\︖|\？"), "???");
7373

7474
// replace full-width chars with normal ones
7575
query = query.toHalfWidth();
@@ -78,24 +78,11 @@ class DictionarySearch {
7878
List<String> filters = getFilters(query);
7979
query = query.split(" ").where((e) => !e.startsWith("#")).join(" ");
8080

81-
// convert query to hiragana if it is Japanese
82-
if(_kKitKanji.isJapanese(query)) {
83-
query = _kKitKanji.toKana(query);
84-
}
85-
86-
// if romaji conversion setting is enabled, convert query to hiragana
87-
String? queryKana;
88-
if(convertToHiragana) {
89-
String t = _kKitRomaji.toHiragana(_kKitRomaji.toKana(query));
90-
// assure that the outcome is japanese
91-
if(_kKitRomaji.isJapanese(t)) queryKana = t;
92-
}
93-
9481
// search in `noIsolates` separte Isolates
9582
FutureGroup<List> searchGroup = FutureGroup();
9683
for (var i = 0; i < noIsolates; i++) {
9784
searchGroup.add(_searchIsolates[i].query(
98-
query, queryKana, filters
85+
query, queryKana, queryDeconjugated, filters
9986
));
10087
}
10188
searchGroup.close();
@@ -104,18 +91,20 @@ class DictionarySearch {
10491
final searchResult =
10592
List<JMdict>.from((await searchGroup.future).expand((e) => e));
10693
// sort and merge the results
107-
final sortResult = sortJmdictList(
108-
searchResult, query, queryKana, languages, convertToHiragana
94+
List<List<JMdict>> sortResult = sortJmdictList(
95+
searchResult, query, queryKana, queryDeconjugated, languages
10996
);
11097
var result = sortResult.expand((element) => element).toList();
11198
_isSearching = false;
11299

113100
// if one or more queries were made while this one was running, run the last
114101
// one
115102
if(_lastBlockedQuery != null){
116-
var t = _lastBlockedQuery;
103+
String t1 = _lastBlockedQuery!.item1;
104+
String? t2 = _lastBlockedQuery!.item2;
105+
String? t3 = _lastBlockedQuery!.item3;
117106
_lastBlockedQuery = null;
118-
result = (await this.query(t!)) ?? [];
107+
result = (await search(t1, t2, t3)) ?? [];
119108
_lastBlockedQuery = null;
120109
}
121110

‎lib/entities/dictionary/search_isolate.dart

+10-6
Original file line numberDiff line numberDiff line change
@@ -108,13 +108,15 @@ class DictionarySearchIsolate {
108108
}
109109

110110
/// Queries the dictionay inside an isolate
111-
Future<List> query(String query, String? queryKana, List<String> filters) async {
111+
Future<List> query(String query, String? queryKana, String? queryDeconjugated,
112+
List<String> filters) async {
113+
112114
_checkInitialized();
113115

114116
List result = [];
115117

116118
if(query != ""){
117-
isolateSendPort!.send(Tuple3(query, queryKana, filters));
119+
isolateSendPort!.send(Tuple4(query, queryKana, queryDeconjugated, filters));
118120
result = await events!.next;
119121
}
120122
else{
@@ -167,21 +169,23 @@ Future<void> _searchInIsar(SendPort p) async {
167169
break;
168170
}
169171

170-
if (message is Tuple3<String, String?, List<String>>) {
172+
if (message is Tuple4<String, String?, String?, List<String>>) {
171173
Stopwatch s = Stopwatch()..start();
172174

173175
String query = message.item1;
174176
String? queryKana = message.item2;
175-
List<String> filters = message.item3;
177+
String? queryDeconjugated = message.item3;
178+
List<String> filters = message.item4;
176179

177180
List<JMdict> searchResults =
178181
buildJMDictQuery(isar, idRangeStart, idRangeEnd, noIsolates,
179-
query, queryKana, filters, langs)
182+
query, queryKana, queryDeconjugated, filters, langs)
180183
.findAllSync();
184+
print(searchResults);
181185

182186
// Send the result to the main isolate.
183187
p.send(searchResults);
184-
debugPrint("Query: $query, QueryKana: $queryKana filters: $filters, results: ${searchResults.length}, time: ${s.elapsed}");
188+
debugPrint("Query: $query, QueryKana: $queryKana, QueryDeconjugated: $queryDeconjugated, filters: $filters, results: ${searchResults.length}, time: ${s.elapsed}");
185189
}
186190
}
187191

‎lib/repositories/dictionary/dictionary_search.dart

+34-20
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22
import 'package:database_builder/database_builder.dart';
33
import 'package:isar/isar.dart';
44

5+
6+
57
/// Builds a search query for the JMDict database in ISAR
68
///
79
/// Searches in the given `isar` for entries with an id between `idRangeStart`
@@ -11,17 +13,18 @@ import 'package:isar/isar.dart';
1113
/// * include ID in kanji / kana / meanings index to split load between isolates
1214
QueryBuilder<JMdict, JMdict, QAfterLimit> buildJMDictQuery(
1315
Isar isar, int idRangeStart, int idRangeEnd, int noIsolates,
14-
String query, String? kanaizedQuery, List<String> filters, List<String> langs)
16+
String query, String? queryKana, String? queryDeconjugated,
17+
List<String> filters, List<String> langs)
1518
{
1619

1720
// check if the search contains a wildcard
1821
bool containsWildcard = query.contains(RegExp(r"\?|\*"));
1922

2023
QueryBuilder<JMdict, JMdict, QFilterCondition> q;
2124
if(!containsWildcard) {
22-
q = normalQuery(isar, idRangeStart, idRangeEnd, query, kanaizedQuery);
25+
q = normalQuery(isar, idRangeStart, idRangeEnd, query, queryKana, queryDeconjugated);
2326
} else {
24-
q = wildcardQuery(isar, idRangeStart, idRangeEnd, query, kanaizedQuery);
27+
q = wildcardQuery(isar, idRangeStart, idRangeEnd, query, queryKana, queryDeconjugated);
2528
}
2629

2730
return q
@@ -44,18 +47,22 @@ QueryBuilder<JMdict, JMdict, QAfterLimit> buildJMDictQuery(
4447
// allow kanji / hiragana matches without wildcard
4548
q.optional(!containsWildcard, (q) =>
4649
q.group((q) =>
47-
q.kanjiIndexesElementStartsWith(kanaizedQuery ?? query)
50+
q.anyOf([queryKana, query, queryDeconjugated].nonNulls, (q, element)
51+
=> q.kanjiIndexesElementStartsWith(element))
4852
.or()
49-
.hiraganasElementStartsWith(kanaizedQuery ?? query)
53+
.anyOf([queryKana, query, queryDeconjugated].nonNulls, (q, element)
54+
=> q.hiraganasElementStartsWith(element))
5055
)
5156
)
5257
.or()
5358
// allow kanji / hiragana matches with wildcard
5459
.optional(containsWildcard, (q) =>
55-
q.group((q) =>
56-
q.kanjisElementMatches(kanaizedQuery ?? query)
60+
q.group((q) =>
61+
q.anyOf([queryKana, query, queryDeconjugated].nonNulls, (q, element)
62+
=> q.kanjisElementMatches(element))
5763
.or()
58-
.hiraganasElementMatches(kanaizedQuery ?? query)
64+
.anyOf([queryKana, query, queryDeconjugated].nonNulls, (q, element)
65+
=> q.hiraganasElementMatches(element))
5966
)
6067
)
6168
.or()
@@ -68,7 +75,7 @@ QueryBuilder<JMdict, JMdict, QAfterLimit> buildJMDictQuery(
6875
.and()
6976
.group((q) =>
7077
q
71-
.optional(!containsWildcard, (q) =>
78+
.optional(!containsWildcard, (q) =>
7279
q.meaningsElement((meaning) =>
7380
meaning.attributesElementStartsWith(query)
7481
)
@@ -90,32 +97,39 @@ QueryBuilder<JMdict, JMdict, QAfterLimit> buildJMDictQuery(
9097

9198

9299
QueryBuilder<JMdict, JMdict, QFilterCondition> normalQuery(
93-
Isar isar, int idRangeStart, int idRangeEnd, String query, String? kanaizedQuery){
100+
Isar isar, int idRangeStart, int idRangeEnd,
101+
String query, String? queryKana, String? queryDeconjugated){
94102

95103
return isar.jmdict.where()
96-
.kanjiIndexesElementStartsWith(kanaizedQuery ?? query)
97-
.or()
98-
.hiraganasElementStartsWith(kanaizedQuery ?? query)
99-
.or()
100-
.meaningsIndexesElementStartsWith(query)
104+
.anyOf([query, queryKana, queryDeconjugated].nonNulls, (q, element)
105+
=> q.kanjiIndexesElementStartsWith(element))
106+
.or()
107+
.anyOf([query, queryKana, queryDeconjugated].nonNulls, (q, element)
108+
=> q.hiraganasElementStartsWith(element))
109+
.or()
110+
.anyOf([query].nonNulls, (q, element)
111+
=> q.meaningsIndexesElementStartsWith(element))
101112
.filter()
102113
// limit this process to one chunk of size (entries.length / num_processes)
103114
.idBetween(idRangeStart, idRangeEnd)
104115
;
105116
}
106117

107118
QueryBuilder<JMdict, JMdict, QAfterFilterCondition> wildcardQuery(
108-
Isar isar, int idRangeStart, int idRangeEnd, String query, String? kanaizedQuery){
119+
Isar isar, int idRangeStart, int idRangeEnd,
120+
String query, String? queryKana, String? queryDeconjugated){
109121

110122
return isar.jmdict.where()
111123
.idBetween(idRangeStart, idRangeEnd)
112124
.filter()
113-
114-
.kanjisElementMatches(kanaizedQuery ?? query)
125+
.anyOf([query, queryKana, queryDeconjugated].nonNulls, (q, element)
126+
=> q.kanjisElementMatches(element))
115127
.or()
116-
.hiraganasElementMatches(kanaizedQuery ?? query)
128+
.anyOf([query, queryKana, queryDeconjugated].nonNulls, (q, element)
129+
=> q.hiraganasElementMatches(element))
117130
.or()
118-
.meaningsIndexesElementMatches(query)
131+
.anyOf([query].nonNulls, (q, element)
132+
=> q.meaningsIndexesElementMatches(element))
119133
;
120134

121135
}

0 commit comments

Comments
 (0)