autocomplete and specs working with Solr 8

.solr_wrapper.yml

@@ -3,10 +3,12 @@
 # before running 'solr_wrapper' from project root
 # port: 8983
 
-# Solr > 7 doesn't work, tokenizing-suggest-v1.0.1.jar not compatible
-# version: 8.11.2
+# single-term autocomplete only works with Solr < 8
+# requires adding tokenizing-suggest-v1.0.1.jar to Solr's contrib directory
+# see README for details
+# version: 7.7.3
 
-version: 7.7.3
+version: 8.11.2
 collection:
   dir: ./.internal_test_app/solr/conf/
   name: blacklight-core

Gemfile

@@ -22,11 +22,6 @@ if File.exist?(file)
     Bundler.ui.warn e.message
   end
 else
-  # Bundler.ui.warn "ENGINE_CART_DESTINATION: #{ENV.fetch('ENGINE_CART_DESTINATION') { 'ENGINE_CART_DESTINATION NOT SET' }}"
-  # Bundler.ui.warn "RAILS_ROOT: #{ENV.fetch('RAILS_ROOT') { 'RAILS_ROOT NOT SET' }}"
-  # Bundler.ui.warn "FILE EXPAND PATH = #{File.expand_path('.internal_test_app', File.dirname(__FILE__))}"
-  # Bundler.ui.warn "CURRENT DIR: #{system 'pwd'}"
-  # Bundler.ui.warn "WHAT IS IN INTERNAL_TEST_APP: #{system('ls ' + file.split('/')[0..-2].join('/'))}"
   Bundler.ui.warn "[EngineCart] Unable to find test application dependencies in #{file}, using placeholder dependencies"
   if ENV['RAILS_VERSION']
     if ENV['RAILS_VERSION'] == 'edge'
@@ -38,5 +33,3 @@ else
   end
 end
 # END ENGINE_CART BLOCK
-
-# eval_gemfile File.expand_path("spec/test_app_templates/Gemfile.extra", File.dirname(__FILE__))

README.md

@@ -1,6 +1,6 @@
 # Blacklight IIIF Search
 
-[![Build Status](https://travis-ci.org/boston-library/blacklight_iiif_search.png?branch=master)](https://travis-ci.org/boston-library/blacklight_iiif_search) [![Coverage Status](https://coveralls.io/repos/github/boston-library/blacklight_iiif_search/badge.svg?branch=master)](https://coveralls.io/github/boston-library/blacklight_iiif_search?branch=master)
+![CI Workflow](https://github.com/boston-library/blacklight_iiif_search/actions/workflows/ruby.yml/badge.svg)
 
 A plugin that provides IIIF Content Search functionality for [Blacklight](https://github.com/projectblacklight/blacklight)-based applications.
 
@@ -25,10 +25,12 @@ This plugin assumes:
 
 Blacklight/Solr Version Compatibility:
 
-blacklight_iiif_search version | works with Blacklight | works with Solr
------------------------ | --------------------- | -----------
-2.0 | ~> 7.0 | 7.*
-1.0 | >= 6.3.0 to < 7.* | 7.*
+| blacklight_iiif_search version | works with Blacklight | works with Solr |
+|--------------------------------|-----------------------|-----------------|
+| 3.0                            | ~> 8.0                | >= 7.0 to < 9.* |
+| 2.0                            | ~> 7.0                | 7.*             |
+| 1.0                            | >= 6.3.0 to < 7.*     | 7.*             |
+
 
 ## Installation
 
@@ -90,6 +92,8 @@ Would return:
 http://host:port/catalog/abcd1234/iiif_suggest?q=blacklight
 ```
 
+_NOTE: In Solr 8.*, the autocomplete suggester service returns the entire field value, not a single term. Single-term autocomplete suggestions are possible with Solr 7.*._
+
 ## Implementation
 In order to successfully deploy this plugin, you'll most likely need to customize a few things to match how your Solr index and/or repository are set up.
 
@@ -153,7 +157,8 @@ For IIIF Content Search autocomplete behavior, we want to limit the suggestions
 
 This is best set up as a separate `<searchComponent>` from any existing autocomplete/suggest functionality that may already be defined in your Solr configuration. The install generator will create a new `<searchComponent>` in solrconfig.xml and several field definitions in the schema.xml file to support the autocomplete behavior. You may need to customize these settings for your implementation.
 
-You also need to add the `tokenizing-suggest-v1.0.1.jar` library to your Solr install's `contrib` directory. This library is needed so that Solr will return single terms for autocomplete queries, rather than the entire full text field.
+**For Solr 7.* only**, you also need to add the `tokenizing-suggest-v1.0.1.jar` library to your Solr install's `contrib` directory. 
+This library is needed so that Solr will return single terms for autocomplete queries, rather than the entire full text field.
 
 _Note_: It's often helpful to test Solr directly to make sure autocomplete is working properly, this can be done like so:
 ```
@@ -171,26 +176,27 @@ $ rake engine_cart:generate
 ```
 $ solr_wrapper
 ```
-This will throw an error, since the Solr config will look for a library that doesn't exist yet.
-3. Copy the `tokenizing-suggest-v1.0.1.jar` library to Solr's `contrib` directory:
+
+3. If running Solr 7.*, copy the `tokenizing-suggest-v1.0.1.jar` library to Solr's `contrib` directory:
 ```
+# first, stop solr_wrapper (Ctrl-C)
 $ cp ./lib/generators/blacklight_iiif_search/templates/solr/lib/tokenizing-suggest-v1.0.1.jar /path/to/solr/contrib
-```
-4. Start up Solr again (run from same new terminal window):
-```
+# then uncomment the 'tokenizing-suggest' lines in .internal_test_app/solr/conf/solrconfig.xml
+# restart solr_wrapper
 $ solr_wrapper
 ```
-5. Index sample documents into Solr (run from `./.internal_test_app`):
+
+4. Index sample documents into Solr (run from `./.internal_test_app`):
 ```
 $ RAILS_ENV=test rake blacklight_iiif_search:index:seed
 ```
-6. Start up the Rails server (run from `./.internal_test_app`):
+5. Start up the Rails server (run from `./.internal_test_app`):
 ```
 $ rails s
 ```
-7. In a browser, go to: `http://127.0.0.1:3000`. You should see the default Blacklight home page.
-8. Test a sample search: `http://127.0.0.1:3000/catalog/7s75dn48d/iiif_search?q=sugar`
-9. Test a sample autocomplete request: `http://127.0.0.1:3000/catalog/7s75dn48d/iiif_suggest?q=be`
+6. In a browser, go to: `http://127.0.0.1:3000`. You should see the default Blacklight home page.
+7. Test a sample search: `http://127.0.0.1:3000/catalog/7s75dn48d/iiif_search?q=sugar`
+8. Test a sample autocomplete request: `http://127.0.0.1:3000/catalog/7s75dn48d/iiif_suggest?q=be`
 
 To see how search snippets work, change the value of the `full_text_field` config to `alternative_title_tsim` in `./.internal_test_app/app/controllers/catalog_controller.rb`, and restart the Rails server.
 

Rakefile

@@ -36,10 +36,13 @@ require 'rubocop/rake_task'
 RuboCop::RakeTask.new(:rubocop)
 
 desc 'Run test suite'
-task ci: [:rubocop, 'engine_cart:generate'] do
+task ci: ['engine_cart:generate'] do
   SolrWrapper.wrap do |solr|
-    FileUtils.cp File.join(__dir__, 'lib', 'generators', 'blacklight_iiif_search', 'templates', 'solr', 'lib', 'tokenizing-suggest-v1.0.1.jar'),
-                 File.join(solr.instance_dir, 'contrib')
+    # single-term autocomplete via tokenizing-suggest currently requires Solr 7.*, see README.md for more info
+    if solr.version.to_f < 8
+      FileUtils.cp(File.join(__dir__, 'lib', 'generators', 'blacklight_iiif_search', 'templates', 'solr', 'lib', 'tokenizing-suggest-v1.0.1.jar'),
+                   File.join(solr.instance_dir, 'contrib'))
+    end
     solr.with_collection do
       within_test_app do
         system 'RAILS_ENV=test rake blacklight_iiif_search:index:seed'

lib/generators/blacklight_iiif_search/install_generator.rb

@@ -18,18 +18,14 @@ class InstallGenerator < Rails::Generators::Base
        1. Injects behavior into CatalogController
        2. Adds a SearchBuilder to ./app/models
        3. Adds BlacklightIiifSearch routes to ./config/routes.rb
-       4. Modifies solrconfig.xml to support contextual autocomplete functionality
+       4. Modifies Solr schema.xml and solrconfig.xml to support contextual autocomplete functionality
       Thanks for installing Blacklight IIIF Search!
     EOS
 
     def verify_blacklight_installed
       return if IO.read('app/controllers/application_controller.rb').include?('include Blacklight::Controller')
       say_status('info', 'BLACKLIGHT NOT INSTALLED; GENERATING BLACKLIGHT', :blue)
 
-      # say_status('info', 'TEST APP GEMFILE LOOKS LIKE:')
-      # system 'pwd'
-      # system 'cat ./Gemfile'
-      # system 'cat ./Gemfile.lock'
       generate 'blacklight:install'
     end
 
@@ -50,7 +46,7 @@ def add_solr_config
     end
 
     def bundle_install
-      Bundler.with_clean_env do
+      Bundler.with_unbundled_env do
         run 'bundle install'
       end
     end

lib/generators/blacklight_iiif_search/solr_generator.rb

@@ -16,18 +16,22 @@ def inject_iiif_suggest_solrconfig
       marker = '</config>'
       inject_into_file 'solr/conf/solrconfig.xml', before: marker do
         "  <!-- BEGIN Blacklight IIIF Search autocomplete config -->
-  <!-- solr-tokenizing_suggester is necessary to return single terms from the suggester -->
-  <lib dir=\"${solr.install.dir:../../../..}/contrib\" regex=\"tokenizing-suggest-v1.0.1.jar\" />\n
+  <!-- tokenizing-suggest is necessary to return only single terms from the suggester -->
+  <!-- to use this, copy the blacklight_iiif_search/lib/generators/blacklight_iiif_search/templates/solr/lib/tokenizing-suggest-v1.0.1.jar -->
+  <!-- to your Solr install's contrib directory, and uncomment the lines below. -->
+  <!-- HOWEVER, this only works in Solr 7.*, see blacklight_iiif_search README for more info. -->
+  <!-- <lib dir=\"${solr.install.dir:../../../..}/contrib\" regex=\"tokenizing-suggest-v1.0.1.jar\" /> -->
   <searchComponent name=\"iiif_suggest\" class=\"solr.SuggestComponent\">
     <lst name=\"suggester\">
       <str name=\"name\">iiifSuggester</str>
-      <str name=\"lookupImpl\">edu.stanford.dlss.search.suggest.analyzing.TokenizingLookupFactory</str>
+      <str name=\"lookupImpl\">AnalyzingInfixLookupFactory</str>
       <str name=\"dictionaryImpl\">DocumentDictionaryFactory</str>
       <str name=\"suggestAnalyzerFieldType\">textSuggest</str>
-      <str name=\"suggestTokenizingAnalyzerFieldType\">textSuggestTokenizer</str>
       <str name=\"contextField\">is_page_of_ssi</str>
       <str name=\"buildOnCommit\">true</str>
       <str name=\"field\">iiif_suggest</str>
+      <!-- <str name=\"lookupImpl\">edu.stanford.dlss.search.suggest.analyzing.TokenizingLookupFactory</str> -->
+      <!-- <str name=\"suggestTokenizingAnalyzerFieldType\">textSuggestTokenizer</str> -->
     </lst>
   </searchComponent>\n
   <requestHandler name=\"/iiif_suggest\" class=\"solr.SearchHandler\" startup=\"lazy\">
@@ -51,6 +55,8 @@ def inject_iiif_suggest_schema
       field_type_marker = '</types>'
       inject_into_file filepath, before: field_type_marker do
         "\n    <!-- BEGIN Blacklight IIIF Search autocomplete config -->
+    <!-- textSuggestTokenizer is only used by edu.stanford.dlss.search.suggest.analyzing.TokenizingLookupFactory -->
+    <!-- this fieldType can be ignored if using Solr > 7, see blacklight_iiif_search README for more info. -->
     <fieldType name=\"textSuggestTokenizer\" class=\"solr.TextField\" positionIncrementGap=\"100\">
       <analyzer>
         <tokenizer class=\"solr.WhitespaceTokenizerFactory\"/>
@@ -79,6 +85,15 @@ def inject_iiif_suggest_schema
   <copyField source=\"all_text_timv\" dest=\"iiif_suggest\"/>
   <!-- END Blacklight IIIF Search autocomplete config -->\n\n"
       end
+
+      textsuggest_marker = /<fieldType[\s]*name="textSuggest"[^>]*>[\s]*<analyzer>[\s]*<tokenizer class="solr.KeywordTokenizerFactory"\/>/
+      gsub_file(filepath, textsuggest_marker) do
+        "<fieldType name=\"textSuggest\" class=\"solr.TextField\" positionIncrementGap=\"100\">
+      <analyzer>
+        <!-- Blacklight IIIF Search autocomplete suggester config requires StandardTokenizerFactory -->
+        <!-- <tokenizer class=\"solr.KeywordTokenizerFactory\"/> -->
+        <tokenizer class=\"solr.StandardTokenizerFactory\"/>"
+      end
     end
 
     def copy_suggester_library

spec/models/blacklight_iiif_search/iiif_suggest_response_spec.rb

@@ -30,7 +30,7 @@
     it 'returns the expected data' do
       expect(response_terms.length).to eq(5)
       expect(response_terms.first[:url]).not_to be_falsey
-      expect(response_terms.first[:match].match(/\A#{suggest_query_term}/)).to be_truthy
+      expect(response_terms.first[:match].match(/#{suggest_query_term}/)).to be_truthy
     end
   end
 

spec/models/blacklight_iiif_search/iiif_suggest_search_spec.rb

@@ -32,7 +32,7 @@
     it 'returns the expected data' do
       terms = suggest_results['suggest'][blacklight_config.iiif_search[:suggester_name]][suggest_query_term]['suggestions']
       expect(terms.length).to eq(5)
-      expect(terms.first['term'].match(/\A#{suggest_query_term}/)).to be_truthy
+      expect(terms.first['term'].match(/#{suggest_query_term}/)).to be_truthy
     end
   end
 end