Blacklight 8 compatibility (#12)

* update Blacklight to '~> 8.0' (fixes #9) * update Ruby to '>= 2.7' * update Rails to '>= 6.1', '< 8' * update default Solr to 8.11.4 * remove Travis builds, use Github Actions
boston-library · Nov 18, 2024 · dff88b8 · dff88b8
1 parent 0b80ab1
commit dff88b8
Show file tree

Hide file tree

Showing 16 changed files with 124 additions and 92 deletions.
diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml
@@ -0,0 +1,38 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+# This workflow will download a prebuilt Ruby version, install dependencies and run tests with Rake
+# For more information see: https://github.com/marketplace/actions/setup-ruby-jruby-and-truffleruby
+
+name: CI
+
+on: [push, pull_request]
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    name: test (ruby ${{ matrix.ruby }} / rails ${{ matrix.rails_version }})
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby-version: ["3.3"]
+        rails_version: ["7.0.8.4", "7.1.3.4", "7.2.1.2"]
+        include:
+          - ruby-version: "3.1"
+            rails_version: "6.1.7.8"
+    env:
+      ENGINE_CART_RAILS_OPTIONS: "--skip-git --skip-listen --skip-spring --skip-keeps --skip-coffee --skip-test"
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby-version }}
+          bundler: "latest"
+      - name: Install dependencies
+        run: bundle install
+      - name: Run linter and tests
+        run: bundle exec rake ci
diff --git a/.solr_wrapper.yml b/.solr_wrapper.yml
@@ -2,7 +2,13 @@
 # you must first run 'rake engine_cart:generate' to create the test app
 # before running 'solr_wrapper' from project root
 # port: 8983
-version: 7.7.3
+
+# single-term autocomplete only works with Solr < 8
+# requires adding tokenizing-suggest-v1.0.1.jar to Solr's contrib directory
+# see README for details
+# version: 7.7.3
+
+version: 8.11.4
 collection:
   dir: ./.internal_test_app/solr/conf/
   name: blacklight-core
diff --git a/.travis.yml b/.travis.yml
diff --git a/Gemfile b/Gemfile
@@ -9,12 +9,11 @@ end
 # To use a debugger
 # gem 'byebug', group: [:development, :test]
 
-gem 'engine_cart'
 # BEGIN ENGINE_CART BLOCK
-# engine_cart: 1.2.0
-# engine_cart stanza: 0.10.0
+# engine_cart: 2.5.0
+# engine_cart stanza: 2.5.0
 # the below comes from engine_cart, a gem used to test this Rails engine gem in the context of a Rails app.
-file = File.expand_path('Gemfile', ENV['ENGINE_CART_DESTINATION'] || ENV['RAILS_ROOT'] || File.expand_path('.internal_test_app', File.dirname(__FILE__)))
+file = File.expand_path('Gemfile', ENV.fetch('ENGINE_CART_DESTINATION') { ENV.fetch('RAILS_ROOT') { File.expand_path('.internal_test_app', File.dirname(__FILE__)) } })
 if File.exist?(file)
   begin
     eval_gemfile file
@@ -24,26 +23,13 @@ if File.exist?(file)
   end
 else
   Bundler.ui.warn "[EngineCart] Unable to find test application dependencies in #{file}, using placeholder dependencies"
-
   if ENV['RAILS_VERSION']
     if ENV['RAILS_VERSION'] == 'edge'
       gem 'rails', github: 'rails/rails'
-      ENV['ENGINE_CART_RAILS_OPTIONS'] = '--edge --skip-turbolinks'
+      ENV['ENGINE_CART_RAILS_OPTIONS'] = '--edge'
     else
       gem 'rails', ENV['RAILS_VERSION']
     end
   end
-
-  case ENV['RAILS_VERSION']
-  when /^5.[12]/, /^6.0/
-    gem 'sass-rails', '~> 5.0'
-  when /^4.2/
-    gem 'responders', '~> 2.0'
-    gem 'sass-rails', '>= 5.0'
-    gem 'coffee-rails', '~> 4.1.0'
-    gem 'json', '~> 1.8'
-  when /^4.[01]/
-    gem 'sass-rails', '< 5.0'
-  end
 end
 # END ENGINE_CART BLOCK
diff --git a/README.md b/README.md
@@ -1,6 +1,6 @@
 # Blacklight IIIF Search
 
-[![Build Status](https://travis-ci.org/boston-library/blacklight_iiif_search.png?branch=master)](https://travis-ci.org/boston-library/blacklight_iiif_search) [![Coverage Status](https://coveralls.io/repos/github/boston-library/blacklight_iiif_search/badge.svg?branch=master)](https://coveralls.io/github/boston-library/blacklight_iiif_search?branch=master)
+![CI Workflow](https://github.com/boston-library/blacklight_iiif_search/actions/workflows/ruby.yml/badge.svg)
 
 A plugin that provides IIIF Content Search functionality for [Blacklight](https://github.com/projectblacklight/blacklight)-based applications.
 
@@ -12,10 +12,8 @@ By integrating the URL for this service into your IIIF Presentation manifests, c
 
 ## Prerequisites
 
-Currently highly opinionated towards Solr as the back-end search index.
-
 This plugin assumes:
-1. You have a working Blacklight application
+1. You have a working Blacklight application, with Solr as the search index
 2. You have items with full text (e.g. scanned books, newspapers, etc.) in your Solr index
 3. The text for these items is indexed in Solr
 4. Each page has its own Solr record, with the corresponding text in a discrete field 
@@ -25,10 +23,12 @@ This plugin assumes:
 
 Blacklight/Solr Version Compatibility:
 
-blacklight_iiif_search version | works with Blacklight | works with Solr
------------------------ | --------------------- | -----------
-2.0 | ~> 7.0 | 7.*
-1.0 | >= 6.3.0 to < 7.* | 7.*
+| blacklight_iiif_search version | works with Blacklight | tested with Solr |
+|--------------------------------|-----------------------|------------------|
+| 3.0                            | ~> 8.0                | >= 7.0 to < 9.*  |
+| 2.0                            | ~> 7.0                | 7.*              |
+| 1.0                            | >= 6.3.0 to < 7.*     | 7.*              |
+
 
 ## Installation
 
@@ -153,9 +153,14 @@ For IIIF Content Search autocomplete behavior, we want to limit the suggestions
 
 This is best set up as a separate `<searchComponent>` from any existing autocomplete/suggest functionality that may already be defined in your Solr configuration. The install generator will create a new `<searchComponent>` in solrconfig.xml and several field definitions in the schema.xml file to support the autocomplete behavior. You may need to customize these settings for your implementation.
 
-You also need to add the `tokenizing-suggest-v1.0.1.jar` library to your Solr install's `contrib` directory. This library is needed so that Solr will return single terms for autocomplete queries, rather than the entire full text field.
+In Solr 8., the autocomplete suggester service currently returns the entire field value, not a single term. Single-term autocomplete suggestions are possible with Solr 7.
+
+To enable single-term autocomplete in Solr 7, you also need to add the `tokenizing-suggest-v1.0.1.jar` library to your Solr install's `contrib` directory (see steps in Test Drive below). 
+This library is needed so that Solr will return single terms for autocomplete queries, rather than the entire full text field.
+
+(The `tokenizing-suggest-v1.0.1.jar` library is not currently compatible with Solr > 7. Further development needed, pull requests welcome!)
 
-_Note_: It's often helpful to test Solr directly to make sure autocomplete is working properly, this can be done like so:
+It's often helpful to test Solr directly to make sure autocomplete is working properly, this can be done like so:
 ```
 http://host:port/solr/[core_name]/iiif_suggest?wt=json&suggest.cfq=[parent_identifier]&q=[query_term]
 ```
@@ -171,26 +176,27 @@ $ rake engine_cart:generate
 ```
 $ solr_wrapper
 ```
-This will throw an error, since the Solr config will look for a library that doesn't exist yet.
-3. Copy the `tokenizing-suggest-v1.0.1.jar` library to Solr's `contrib` directory:
+
+3. If running Solr 7.*, copy the `tokenizing-suggest-v1.0.1.jar` library to Solr's `contrib` directory:
 ```
+# first, stop solr_wrapper (Ctrl-C)
 $ cp ./lib/generators/blacklight_iiif_search/templates/solr/lib/tokenizing-suggest-v1.0.1.jar /path/to/solr/contrib
-```
-4. Start up Solr again (run from same new terminal window):
-```
+# then uncomment the 'tokenizing-suggest' lines in .internal_test_app/solr/conf/solrconfig.xml
+# restart solr_wrapper
 $ solr_wrapper
 ```
-5. Index sample documents into Solr (run from `./.internal_test_app`):
+
+4. Index sample documents into Solr (run from `./.internal_test_app`):
 ```
 $ RAILS_ENV=test rake blacklight_iiif_search:index:seed
 ```
-6. Start up the Rails server (run from `./.internal_test_app`):
+5. Start up the Rails server (run from `./.internal_test_app`):
 ```
 $ rails s
 ```
-7. In a browser, go to: `http://127.0.0.1:3000`. You should see the default Blacklight home page.
-8. Test a sample search: `http://127.0.0.1:3000/catalog/7s75dn48d/iiif_search?q=sugar`
-9. Test a sample autocomplete request: `http://127.0.0.1:3000/catalog/7s75dn48d/iiif_suggest?q=be`
+6. In a browser, go to: `http://127.0.0.1:3000`. You should see the default Blacklight home page.
+7. Test a sample search: `http://127.0.0.1:3000/catalog/7s75dn48d/iiif_search?q=sugar`
+8. Test a sample autocomplete request: `http://127.0.0.1:3000/catalog/7s75dn48d/iiif_suggest?q=be`
 
 To see how search snippets work, change the value of the `full_text_field` config to `alternative_title_tsim` in `./.internal_test_app/app/controllers/catalog_controller.rb`, and restart the Rails server.
 
@@ -205,7 +211,7 @@ After cloning the repository, and running `bundle install`, run `rake ci` from t
 
 ## Credits
 
-This project was developed as part of the [Newspapers in Samvera](https://www.imls.gov/grants/awarded/lg-70-17-0043-17) grant. Thanks to the Institute of Museum and Library Services for their support.
+This project was originally developed as part of the [Newspapers in Samvera](https://www.imls.gov/grants/awarded/lg-70-17-0043-17) grant. Thanks to the Institute of Museum and Library Services for their support.
 
 Inspiration for this code was drawn from Stanford University Digital Library's [content_search](https://github.com/sul-dlss/content_search) and NCSU Libraries' [ocracoke](https://github.com/NCSU-Libraries/ocracoke).
 

diff --git a/Rakefile b/Rakefile
@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'rubygems'
+require 'rails'
 begin
   require 'bundler/setup'
 rescue LoadError
@@ -36,8 +38,11 @@ RuboCop::RakeTask.new(:rubocop)
 desc 'Run test suite'
 task ci: [:rubocop, 'engine_cart:generate'] do
   SolrWrapper.wrap do |solr|
-    FileUtils.cp File.join(__dir__, 'lib', 'generators', 'blacklight_iiif_search', 'templates', 'solr', 'lib', 'tokenizing-suggest-v1.0.1.jar'),
-                 File.join(solr.instance_dir, 'contrib')
+    # single-term autocomplete via tokenizing-suggest currently requires Solr 7.*, see README.md for more info
+    if solr.version.to_f < 8
+      FileUtils.cp(File.join(__dir__, 'lib', 'generators', 'blacklight_iiif_search', 'templates', 'solr', 'lib', 'tokenizing-suggest-v1.0.1.jar'),
+                   File.join(solr.instance_dir, 'contrib'))
+    end
     solr.with_collection do
       within_test_app do
         system 'RAILS_ENV=test rake blacklight_iiif_search:index:seed'

diff --git a/app/controllers/concerns/blacklight_iiif_search/controller.rb b/app/controllers/concerns/blacklight_iiif_search/controller.rb
@@ -11,23 +11,19 @@ module Controller
     end
 
     def iiif_search
-      _parent_response, @parent_document = search_service.fetch(params[:solr_document_id])
-      iiif_search = IiifSearch.new(iiif_search_params, iiif_search_config,
-                                   @parent_document)
+      @parent_document = search_service.fetch(params[:solr_document_id])
+      iiif_search = IiifSearch.new(iiif_search_params, iiif_search_config, @parent_document)
       iiif_search_service = search_service_class.new(config: blacklight_config,
                                                      user_params: iiif_search.solr_params)
-      @response, _document_list = iiif_search_service.search_results
-      iiif_search_response = IiifSearchResponse.new(@response,
-                                                    @parent_document,
-                                                    self)
-      render json: iiif_search_response.annotation_list,
-             content_type: 'application/json'
+      @response = iiif_search_service.search_results
+      iiif_search_response = IiifSearchResponse.new(@response, @parent_document, self)
+
+      render json: iiif_search_response.annotation_list, content_type: 'application/json'
     end
 
     def iiif_suggest
       suggest_search = IiifSuggestSearch.new(params, Blacklight.default_index, self)
-      render json: suggest_search.response,
-             content_type: 'application/json'
+      render json: suggest_search.response, content_type: 'application/json'
     end
 
     def iiif_search_config

diff --git a/blacklight_iiif_search.gemspec b/blacklight_iiif_search.gemspec
@@ -22,14 +22,16 @@ Gem::Specification.new do |s|
   s.metadata["homepage_uri"] = s.homepage
   s.metadata["source_code_uri"] = s.homepage
 
-  s.add_dependency 'rails', '>= 6', '< 7.3'
-  s.add_dependency 'blacklight', '~> 7.0'
+  s.required_ruby_version = '>= 2.7'
+
+  s.add_dependency 'rails', '>= 6.1', '< 8'
+  s.add_dependency 'blacklight', '~> 8.0'
   s.add_dependency 'iiif-presentation'
   s.add_dependency 'ffi', '~> 1.16.3' # https://github.com/ffi/ffi/issues/1103
 
-  s.add_development_dependency 'rspec-rails', '~> 3.0'
+  s.add_development_dependency 'rspec-rails', '>= 6.1', '< 8'
   s.add_development_dependency 'solr_wrapper', '~> 4.0'
   s.add_development_dependency 'engine_cart', '~> 2.1'
   s.add_development_dependency 'byebug'
-  s.add_development_dependency 'bixby', '~> 4.0.0'
+  s.add_development_dependency 'bixby', '~> 5.0'
 end
diff --git a/lib/generators/blacklight_iiif_search/install_generator.rb b/lib/generators/blacklight_iiif_search/install_generator.rb
@@ -18,13 +18,14 @@ class InstallGenerator < Rails::Generators::Base
        1. Injects behavior into CatalogController
        2. Adds a SearchBuilder to ./app/models
        3. Adds BlacklightIiifSearch routes to ./config/routes.rb
-       4. Modifies solrconfig.xml to support contextual autocomplete functionality
+       4. Modifies Solr schema.xml and solrconfig.xml to support contextual autocomplete functionality
       Thanks for installing Blacklight IIIF Search!
     EOS
 
     def verify_blacklight_installed
       return if IO.read('app/controllers/application_controller.rb').include?('include Blacklight::Controller')
       say_status('info', 'BLACKLIGHT NOT INSTALLED; GENERATING BLACKLIGHT', :blue)
+
       generate 'blacklight:install'
     end
 
@@ -45,7 +46,7 @@ def add_solr_config
     end
 
     def bundle_install
-      Bundler.with_clean_env do
+      Bundler.with_unbundled_env do
         run 'bundle install'
       end
     end

diff --git a/lib/generators/blacklight_iiif_search/routes_generator.rb b/lib/generators/blacklight_iiif_search/routes_generator.rb
@@ -10,7 +10,7 @@ class RoutesGenerator < Rails::Generators::Base
     desc 'This generator makes the following changes to your app:
         1. Injects route declarations into your routes.rb'
 
-    # Add CommonwealthVlrEngine to the routes
+    # Add BlacklightIiifSearch to the routes
     def inject_iiif_search_routes
       return if IO.read('config/routes.rb').include?('BlacklightIiifSearch::Routes')
 

diff --git a/lib/generators/blacklight_iiif_search/solr_generator.rb b/lib/generators/blacklight_iiif_search/solr_generator.rb
@@ -16,18 +16,22 @@ def inject_iiif_suggest_solrconfig
       marker = '</config>'
       inject_into_file 'solr/conf/solrconfig.xml', before: marker do
         "  <!-- BEGIN Blacklight IIIF Search autocomplete config -->
-  <!-- solr-tokenizing_suggester is necessary to return single terms from the suggester -->
-  <lib dir=\"${solr.install.dir:../../../..}/contrib\" regex=\"tokenizing-suggest-v1.0.1.jar\" />\n
+  <!-- tokenizing-suggest is necessary to return only single terms from the suggester -->
+  <!-- to use this, copy the blacklight_iiif_search/lib/generators/blacklight_iiif_search/templates/solr/lib/tokenizing-suggest-v1.0.1.jar -->
+  <!-- to your Solr install's contrib directory, and uncomment the lines below. -->
+  <!-- HOWEVER, this only works in Solr 7.*, see blacklight_iiif_search README for more info. -->
+  <!-- <lib dir=\"${solr.install.dir:../../../..}/contrib\" regex=\"tokenizing-suggest-v1.0.1.jar\" /> -->
   <searchComponent name=\"iiif_suggest\" class=\"solr.SuggestComponent\">
     <lst name=\"suggester\">
       <str name=\"name\">iiifSuggester</str>
-      <str name=\"lookupImpl\">edu.stanford.dlss.search.suggest.analyzing.TokenizingLookupFactory</str>
+      <str name=\"lookupImpl\">AnalyzingInfixLookupFactory</str>
       <str name=\"dictionaryImpl\">DocumentDictionaryFactory</str>
       <str name=\"suggestAnalyzerFieldType\">textSuggest</str>
-      <str name=\"suggestTokenizingAnalyzerFieldType\">textSuggestTokenizer</str>
       <str name=\"contextField\">is_page_of_ssi</str>
       <str name=\"buildOnCommit\">true</str>
       <str name=\"field\">iiif_suggest</str>
+      <!-- <str name=\"lookupImpl\">edu.stanford.dlss.search.suggest.analyzing.TokenizingLookupFactory</str> -->
+      <!-- <str name=\"suggestTokenizingAnalyzerFieldType\">textSuggestTokenizer</str> -->
     </lst>
   </searchComponent>\n
   <requestHandler name=\"/iiif_suggest\" class=\"solr.SearchHandler\" startup=\"lazy\">
@@ -51,6 +55,8 @@ def inject_iiif_suggest_schema
       field_type_marker = '</types>'
       inject_into_file filepath, before: field_type_marker do
         "\n    <!-- BEGIN Blacklight IIIF Search autocomplete config -->
+    <!-- textSuggestTokenizer is only used by edu.stanford.dlss.search.suggest.analyzing.TokenizingLookupFactory -->
+    <!-- this fieldType can be ignored if using Solr > 7, see blacklight_iiif_search README for more info. -->
     <fieldType name=\"textSuggestTokenizer\" class=\"solr.TextField\" positionIncrementGap=\"100\">
       <analyzer>
         <tokenizer class=\"solr.WhitespaceTokenizerFactory\"/>
@@ -79,6 +85,15 @@ def inject_iiif_suggest_schema
   <copyField source=\"all_text_timv\" dest=\"iiif_suggest\"/>
   <!-- END Blacklight IIIF Search autocomplete config -->\n\n"
       end
+
+      textsuggest_marker = /<fieldType[\s]*name="textSuggest"[^>]*>[\s]*<analyzer>[\s]*<tokenizer class="solr.KeywordTokenizerFactory"\/>/
+      gsub_file(filepath, textsuggest_marker) do
+        "<fieldType name=\"textSuggest\" class=\"solr.TextField\" positionIncrementGap=\"100\">
+      <analyzer>
+        <!-- Blacklight IIIF Search autocomplete suggester config requires StandardTokenizerFactory -->
+        <!-- <tokenizer class=\"solr.KeywordTokenizerFactory\"/> -->
+        <tokenizer class=\"solr.StandardTokenizerFactory\"/>"
+      end
     end
 
     def copy_suggester_library

diff --git a/lib/generators/blacklight_iiif_search/templates/solr/lib/solr-tokenizing_suggester-7.x.jar b/lib/generators/blacklight_iiif_search/templates/solr/lib/solr-tokenizing_suggester-7.x.jar
diff --git a/spec/iiif_search_shared.rb b/spec/iiif_search_shared.rb
@@ -16,8 +16,8 @@
   let(:suggest_search_params) do
     { q: suggest_query_term, solr_document_id: parent_id, date: 'foo' }
   end
-  let(:parent_document) { controller.search_service.fetch(parent_id)[1] }
-  let(:page_document) { controller.search_service.fetch(page_id)[1] }
+  let(:parent_document) { controller.search_service.fetch(parent_id) }
+  let(:page_document) { controller.search_service.fetch(page_id) }
   let(:repository) { Blacklight::Solr::Repository.new(blacklight_config) }
 
   let(:iiif_search) do