Add new fields to the extraction definition to determine if a file ne…

…eds to have its text extracted

.gitignore

@@ -36,6 +36,7 @@
 
 # Ignore master key for decrypting credentials and more.
 /config/master.key
+/config/credentials.yml.enc
 
 /app/assets/builds/*
 !/app/assets/builds/.keep

app/models/extraction_definition.rb

@@ -3,7 +3,7 @@
 # Used to store the information for running an extraction
 #
 class ExtractionDefinition < ApplicationRecord
-  FORMATS = %w[JSON XML HTML PDF].freeze
+  FORMATS = %w[JSON XML HTML].freeze
 
   # The destination is used for Enrichment Extractions
   # To know where to pull the records that are to be enriched from
@@ -36,6 +36,7 @@ class ExtractionDefinition < ApplicationRecord
 
   validates :name, uniqueness: true
   validates :split_selector, presence: true, if: :split?
+  validates :extracted_file_format, presence: true, if: :extract_text_from_file?
 
   validates :throttle, numericality: { only_integer: true, greater_than_or_equal_to: 0, less_than_or_equal_to: 60_000 }
 

db/migrate/20240305235829_add_extract_text_from_file_to_extraction_definition.rb

@@ -0,0 +1,6 @@
+class AddExtractTextFromFileToExtractionDefinition < ActiveRecord::Migration[7.0]
+  def change
+    add_column :extraction_definitions, :extract_text_from_file, :boolean, default: false, null: false
+    add_column :extraction_definitions, :extracted_file_format, :string
+  end
+end

spec/models/extraction_definition_spec.rb

@@ -42,6 +42,12 @@
 
       it { is_expected.to validate_presence_of(:split_selector).with_message("can't be blank") } 
     end
+
+    context 'when the extraction definition needs to extract the text from a file' do
+      subject! { build(:extraction_definition, pipeline: pipeline1, extract_text_from_file: true) }
+
+      it { is_expected.to validate_presence_of(:extracted_file_format).with_message("can't be blank") } 
+    end
   end
 
   describe '#validation numericality' do