Log overlay in task

BrainPortal/app/models/cluster_task.rb

@@ -2383,10 +2383,15 @@ def singularity_commands(command_script)
     # Some of them might be patterns (e.g. /a/b/data*.squashfs) that need to
     # be resolved locally.
     # This will be a string "--overlay=path:ro --overlay=path:ro" etc.
-    overlay_paths = self.tool_config.singularity_overlays_full_paths.map do |path|
-      paths = Dir.glob(path) # checks on the local file system
-      cb_error "Can't find any overlays matching '#{path}'" if paths.blank?
-      paths
+    overlay_paths = self.tool_config.singularity_overlays_full_paths.map do |knd, id_or_name, paths|
+      self.addlog("Processing container overlay spec #{knd} #{id_or_name}:")
+      paths.map do |path|
+        self.addlog(" - checking #{path} ... ")
+        local_paths = Dir.glob(path) # assume no glob expression in overlay files
+        cb_error "Can't find any local file matching '#{path}'" if local_paths.blank?
+        self.addlog("   found local file(s) #{local_paths.join', '}")
+        local_paths
+      end
     end.flatten
     overlay_mounts = overlay_paths.inject("") do |sing_opts,path|
       "#{sing_opts} --overlay=#{path.bash_escape}:ro"
@@ -2557,7 +2562,9 @@ def local_dp_storage_paths #:nodoc:
 
   # Just invokes the same method on the task's ToolConfig.
   def ext3capture_basenames
-    self.tool_config.ext3capture_basenames
+    names = self.tool_config.ext3capture_basenames
+    self.addlog("Overlaying ext3 capturing basenames #{names}")
+    names
   end
 
   # This method creates an empty +filename+ with +size+ bytes

BrainPortal/app/models/tool_config.rb

@@ -357,7 +357,8 @@ def cbrain_task_class
   end
 
   # Returns an array of full paths to the Singularity overlay files that
-  # need to be mounted, as configured by the admin. Some of them might
+  # need to be mounted, along with explanation of their origin.
+  # Some of paths might
   # be patterns and will need to be resolved at run time. The dsl is
   #      # A file located on the cluster
   #        file:/path/to/file.squashfs
@@ -372,30 +373,30 @@ def singularity_overlays_full_paths
     specs.map do |knd, id_or_name|
 
       # Old style file spec (legacy, to be removed)
-      next knd if knd =~ /^\//  # FIXME delete it after successful migration
+      next ["Local File", knd, [knd]] if knd =~ /^\//  # FIXME delete it after successful migration
 
       case knd
       when 'dp'
         dp = DataProvider.where_id_or_name(id_or_name).first
         cb_error "Can't find DataProvider #{id_or_name} for fetching overlays" if ! dp
         dp_ovs = dp.singularity_overlays_full_paths rescue nil
         cb_error "DataProvider #{id_or_name} does not have any overlays configured." if dp_ovs.blank?
-        dp_ovs
+        ["Data Provider", id_or_name, dp_ovs]
       when 'file'
         cb_error "Provide absolute path for overlay file '#{id_or_name}'." if (Pathname.new id_or_name).relative?
-        id_or_name  # for local file, it is full file name (no ids)
+        ["local file", id_or_name, [id_or_name]]  # for local file, it is full file name (no ids)
       when 'userfile'
         # db registered file, note admin can access all files
         userfile = SingleFile.where(:id => id_or_name).last
         cb_error "Userfile with id '#{id_or_name}' for overlay fetching not found." if ! userfile
         userfile.sync_to_cache() rescue cb_error "Userfile with id '#{id_or_name}' for fetching overlay failed to synchronize."
-        userfile.cache_full_path()
+        ["registered file with id", id_or_name, [userfile.cache_full_path()]]
       when 'ext3capture'
-        [] # flatten will remove all that
+        nil  # handled separately
       else
         cb_error "Invalid '#{knd}:#{id_or_name}' overlay."
       end
-    end.flatten.uniq
+    end.uniq.compact
   end
 
   # Returns an array of the data providers that are