From 1cd25778c228cfbee4935713fbd8972542bcf371 Mon Sep 17 00:00:00 2001 From: MontrealSergiy Date: Tue, 7 May 2024 09:25:33 -0400 Subject: [PATCH 1/5] log overlay info into task #1387 --- BrainPortal/app/models/cluster_task.rb | 13 +++++++++---- BrainPortal/app/models/tool_config.rb | 12 ++++++------ 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/BrainPortal/app/models/cluster_task.rb b/BrainPortal/app/models/cluster_task.rb index e2623f0c2..d079f66f8 100644 --- a/BrainPortal/app/models/cluster_task.rb +++ b/BrainPortal/app/models/cluster_task.rb @@ -2383,10 +2383,15 @@ def singularity_commands(command_script) # Some of them might be patterns (e.g. /a/b/data*.squashfs) that need to # be resolved locally. # This will be a string "--overlay=path:ro --overlay=path:ro" etc. - overlay_paths = self.tool_config.singularity_overlays_full_paths.map do |path| - paths = Dir.glob(path) # checks on the local file system - cb_error "Can't find any overlays matching '#{path}'" if paths.blank? - paths + overlay_paths = self.tool_config.singularity_overlays_full_paths.map do |knd, id_or_name, paths| + self.addlog("Processing container overlay spec #{knd} #{id_or_name}:") + paths.map do |path| + self.addlog(" - checking #{path} ... ") + local_paths = Dir.glob(path) + cb_error "Can't find any local file matching '#{path}'" if local_paths.blank? + self.addlog (" found local files #{local_paths.join', '}") + local_paths + end end.flatten overlay_mounts = overlay_paths.inject("") do |sing_opts,path| "#{sing_opts} --overlay=#{path.bash_escape}:ro" diff --git a/BrainPortal/app/models/tool_config.rb b/BrainPortal/app/models/tool_config.rb index 8f85347cc..4f5388421 100644 --- a/BrainPortal/app/models/tool_config.rb +++ b/BrainPortal/app/models/tool_config.rb @@ -372,7 +372,7 @@ def singularity_overlays_full_paths specs.map do |knd, id_or_name| # Old style file spec (legacy, to be removed) - next knd if knd =~ /^\// # FIXME delete it after successful migration + next ["Local File", knd, knd] if knd =~ /^\// # FIXME delete it after successful migration case knd when 'dp' @@ -380,22 +380,22 @@ def singularity_overlays_full_paths cb_error "Can't find DataProvider #{id_or_name} for fetching overlays" if ! dp dp_ovs = dp.singularity_overlays_full_paths rescue nil cb_error "DataProvider #{id_or_name} does not have any overlays configured." if dp_ovs.blank? - dp_ovs + ["Data Provider", "#{id_or_name}", dp_ovs] when 'file' cb_error "Provide absolute path for overlay file '#{id_or_name}'." if (Pathname.new id_or_name).relative? - id_or_name # for local file, it is full file name (no ids) + ["local file", "#{id_or_name}", ["#{id_or_name}"]] # for local file, it is full file name (no ids) when 'userfile' # db registered file, note admin can access all files userfile = SingleFile.where(:id => id_or_name).last cb_error "Userfile with id '#{id_or_name}' for overlay fetching not found." if ! userfile userfile.sync_to_cache() rescue cb_error "Userfile with id '#{id_or_name}' for fetching overlay failed to synchronize." - userfile.cache_full_path() + ["registered file with id", "{id_or_name}", [userfile.cache_full_path()] when 'ext3capture' - [] # flatten will remove all that + nil # handled separately else cb_error "Invalid '#{knd}:#{id_or_name}' overlay." end - end.flatten.uniq + end.uniq.compact end # Returns an array of the data providers that are From be9000457e8c0de5da46e8d6c743ed609de2000d Mon Sep 17 00:00:00 2001 From: MontrealSergiy Date: Wed, 8 May 2024 17:59:03 -0400 Subject: [PATCH 2/5] log ext3 overlay info into task #1387 --- BrainPortal/app/models/cluster_task.rb | 4 +++- BrainPortal/app/models/tool_config.rb | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/BrainPortal/app/models/cluster_task.rb b/BrainPortal/app/models/cluster_task.rb index d079f66f8..0efee9500 100644 --- a/BrainPortal/app/models/cluster_task.rb +++ b/BrainPortal/app/models/cluster_task.rb @@ -2562,7 +2562,9 @@ def local_dp_storage_paths #:nodoc: # Just invokes the same method on the task's ToolConfig. def ext3capture_basenames - self.tool_config.ext3capture_basenames + names = self.tool_config.ext3capture_basenames + self.addlog("Overlaying ext3 capturing basenames #{names}") + names end # This method creates an empty +filename+ with +size+ bytes diff --git a/BrainPortal/app/models/tool_config.rb b/BrainPortal/app/models/tool_config.rb index 4f5388421..98c3503d2 100644 --- a/BrainPortal/app/models/tool_config.rb +++ b/BrainPortal/app/models/tool_config.rb @@ -357,7 +357,8 @@ def cbrain_task_class end # Returns an array of full paths to the Singularity overlay files that - # need to be mounted, as configured by the admin. Some of them might + # need to be mounted, along with explanation of their origin. + # Some of paths might # be patterns and will need to be resolved at run time. The dsl is # # A file located on the cluster # file:/path/to/file.squashfs @@ -389,7 +390,7 @@ def singularity_overlays_full_paths userfile = SingleFile.where(:id => id_or_name).last cb_error "Userfile with id '#{id_or_name}' for overlay fetching not found." if ! userfile userfile.sync_to_cache() rescue cb_error "Userfile with id '#{id_or_name}' for fetching overlay failed to synchronize." - ["registered file with id", "{id_or_name}", [userfile.cache_full_path()] + ["registered file with id", "{id_or_name}", userfile.cache_full_path()] when 'ext3capture' nil # handled separately else From 98581959433e410b62a55a51cba1384de06f1e28 Mon Sep 17 00:00:00 2001 From: MontrealSergiy Date: Mon, 13 May 2024 19:01:24 -0400 Subject: [PATCH 3/5] bugfix log ext3 overlay info into task #1387 --- BrainPortal/app/models/tool_config.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/BrainPortal/app/models/tool_config.rb b/BrainPortal/app/models/tool_config.rb index 98c3503d2..353218f44 100644 --- a/BrainPortal/app/models/tool_config.rb +++ b/BrainPortal/app/models/tool_config.rb @@ -373,7 +373,7 @@ def singularity_overlays_full_paths specs.map do |knd, id_or_name| # Old style file spec (legacy, to be removed) - next ["Local File", knd, knd] if knd =~ /^\// # FIXME delete it after successful migration + next ["Local File", knd, [knd]] if knd =~ /^\// # FIXME delete it after successful migration case knd when 'dp' @@ -390,7 +390,7 @@ def singularity_overlays_full_paths userfile = SingleFile.where(:id => id_or_name).last cb_error "Userfile with id '#{id_or_name}' for overlay fetching not found." if ! userfile userfile.sync_to_cache() rescue cb_error "Userfile with id '#{id_or_name}' for fetching overlay failed to synchronize." - ["registered file with id", "{id_or_name}", userfile.cache_full_path()] + ["registered file with id", "{id_or_name}", [userfile.cache_full_path()]] when 'ext3capture' nil # handled separately else From 26aeab7923b15d09031c9537cb0a1da80934291b Mon Sep 17 00:00:00 2001 From: MontrealSergiy Date: Mon, 13 May 2024 23:01:08 -0400 Subject: [PATCH 4/5] improvement log ext3 overlay info into task #1387 --- BrainPortal/app/models/tool_config.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/BrainPortal/app/models/tool_config.rb b/BrainPortal/app/models/tool_config.rb index 353218f44..f67b8270b 100644 --- a/BrainPortal/app/models/tool_config.rb +++ b/BrainPortal/app/models/tool_config.rb @@ -381,16 +381,16 @@ def singularity_overlays_full_paths cb_error "Can't find DataProvider #{id_or_name} for fetching overlays" if ! dp dp_ovs = dp.singularity_overlays_full_paths rescue nil cb_error "DataProvider #{id_or_name} does not have any overlays configured." if dp_ovs.blank? - ["Data Provider", "#{id_or_name}", dp_ovs] + ["Data Provider", id_or_name, dp_ovs] when 'file' cb_error "Provide absolute path for overlay file '#{id_or_name}'." if (Pathname.new id_or_name).relative? - ["local file", "#{id_or_name}", ["#{id_or_name}"]] # for local file, it is full file name (no ids) + ["local file", id_or_name, [id_or_name]] # for local file, it is full file name (no ids) when 'userfile' # db registered file, note admin can access all files userfile = SingleFile.where(:id => id_or_name).last cb_error "Userfile with id '#{id_or_name}' for overlay fetching not found." if ! userfile userfile.sync_to_cache() rescue cb_error "Userfile with id '#{id_or_name}' for fetching overlay failed to synchronize." - ["registered file with id", "{id_or_name}", [userfile.cache_full_path()]] + ["registered file with id", id_or_name, [userfile.cache_full_path()]] when 'ext3capture' nil # handled separately else From ed39d51644931e752ca650d072c5061d5de69e78 Mon Sep 17 00:00:00 2001 From: MontrealSergiy Date: Mon, 13 May 2024 23:05:24 -0400 Subject: [PATCH 5/5] improvement log ext3 overlay info into task #1387 --- BrainPortal/app/models/cluster_task.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/BrainPortal/app/models/cluster_task.rb b/BrainPortal/app/models/cluster_task.rb index 0efee9500..92be5c3d6 100644 --- a/BrainPortal/app/models/cluster_task.rb +++ b/BrainPortal/app/models/cluster_task.rb @@ -2387,9 +2387,9 @@ def singularity_commands(command_script) self.addlog("Processing container overlay spec #{knd} #{id_or_name}:") paths.map do |path| self.addlog(" - checking #{path} ... ") - local_paths = Dir.glob(path) + local_paths = Dir.glob(path) # assume no glob expression in overlay files cb_error "Can't find any local file matching '#{path}'" if local_paths.blank? - self.addlog (" found local files #{local_paths.join', '}") + self.addlog(" found local file(s) #{local_paths.join', '}") local_paths end end.flatten