Development checkpoint.

Author: olsonanl

AppService.spec

@@ -70,7 +70,7 @@ module AppService
     typedef structure {
         task_id parent_id;
 	workspace_id workspace;
-    } start_params;
+    } StartParams;
     funcdef start_app2(app_id, task_parameters params, StartParams start_params)
 	returns (Task task);
 

app_specs/Date.json

@@ -1,6 +1,6 @@
 {
 	"id":"Date",
-	"script":"App-Date_bob",
+	"script":"App-Date",
 	"label":"Date",
 	"description":"Returns the current date and time.",
 	"parameters":[

app_specs/Sleep.json

@@ -12,5 +12,7 @@
 		"desc": "Time to sleep, in seconds.",
 		"type": "int"
 	    }
-	]
+	],
+	"default_ram": "1M",
+	"default_cpu": 1
 }

lib/Bio/KBase/AppService/AppScript.pm

@@ -18,20 +18,24 @@ use LWP::UserAgent;
 use REST::Client;
 use Bio::KBase::AppService::AppConfig ':all';
 
+use Getopt::Long::Descriptive;
 use base 'Class::Accessor';
 
 use Data::Dumper;
 
-__PACKAGE__->mk_accessors(qw(callback donot_create_job_result donot_create_result_folder
+__PACKAGE__->mk_accessors(qw(execute_callback preflight_callback donot_create_job_result donot_create_result_folder
 			     workspace_url workspace params app_definition result_folder
+			     app_def params proc_params stdout_file stderr_file
+			     hostname json
 			     task_id app_service_url));
 
 sub new
 {
-    my($class, $callback) = @_;
+    my($class, $execute_callback, $preflight_callback) = @_;
 
     my $self = {
-	callback => $callback,
+	execute_callback => $execute_callback,
+	preflight_callback => $preflight_callback,
     };
     return bless $self, $class;
 }
@@ -48,32 +52,28 @@ sub run
 {
     my($self, $args) = @_;
 
-    #
-    # Hack to finding task id.
-    #
-    my $host = `hostname -f`;
-    $host = `hostname` if !$host;
-    chomp $host;
+    $self->set_task_id();
 
-    my $task_id = 'TBD';
-    if ($ENV{AWE_TASK_ID})
-    {
-	$task_id = $ENV{AWE_TASK_ID};
-    }
-    elsif ($ENV{PWD} =~ /([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})_\d+_\d+$/i)
-    {
-	$task_id = $1;
-    }
-    else
+    my $opt;
+    do {
+	local @ARGV = @$args;
+	($opt, my $usage) = describe_options("%c %o app-service-url app-definition.json param-values.json [stdout-file stderr-file]",
+					     ["preflight", "Run the app in preflight mode. Print a JSON object representing the expected runtime, requested CPU count, and memory use for this application invocation."],
+					     ["help|h", "Show this help message."]);
+	print($usage->text), exit(0) if $opt->help;
+	die($usage->text) unless @ARGV == 3 or @ARGV == 5;
+	
+	my $appserv_url = shift @ARGV;
+	$self->app_service_url($appserv_url);
+	$args = [@ARGV];
+    };
+
+    $self->process_parameters($args);
+
+    if ($opt->preflight)
     {
-	$task_id = "UNK-$host-$$";
+	return $self->run_preflight();
     }
-    $self->task_id($task_id);
-
-    @$args == 3 or @$args == 5 or die "Usage: $0 app-service-url app-definition.json param-values.json [stdout-file stderr-file]\n";
-    
-    my $appserv_url = shift @$args;
-    $self->app_service_url($appserv_url);
 
     #
     # If we are running at the terminal, do not set up this infrastructure.
@@ -87,7 +87,7 @@ sub run
 
     my $ua = LWP::UserAgent->new();
     my $rest = REST::Client->new();
-    $rest->setHost("$appserv_url/$task_id");
+    $rest->setHost($self->appservice_url . "/" . $self->task_id);
     $self->{rest} = $rest;
 
     my $sel = IO::Select->new();
@@ -110,7 +110,7 @@ sub run
     }
 
     $self->write_block("pid", $pid);
-    $self->write_block("hostname", $host);
+    $self->write_block("hostname", $self->hostname);
 
     $stdout_pipe->reader();
     $stderr_pipe->reader();
@@ -166,17 +166,59 @@ sub run
     return $rc;
 }
 
+sub run_preflight
+{
+    my($self) = @_;
+
+    return unless $self->preflight_callback();
+
+    $self->preflight_callback()->($self, $self->app_def, $self->params, $self->proc_params);
+}
+
+sub set_task_id
+{
+    my($self) = @_;
+
+    #
+    # Hack to finding task id.
+    #
+    my $host = `hostname -f`;
+    $host = `hostname` if !$host;
+    chomp $host;
+    $self->hostname($host);
+
+    my $task_id = 'TBD';
+    if ($ENV{AWE_TASK_ID})
+    {
+	$task_id = $ENV{AWE_TASK_ID};
+    }
+    elsif ($ENV{SLURM_JOB_ID})
+    {
+	$task_id = $ENV{SLURM_JOB_ID};
+    }
+    elsif ($ENV{PWD} =~ /([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})_\d+_\d+$/i)
+    {
+	$task_id = $1;
+    }
+    else
+    {
+	$task_id = "UNK-$host-$$";
+    }
+    $self->task_id($task_id);
+}
+
 sub write_block
 {
     my($self, $path, $data) = @_;
     $self->{rest}->POST($path, $data);
 }
 
-sub subproc_run
+sub process_parameters
 {
     my($self, $args) = @_;
 
     my $json = JSON::XS->new->pretty(1)->relaxed(1);
+    $self->json($json);
 
     my $app_def_file = shift @$args;
     my $params_file = shift @$args;
@@ -185,8 +227,31 @@ sub subproc_run
     my $stderr_file = shift @$args;
 
     my $app_def = $json->decode(scalar read_file($app_def_file));
-    my $params =  $json->decode(scalar read_file($params_file));
 
+    my $params_txt;
+    my $params_from_cmdline;
+    my $file_error;
+    if (open(my $fh, "<", $params_file))
+    {
+	$params_txt = read_file($fh);
+    }
+    else
+    {
+	$file_error = $!;
+	# for immediate-mode testing, allow specification of params as inline text.
+	$params_from_cmdline = 1;
+	$params_txt = $params_file;
+    }
+
+    my $params = eval { $json->decode($params_txt); };
+    if (!$params)
+    {
+	die "Error reading or parsing params (file error $!; parse error $@)";
+    }
+    if (ref($params) ne 'HASH')
+    {
+	die "Invalid parameters (must be JSON object)";
+    }
     #
     # Preprocess parameters to create hash of named parameters, looking for
     # missing required values and filling in defaults.
@@ -224,68 +289,63 @@ sub subproc_run
     {
 	die "Errors found in parameter processing:\n    " . join("\n    ", @errors), "\n";
     }
+
+    $self->app_def($app_def);
+    $self->params($params);
+    $self->proc_params(\%proc_param);
+    $self->stdout_file($stdout_file);
+    $self->stderr_file($stderr_file);
 
+}
+
+sub subproc_run
+{
+    my($self, $args) = @_;
+    
     my $ws = Bio::P3::Workspace::WorkspaceClientExt->new($self->workspace_url);
     $self->{workspace} = $ws;
-    $self->{params} = \%proc_param;
-    $self->{app_definition} = $app_def; 
 
     if (!defined($self->donot_create_result_folder()) || $self->donot_create_result_folder() == 0) {
     	$self->create_result_folder();
     }
-
-    my $host = `hostname -f`;
-    $host = `hostname` if !$host;
-    chomp $host;
     my $start_time = gettimeofday;
 
     my $job_output;
-    if ($stdout_file)
+    if ($self->stdout_file)
     {
-	my $stdout_fh = IO::File->new($stdout_file, "w+");
-	my $stderr_fh = IO::File->new($stderr_file, "w+");
+	my $stdout_fh = IO::File->new($self->stdout_file, "w+");
+	my $stderr_fh = IO::File->new($self->stderr_file, "w+");
 
-	capture(sub { $job_output = $self->callback->($self, $app_def, $params, \%proc_param) } , stdout => $stdout_fh, stderr => $stderr_fh);
+	capture(sub { $job_output = $self->execute_callback->($self, $self->app_def, $self->params, $self->proc_params) } , stdout => $stdout_fh, stderr => $stderr_fh);
     }
     else
     {
-	$job_output = $self->callback->($self, $app_def, $params, \%proc_param);
+	$job_output = $self->execute_callback->($self, $self->app_def, $self->params, $self->proc_params);
     }
 
-    if (!defined($self->donot_create_result_folder()) || $self->donot_create_result_folder() == 0)
+    if ($self->params->{output_path} && $self->params->{output_path} && !$self->donot_create_result_folder())
     {
 	my $end_time = gettimeofday;
 	my $elap = $end_time - $start_time;
 
 	my $files = $ws->ls({ paths => [ $self->result_folder ], recursive => 1});
 
-	#
-	# Hack to finding task id.
-	#
-	my $task_id = 'TBD';
-	if ($ENV{PWD} =~ /([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})_\d+_\d+$/i)
-	{
-	    $task_id = $1;
-	}
-	else
-	{
-	    $task_id = "Not found in '$ENV{PWD}'";
-	}
+	my $task_id = $self->task_id;
 
 	my $job_obj = {
 	    id => $task_id,
-	    app => $app_def,
-	    parameters => \%proc_param,
+	    app => $self->app_def,
+	    parameters => $self->roc_params,
 	    start_time => $start_time,
 	    end_time => $end_time,
 	    elapsed_time => $elap,
-	    hostname => $host,
+	    hostname => $self->hostname,
 	    output_files => [ map { [ $_->[2] . $_->[0], $_->[4] ] } @{$files->{$self->result_folder}}],
 	    job_output => $job_output,
 	};
 
 	my $file = $self->params->{output_path} . "/" . $self->params->{output_file};
-	$ws->save_data_to_file($json->encode($job_obj), {}, $file, 'job_result',1);
+	$ws->save_data_to_file($self->json->encode($job_obj), {}, $file, 'job_result',1);
     }
     delete $self->{workspace};
 }
@@ -295,10 +355,15 @@ sub create_result_folder
 {
     my($self) = @_;
 
-    my $base_folder = $self->params->{output_path};
-    my $result_folder = $base_folder . "/." . $self->params->{output_file};
-    $self->result_folder($result_folder);
-    $self->workspace->create({overwrite => 1, objects => [[$result_folder, 'folder', { application_type => $self->app_definition->{id}}]]});
+    my $params = $self->params;
+
+    if ($params->{output_path} && $params->{output_file})
+    {
+	my $base_folder = $params->{output_path};
+	my $result_folder = $base_folder . "/." . $params->{output_file};
+	$self->result_folder($result_folder);
+	$self->workspace->create({overwrite => 1, objects => [[$result_folder, 'folder', { application_type => $self->app_definition->{id}}]]});
+    }
 }
 
 sub token