Wrapup of new app service code for new release.

Author: olsonanl

AppConfig.pm.tt

@@ -16,24 +16,35 @@ use constant	    binning_spades_threads => '[% binning_spades_threads %]';
 use constant	    binning_spades_ram => '[% binning_spades_ram %]';
 
 use constant	    sched_db_host => '[% sched_db_host %]';
+use constant	    sched_db_port => '[% sched_db_port %]';
 use constant	    sched_db_user => '[% sched_db_user %]';
 use constant	    sched_db_pass => '[% sched_db_pass %]';
 use constant	    sched_db_name => '[% sched_db_name %]';
 
+use constant	    redis_host => '[% redis_host %]';
+use constant	    redis_port => '[% redis_port %]';
+use constant	    redis_db => '[% redis_db %]';
+
 use constant	    slurm_control_task_partition => '[% slurm_control_task_partition %]';
 
 use constant	    bebop_binning_user => '[% bebop_binning_user %]';
 use constant	    bebop_binning_key => '[% bebop_binning_key %]';
 
+use constant	    app_directory => '[% app_directory %]';
+use constant	    app_service_url => '[% app_service_url %]';
+
 use base 'Exporter';
 our @EXPORT_OK = qw(data_api_url github_issue_repo_owner github_issue_repo_name github_issue_token
 		    db_host db_user db_pass db_name 
 		    seedtk reference_data_dir
 		    bebop_binning_user bebop_binning_key
-		    sched_db_host sched_db_user sched_db_pass sched_db_name
+		    sched_db_host sched_db_port sched_db_user sched_db_pass sched_db_name
 		    slurm_control_task_partition
 		    binning_spades_threads binning_spades_ram
-		    binning_genome_annotation_clientgroup mash_reference_sketch);
+		    binning_genome_annotation_clientgroup mash_reference_sketch
+		    app_directory app_service_url
+		    redis_host redis_port redis_db
+		    );
 our %EXPORT_TAGS = (all => [@EXPORT_OK]);
 1;
 

AppService.spec

@@ -42,6 +42,7 @@ module AppService
 	string submit_time;
 	string start_time;
 	string completed_time;
+	string elapsed_time;
 
 	string stdout_shock_node;
 	string stderr_shock_node;
@@ -79,6 +80,8 @@ module AppService
 
     funcdef query_task_summary() returns (mapping<task_status status, int count> status);
 
+    funcdef query_app_summary() returns (mapping<app_id app, int count> status);
+
     typedef structure {
 	string stdout_url;
 	string stderr_url;
@@ -91,6 +94,17 @@ module AppService
     funcdef enumerate_tasks(int offset, int count)
 	returns (list<Task>);
 
+    typedef structure {
+ 	string start_time;
+	string end_time;
+	app_id app;
+	string search;
+	string status;
+    } SimpleTaskFilter;
+    funcdef enumerate_tasks_filtered(int offset, int count, SimpleTaskFilter simple_filter)
+	returns (list<Task> tasks, int total_tasks);
+
     funcdef kill_task(task_id id) returns (int killed, string msg);
-    funcdef rerun_task(task_id id) returns (task_id new_task);
+    funcdef kill_tasks(list<task_id> ids) returns (mapping<task_id, structure { int killed; string msg; }>);
+    funcdef rerun_task(task_id id) returns (Task task);
 };

Makefile

@@ -11,6 +11,7 @@ SERVICE_MODULE = lib/Bio/KBase/AppService/Service.pm
 SERVICE = app_service
 SERVICE_PORT = 7124
 
+SLURM_PATH = /disks/patric-common/slurm
 SLURM_CONTROL_TASK_PARTITION = watcher
 
 SERVICE_URL = http://p3.theseed.org/services/$(SERVICE)
@@ -19,6 +20,7 @@ SERVICE_NAME = AppService
 SERVICE_NAME_PY = $(SERVICE_NAME)
 
 SERVICE_PSGI_FILE = $(SERVICE_NAME).psgi
+SERVICE_ASYNC_PSGI_FILE = $(SERVICE_NAME)Async.psgi
 
 SRC_SERVICE_PERL = $(wildcard service-scripts/*.pl)
 BIN_SERVICE_PERL = $(addprefix $(BIN_DIR)/,$(basename $(notdir $(SRC_SERVICE_PERL))))
@@ -41,6 +43,8 @@ REFERENCE_DATA_DIR = /tmp
 
 MASH_REFERENCE_SKETCH = /vol/patric3/production/data/trees/listOfRepRefGenomeFnaFiles.txt.msh 
 
+APP_DIRECTORY = app_specs
+
 ifdef TEMPDIR
 TPAGE_TEMPDIR = --define kb_tempdir=$(TEMPDIR)
 endif
@@ -62,16 +66,22 @@ TPAGE_ARGS = \
 	--define kb_service_name=$(SERVICE) \
 	--define kb_service_port=$(SERVICE_PORT) \
 	--define kb_psgi=$(SERVICE_PSGI_FILE) \
+	--define kb_async_psgi=$(SERVICE_ASYNC_PSGI_FILE) \
 	--define kb_starman_workers=$(STARMAN_WORKERS) \
 	--define data_api_url=$(DATA_API_URL) \
+	--define redis_host=$(REDIS_HOST) \
+	--define redis_port=$(REDIS_PORT) \
+	--define redis_db=$(REDIS_DB) \
 	--define db_host=$(DB_HOST) \
 	--define db_user=$(DB_USER) \
 	--define db_pass=$(DB_PASS) \
 	--define db_name=$(DB_NAME) \
 	--define sched_db_host=$(SCHED_DB_HOST) \
+	--define sched_db_port=$(SCHED_DB_PORT) \
 	--define sched_db_user=$(SCHED_DB_USER) \
 	--define sched_db_pass=$(SCHED_DB_PASS) \
 	--define sched_db_name=$(SCHED_DB_NAME) \
+	--define slurm_path=$(SLURM_PATH) \
 	--define slurm_control_task_partition=$(SLURM_CONTROL_TASK_PARTITION) \
 	--define seedtk=$(SEEDTK) \
 	--define github_issue_repo_owner=$(GITHUB_ISSUE_REPO_OWNER) \
@@ -84,6 +94,8 @@ TPAGE_ARGS = \
 	--define bebop_binning_key=$(BEBOP_BINNING_KEY) \
 	--define bebop_binning_user=$(BEBOP_BINNING_USER) \
 	--define mash_reference_sketch=$(MASH_REFERENCE_SKETCH) \
+	--define app_directory=$(APP_DIRECTORY) \
+	--define app_service_url=$(SERVICE_URL) \
 	$(TPAGE_SERVICE_LOGDIR) \
 	$(TPAGE_TEMPDIR)
 

README.md

@@ -1,2 +1,3 @@
 app_service
 =================
+

app_specs/GenomeAssembly2.json

@@ -179,6 +179,14 @@
       "default": null,
       "desc": "Basename for the generated output files. Defaults to the basename of the input data.",
       "type": "wsid"
+    },
+    {
+      "id": "debug_level",
+      "label": "Debug level",
+      "required": 0,
+      "default": 0,
+      "desc": "Debugging level.",
+      "type": "int"
     }
   ]
 }

app_specs/MetagenomicReadMapping.json

@@ -1,7 +1,7 @@
 {
   "id": "MetagenomicReadMapping",
   "label": "Metagenomic read mapping",
-  "script": "AppWrapper-MetagenomicReadMapping",
+  "script": "App-MetagenomicReadMapping",
   "description": "Map metagenomic reads to a defined gene set",
   "default_memory": "100G",
   "parameters": [

deploy.cfg

@@ -9,7 +9,7 @@ app-directory = app_specs
 ; Directory into which task status (stdout/err etc) are written
 ;
 
-; task-status-dir =
+task-status-dir = /disks/p3/task_status
 
 ;
 ; AWE client group to submit jobs iwth

lib/AppServiceAsync.psgi

@@ -1,70 +1,59 @@
+use Twiggy;
+
 use Bio::KBase::AppService::AppServiceImpl;
 use Bio::KBase::AppService::Monitor;
 use Bio::KBase::AppService::Quick;
 use Bio::KBase::AppService::AsyncService;
 
 use Bio::KBase::AppService::AppSpecs;
-use Bio::KBase::AppService::Scheduler;
 use Bio::KBase::AppService::SlurmCluster;
 
-#use Carp::Always;
+# use Carp::Always;
 
 use Plack::Middleware::CrossOrigin;
 use Plack::Builder;
 use Data::Dumper;
+use Log::Dispatch;
+use Log::Dispatch::File;
 
 my @dispatch;
 
 my $obj = Bio::KBase::AppService::AppServiceImpl->new;
 
-my $specs = Bio::KBase::AppService::AppSpecs->new($obj->{app_dir});
-print Dumper($specs);
-
-my $sched = Bio::KBase::AppService::Scheduler->new(specs => $specs);
-$sched->{task_start_disable} = 0;
-$sched->load_apps();
-
-my $cluster = Bio::KBase::AppService::SlurmCluster->new('TSlurm',
-							schema => $sched->schema);
-
-my $shared_cluster = Bio::KBase::AppService::SlurmCluster->new('Bebop',
-							schema => $sched->schema,
-							resources => ["-p bdws",
-								      "-N 1",
-								      "--ntasks-per-node 1",
-								      "--time 1:00:00"]);
-my $bebop_cluster = Bio::KBase::AppService::SlurmCluster->new('Bebop',
-							schema => $sched->schema,
-							resources => [
-								      "-p bdwd",
-								      "-x bdwd-0050",
-								      # "-p bdwall",
-								      "-N 1",
-								      "-A PATRIC",
-								      "--ntasks-per-node 1"],
-							environment_config => ['module add jdk'], ['module add gnuplot']);
+my $logger;
+if (my $f = $ENV{APP_SERVICE_STDERR})
+{
+    $logger = Log::Dispatch->new;
+    $logger->add( Log::Dispatch::File->new(filename => $f, min_level => 'debug'));
+}
 
 
+my $specs = Bio::KBase::AppService::AppSpecs->new($obj->{app_dir});
+print Dumper($specs);
 
-$sched->default_cluster($cluster);
-
-$obj->{util}->scheduler($sched);
 Bio::KBase::AppService::Monitor::set_impl($obj);
 Bio::KBase::AppService::Quick::set_impl($obj);
 
 my $server = Bio::KBase::AppService::AsyncService->new(impl => $obj);
 
 my $rpc_handler = sub { $server->handle_rpc(@_); };
 
+print Dumper($logger);
+
 $handler = builder {
-    mount "/ping" => sub { $server->ping(@_); };
-    mount "/auth_ping" => sub { $server->auth_ping(@_); };
-    mount "/task_info" => sub { $obj->_task_info(@_); };
-    mount "/monitor" => Bio::KBase::AppService::Monitor->psgi_app;
-    mount "/quick" => Bio::KBase::AppService::Quick->psgi_app;
-    mount "/" => $rpc_handler;
+    if ($logger)
+    {
+	enable 'LogDispatch', logger => $logger;
+	enable 'LogStderr', no_tie => 1;
+    }
+    builder {
+	mount "/ping" => sub { $server->ping(@_); };
+	mount "/auth_ping" => sub { $server->auth_ping(@_); };
+	mount "/task_info" => sub { $obj->_task_info(@_); };
+	mount "/monitor" => Bio::KBase::AppService::Monitor->psgi_app;
+	mount "/quick" => Bio::KBase::AppService::Quick->psgi_app;
+	mount "/" => $rpc_handler;
+    };
 };
 
-$sched->start_timers();
-
-$handler = Plack::Middleware::CrossOrigin->wrap( $handler, origins => "*", headers => "*");
+$handler = Plack::Middleware::CrossOrigin->wrap( $handler, origins => "*", headers => "*", max_age =>  86400);

lib/Bio/KBase/AppService/AppScript-devslurm.pm

@@ -216,7 +216,7 @@ sub run
     {
 	open(my $fh, ">", $opt->preflight) or die "Cannot write preflight to " . $opt->preflight . ": $!";
 	my $data = $self->run_preflight();
-	print Dumper($data);
+	print STDERR  Dumper($data);
 	if (ref($data) eq 'HASH')
 	{
 	    print $fh $self->json->encode($data);

lib/Bio/KBase/AppService/AppScript.pm

@@ -25,7 +25,7 @@ use Data::Dumper;
 
 __PACKAGE__->mk_accessors(qw(execute_callback preflight_callback donot_create_job_result donot_create_result_folder
 			     workspace_url workspace params raw_params app_definition result_folder
-			     json host
+			     json
 			     task_id app_service_url));
 
 =head1 NAME
@@ -172,15 +172,10 @@ sub new
 {
     my($class, $execute_callback, $preflight_callback) = @_;
 
-    my $host = `hostname -f`;
-    $host = `hostname` if !$host;
-    chomp $host;
-
     my $self = {
 	execute_callback => $execute_callback,
 	preflight_callback => $preflight_callback,
 	start_time => scalar gettimeofday,
-	host => $host,
 	json => JSON::XS->new->pretty(1),
     };
 
@@ -191,6 +186,20 @@ sub new
     return $self;
 }
 
+sub host
+{
+    my($self) = @_;
+    my $host;
+    if (!defined($host = $self->{host}))
+    {
+	$host = `hostname -f`;
+	$host = `hostname` if !$host;
+	chomp $host;
+	$self->{host} = $host;
+    }
+    return $host;
+}
+
 sub run_preflight
 {
     my($self) = @_;
@@ -257,7 +266,6 @@ sub run
     {
 	open(my $fh, ">", $opt->preflight) or die "Cannot write preflight to " . $opt->preflight . ": $!";
 	my $data = $self->run_preflight();
-	print Dumper($data);
 	if (ref($data) eq 'HASH')
 	{
 	    print $fh $self->json->encode($data);
@@ -427,10 +435,6 @@ sub set_task_id
     #
     # Hack to finding task id.
     #
-    my $host = `hostname -f`;
-    $host = `hostname` if !$host;
-    chomp $host;
-    $self->host($host);
 
     my $task_id = 'TBD';
     if ($ENV{AWE_TASK_ID})
@@ -447,6 +451,7 @@ sub set_task_id
     }
     else
     {
+	my $host = $self->host;
 	$task_id = "UNK-$host-$$";
 	$task_id =~ s/\./_/g;
     }
@@ -462,6 +467,14 @@ sub create_result_folder
     my $result_folder = $base_folder . "/." . $self->params->{output_file};
     $self->result_folder($result_folder);
     $self->workspace->create({overwrite => 1, objects => [[$result_folder, 'folder', { application_type => $self->app_definition->{id}}]]});
+
+    #
+    # Remove any job failed reports that were left from a prior run.
+    #
+    eval { $self->workspace->delete({ objects => ["$result_folder/JobFailed.txt"] }) };
+    # if ($@) { warn "delete $result_folder/JobFailed.txt failed: $@"; }
+    eval { $self->workspace->delete({ objects => ["$result_folder/JobFailed.html"] }) };
+    # if ($@) { warn "delete $result_folder/JobFailed.html failed: $@"; }
 }
 
 sub token
@@ -521,6 +534,22 @@ sub preprocess_parameters
 	if (exists($params->{$id}))
 	{
 	    my $value = $params->{$param->{id}};
+
+	    if ($param->{type} eq 'bool')
+	    {
+		if ($value eq 'false')
+		{
+		    print STDERR "Fixing false value to 0 for $id\n";
+		    $value = 0;
+		}
+		elsif ($value eq 'true')
+		{
+		    print STDERR "Fixing true value to 1 for $id\n";
+		    $value = 1;
+		}
+	    }
+		    
+	    
 	    #
 	    # Maybe validate.
 	    #