lib/SGN/Controller/solGS/AnalysisQueue.pm

   1 package SGN::Controller::solGS::AnalysisQueue;
   2
   3 use Moose;
   4 use namespace::autoclean;
   5 use File::Path qw / mkpath  /;
   6 use File::Spec::Functions qw / catfile catdir/;
   7 use File::Slurp qw /write_file read_file/;
   8 use JSON;
   9 use CXGN::Tools::Run;
  10 use Try::Tiny;
  11 use Storable qw/ nstore retrieve /;
  12 use Carp qw/ carp confess croak /;
  13 use Scalar::Util 'reftype';
  14 use URI;
  15
  16
  17 BEGIN { extends 'Catalyst::Controller::REST' }
  18
  19 __PACKAGE__->config(
  20     default   => 'application/json',
  21     stash_key => 'rest',
  22     map       => { 'application/json' => 'JSON' },
  23 );
  24
  25
  26 sub check_user_login : Path('/solgs/check/user/login') Args(0) {
  27     my ( $self, $c ) = @_;
  28
  29     my $user = $c->user();
  30     $c->stash->{rest}{loggedin} = 0;
  31
  32     if ($user) {
  33         $c->stash->{rest} = {
  34           contact => $self->get_user_detail($c),
  35           loggedin => 1
  36         };
  37     }
  38
  39 }
  40
  41 sub save_analysis_profile : Path('/solgs/save/analysis/profile') Args(0) {
  42     my ( $self, $c ) = @_;
  43
  44     my $analysis_profile = $c->req->params;
  45     $c->stash->{analysis_profile} = $analysis_profile;
  46
  47     my $analysis_page = $analysis_profile->{analysis_page};
  48     $c->stash->{analysis_page} = $analysis_page;
  49
  50     $c->stash->{rest}{result} = 0;
  51     $self->save_profile($c);
  52     my $error_saving = $c->stash->{error};
  53
  54     if ( !$error_saving ) {
  55       $c->stash->{rest}{result} = 1;
  56
  57     }
  58
  59 }
  60
  61 sub run_saved_analysis : Path('/solgs/run/saved/analysis/') Args(0) {
  62     my ( $self, $c ) = @_;
  63
  64     my $analysis_profile = $c->req->params;
  65     $c->stash->{analysis_profile} = $analysis_profile;
  66     $self->parse_arguments($c);
  67     $self->structure_output_details($c);
  68     $self->run_analysis($c);
  69
  70     $c->stash->{rest} = {
  71       result => $c->stash->{status},
  72       arguments => $analysis_profile->{arguments}
  73     };
  74
  75 }
  76
  77 sub check_analysis_name : Path('/solgs/check/analysis/name') Args() {
  78     my ( $self, $c ) = @_;
  79
  80     my $new_name = $c->req->param('name');
  81     my $match    = $self->check_analyses_names( $c, $new_name );
  82
  83     $c->stash->{rest}{analysis_exists} = $match;
  84
  85 }
  86
  87 sub display_analysis_status : Path('/solgs/display/analysis/status') Args(0) {
  88     my ( $self, $c ) = @_;
  89
  90     $c->stash->{rest}{data} = $self->get_user_solgs_analyses($c);
  91 }
  92
  93 sub check_analyses_names {
  94     my ( $self, $c, $new_name ) = @_;
  95
  96     my $logged_names = $self->check_log_analyses_names($c);
  97
  98     my $log_match;
  99     if ($logged_names) {
 100         $log_match = grep { $_ =~ /$new_name/i } @$logged_names;
 101     }
 102
 103     my $db_match;
 104
 105     my $sp_person_id = $c->user() ? $c->user->get_object()->get_sp_person_id() : undef;
 106     if ($new_name) {
 107         my $schema = $c->dbic_schema("Bio::Chado::Schema", undef, $sp_person_id);
 108         $db_match =
 109           $schema->resultset("Project::Project")->find( { name => $new_name } );
 110     }
 111
 112     my $match = $log_match || $db_match ? 1 : undef;
 113
 114     return $match;
 115
 116 }
 117
 118 sub check_log_analyses_names {
 119     my ( $self, $c ) = @_;
 120
 121     my $log_file = $self->analysis_log_file($c);
 122     my $names    = qx(cut -f 2 $log_file);
 123
 124     if ($names) {
 125         my @names = split( /\n/, $names );
 126
 127         shift(@names);
 128         return \@names;
 129     }
 130     else {
 131         return 0;
 132     }
 133 }
 134
 135 sub save_profile {
 136     my ( $self, $c ) = @_;
 137
 138     $self->analysis_log_file($c);
 139     my $log_file = $c->stash->{analysis_log_file};
 140
 141     $self->add_log_headers($c);
 142
 143     $self->format_log_entry($c);
 144     my $log_entry = $c->stash->{formatted_log_entry};
 145
 146     write_file( $log_file, { binmode => ':utf8', append => 1 }, $log_entry );
 147
 148 }
 149
 150 sub add_log_headers {
 151     my ( $self, $c ) = @_;
 152
 153     $self->analysis_log_file($c);
 154     my $log_file = $c->stash->{analysis_log_file};
 155
 156     my $headers = read_file( $log_file, { binmode => ':utf8' } );
 157
 158     unless ($headers) {
 159         $headers =
 160             'User_name' . "\t"
 161           . 'Analysis_name' . "\t"
 162           . "Analysis_page" . "\t"
 163           . "Status" . "\t"
 164           . "Submitted on" . "\t"
 165           . "Arguments" . "\n";
 166
 167         write_file( $log_file, { binmode => ':utf8' }, $headers );
 168     }
 169
 170 }
 171
 172 sub index_log_file_headers {
 173     my ( $self, $c ) = @_;
 174
 175     no warnings 'uninitialized';
 176
 177     $self->analysis_log_file($c);
 178     my $log_file = $c->stash->{analysis_log_file};
 179
 180     my @headers =
 181       split( /\t/, ( read_file( $log_file, { binmode => ':utf8' } ) )[0] );
 182
 183     my $header_index = {};
 184     my $cnt          = 0;
 185
 186     foreach my $header (@headers) {
 187         $header_index->{$header} = $cnt;
 188         $cnt++;
 189     }
 190
 191     $c->stash->{header_index} = $header_index;
 192
 193 }
 194
 195 sub create_itemized_prediction_log_entries {
 196     my ( $self, $c, $analysis_log ) = @_;
 197
 198     $analysis_log = $self->log_analysis_time($analysis_log);
 199
 200     my $json = JSON->new;
 201     my $args = $json->decode( $analysis_log->{arguments} );
 202
 203     my $trait_ids = $args->{training_traits_ids};
 204
 205     my $analysis_type = $args->{analysis_type};
 206
 207     my $url_args = {
 208         'training_pop_id'        => $args->{training_pop_id}->[0],
 209         'selection_pop_id'       => $args->{selection_pop_id}->[0],
 210         'genotyping_protocol_id' => $args->{genotyping_protocol_id},
 211         'data_set_type'          => $args->{data_set_type},
 212     };
 213
 214     my $entries;
 215     foreach my $trait_id (@$trait_ids) {
 216         $c->controller('solGS::Trait')->get_trait_details( $c, $trait_id );
 217         my $trait_abbr = $c->stash->{trait_abbr};
 218         $url_args->{trait_id} = $trait_id;
 219
 220         my $analysis_page;
 221         if ( $analysis_type =~ /selection_prediction/ ) {
 222             $analysis_page =
 223               $c->controller('solGS::Path')->selection_page_url($url_args);
 224         }
 225         else {
 226             $analysis_page =
 227               $c->controller('solGS::Path')->model_page_url($url_args);
 228             $analysis_type =
 229               $c->controller('solGS::Path')->page_type( $c, $analysis_page );
 230         }
 231
 232         my $analysis_name =
 233           $analysis_log->{analysis_name} . ' -- ' . $trait_abbr;
 234
 235         $args->{analysis_page}       = $analysis_page;
 236         $args->{analysis_name}       = $analysis_name;
 237         $args->{trait_id}            = [$trait_id];
 238         $args->{training_traits_ids} = [$trait_id];
 239         $args->{analysis_type}       = $analysis_type;
 240
 241         $entries .= join(
 242             "\t",
 243             (
 244                 $analysis_log->{user_name}, $analysis_name,
 245                 $analysis_page,             'Submitted',
 246                 $args->{analysis_time},     $json->encode($args),
 247             )
 248         );
 249
 250         $entries .= "\n";
 251
 252     }
 253
 254     return $entries;
 255
 256 }
 257
 258 sub log_analysis_time {
 259     my ( $self, $analysis_log ) = @_;
 260
 261     my $analysis_time = POSIX::strftime( "%m/%d/%Y %H:%M", localtime );
 262
 263     my $json = JSON->new;
 264     my $args = $json->decode( $analysis_log->{arguments} );
 265
 266     $args->{analysis_time}     = $analysis_time;
 267     $analysis_log->{arguments} = $json->encode($args);
 268
 269     return $analysis_log;
 270
 271 }
 272
 273 sub format_log_entry {
 274     my ( $self, $c ) = @_;
 275
 276     my $profile = $c->stash->{analysis_profile};
 277     $profile = $self->log_analysis_time($profile);
 278     my $args = $profile->{arguments};
 279
 280     my $json = JSON->new;
 281     my $time = $json->decode($args)->{analysis_time};
 282
 283     my $traits_args = $json->decode($args);
 284     my $traits_ids =
 285       $traits_args->{training_traits_ids} || $traits_args->{trait_id};
 286     my @traits_ids = ref($traits_ids) eq 'ARRAY' ? @$traits_ids : ($traits_ids);
 287
 288     my $analysis_page;
 289     my $analysis_type = $traits_args->{analysis_type};
 290
 291     if ( @traits_ids > 1 && $analysis_type =~ /selection/ ) {
 292         $analysis_page = $traits_args->{referer};
 293     }
 294     else {
 295         $analysis_page = $traits_args->{analysis_page};
 296     }
 297
 298     my $entry = join(
 299         "\t",
 300         (
 301             $profile->{user_name}, $profile->{analysis_name},
 302             $analysis_page,        'Submitted',
 303             $time,                 $args
 304         )
 305     );
 306
 307     $entry .= "\n";
 308
 309     if ( @traits_ids > 1 && $analysis_type =~ /model|selection/ ) {
 310         my $traits_entries =
 311           $self->create_itemized_prediction_log_entries( $c, $profile );
 312         $entry .= $traits_entries;
 313     }
 314
 315     $c->stash->{formatted_log_entry} = $entry;
 316
 317 }
 318
 319 sub analysis_report_job_args {
 320     my ( $self, $c, $status_check_duration ) = @_;
 321
 322     my $analysis_details = $c->stash->{bg_job_output_details};
 323
 324     my $temp_dir =
 325       $c->stash->{analysis_tempfiles_dir} || $c->stash->{solgs_tempfiles_dir};
 326
 327     my $temp_file_template = "analysis-status";
 328     my $cluster_files      = $c->controller('solGS::AsyncJob')
 329       ->create_cluster_accessible_tmp_files( $c, $temp_file_template );
 330     my $out_file = $cluster_files->{out_file_temp};
 331     my $err_file = $cluster_files->{err_file_temp};
 332     my $in_file  = $cluster_files->{in_file_temp};
 333
 334     my $config_args = {
 335         'temp_dir'     => $temp_dir,
 336         'out_file'     => $out_file,
 337         'err_file'     => $err_file,
 338         'cluster_host' => 'localhost'
 339     };
 340
 341     my $report_file = $c->controller('solGS::Files')
 342       ->create_tempfile( $temp_dir, 'analysis-report-args' );
 343     nstore $analysis_details, $report_file
 344       or croak
 345       "analysis_report_job_args: $! serializing output_details to $report_file";
 346
 347     my $job_config = $c->controller('solGS::AsyncJob')
 348       ->create_cluster_config( $c, $config_args );
 349
 350     $status_check_duration =
 351       ' --status_check_duration ' . $status_check_duration
 352       if $status_check_duration;
 353
 354     my $cmd =
 355         'mx-run solGS::AnalysisReport'
 356       . ' --output_details_file '
 357       . $report_file
 358       . $status_check_duration;
 359
 360     my $job_args = {
 361         'cmd'            => $cmd,
 362         'config'         => $job_config,
 363         'background_job' => $c->stash->{background_job},
 364         'temp_dir'       => $temp_dir,
 365     };
 366
 367     $c->stash->{analysis_report_job_args} = $job_args;
 368
 369 }
 370
 371 sub get_analysis_report_job_args_file {
 372     my ( $self, $c, $status_check_duration ) = @_;
 373
 374     $self->analysis_report_job_args( $c, $status_check_duration );
 375     my $analysis_job_args = $c->stash->{analysis_report_job_args};
 376
 377     my $temp_dir = $c->stash->{solgs_tempfiles_dir};
 378
 379     my $report_file = $c->controller('solGS::Files')
 380       ->create_tempfile( $temp_dir, 'analysis-report-job-args' );
 381     nstore $analysis_job_args, $report_file
 382       or croak
 383 "get_analysis_report_job_args_file: $! serializing output_details to $report_file";
 384
 385     $c->stash->{analysis_report_job_args_file} = $report_file;
 386
 387 }
 388
 389 sub email_analysis_report {
 390     my ( $self, $c ) = @_;
 391
 392     $self->analysis_report_job_args($c);
 393     my $job_args = $c->stash->{analysis_report_job_args};
 394
 395     my $job =
 396       $c->controller('solGS::AsyncJob')->submit_job_cluster( $c, $job_args );
 397
 398 }
 399
 400 sub parse_arguments {
 401     my ( $self, $c ) = @_;
 402
 403     my $analysis_data = $c->stash->{analysis_profile};
 404     my $arguments     = $analysis_data->{arguments};
 405
 406     if ($arguments) {
 407         $c->controller('solGS::Utils')->stash_json_args( $c, $arguments );
 408     }
 409
 410 }
 411
 412 sub structure_output_details {
 413     my ( $self, $c ) = @_;
 414
 415     my $analysis_data = $c->stash->{analysis_profile};
 416     my $analysis_page = $analysis_data->{analysis_page};
 417
 418     my $uri_base = $c->req->base;
 419     my $referer  = $c->req->referer || $analysis_page;
 420     $referer =~ s/$uri_base//;
 421
 422     my $base = $c->stash->{hostname};
 423     $referer = $base . "/" . $referer;
 424
 425     my $output_details = {};
 426
 427     my $match_pages =
 428         'solgs\/traits\/all\/population\/'
 429       . '|solgs\/trait\/'
 430       . '|solgs\/model\/combined\/trials\/'
 431       . '|solgs\/models\/combined\/trials\/';
 432
 433     if ( $analysis_page =~ m/$match_pages/ ) {
 434         $output_details = $self->structure_training_modeling_output($c);
 435     }
 436     elsif ( $analysis_page =~ m/solgs\/population\// ) {
 437         $output_details = $self->structure_training_single_pop_data_output($c);
 438     }
 439     elsif ( $analysis_page =~ m/solgs\/populations\/combined\// ) {
 440         $output_details =
 441           $self->structure_training_combined_pops_data_output($c);
 442     }
 443     elsif ( $analysis_page =~
 444 m/solgs\/selection\/(\d+|\w+_\d+)\/model\/|solgs\/combined\/model\/\d+\/selection\//
 445       )
 446     {
 447         $output_details = $self->structure_selection_prediction_output($c);
 448     }
 449     elsif ( $analysis_page =~ m/kinship\/analysis/ ) {
 450         $output_details = $self->structure_kinship_analysis_output($c);
 451     }
 452     elsif ( $analysis_page =~ m/pca\/analysis/ ) {
 453         $output_details = $self->structure_pca_analysis_output($c);
 454     }
 455     elsif ( $analysis_page =~ m/cluster\/analysis/ ) {
 456         $output_details = $self->structure_cluster_analysis_output($c);
 457     }
 458
 459     $self->analysis_log_file($c);
 460     my $log_file = $c->stash->{analysis_log_file};
 461
 462     my $mail_list = $self->mailing_list($c);
 463
 464     $output_details->{analysis_profile}  = $analysis_data;
 465     $output_details->{contact_page}      = $base . '/contact/form';
 466     $output_details->{data_set_type}     = $c->stash->{data_set_type};
 467     $output_details->{analysis_log_file} = $log_file;
 468     $output_details->{host}              = qq | $base |;
 469     $output_details->{referer}           = qq | $referer |;
 470     $output_details->{mailing_list}      = $mail_list;
 471
 472     $c->stash->{bg_job_output_details} = $output_details;
 473
 474 }
 475
 476 sub mailing_list {
 477     my ( $self, $c ) = @_;
 478
 479     my $mail_list = $c->config->{cluster_job_email};
 480
 481     if ( !$mail_list ) {
 482         $mail_list = 'cluster-jobs@solgenomics.net';
 483     }
 484
 485     return $mail_list;
 486 }
 487
 488 sub structure_kinship_analysis_output {
 489     my ( $self, $c ) = @_;
 490
 491     my $analysis_data = $c->stash->{analysis_profile};
 492     my $analysis_page = $analysis_data->{analysis_page};
 493
 494     my $protocol_id = $c->stash->{genotyping_protocol_id};
 495
 496     $c->controller('solGS::Kinship')->stash_kinship_pop_id($c);
 497     my $pop_id = $c->stash->{kinship_pop_id};
 498
 499     my $base = $c->stash->{hostname};
 500
 501     my $kinship_page = $base . $analysis_page;
 502     $analysis_data->{analysis_page} = $kinship_page;
 503
 504     my %output_details = ();
 505
 506     my $trait_id = $c->stash->{trait_id};
 507
 508     $c->controller('solGS::Files')
 509       ->genotype_file_name( $c, $pop_id, $protocol_id );
 510     my $geno_file = $c->stash->{genotype_file_name};
 511
 512     my $coef_files = $c->controller('solGS::Kinship')
 513       ->get_kinship_coef_files( $c, $pop_id, $protocol_id, $trait_id );
 514     my $matrix_file = $coef_files->{matrix_file_adj};
 515
 516     $output_details{ 'kinship_' . $pop_id } = {
 517         'output_page'    => $kinship_page,
 518         'kinship_pop_id' => $pop_id,
 519         'genotype_file'  => $geno_file,
 520         'matrix_file'    => $matrix_file,
 521     };
 522
 523     return \%output_details;
 524 }
 525
 526 sub structure_pca_analysis_output {
 527     my ( $self, $c ) = @_;
 528
 529     my $analysis_data = $c->stash->{analysis_profile};
 530     my $analysis_page = $analysis_data->{analysis_page};
 531
 532     my $base = $c->stash->{hostname};
 533
 534     my $pca_page = $base . $analysis_page;
 535     $analysis_data->{analysis_page} = $pca_page;
 536     my $pop_id = $c->stash->{pca_pop_id};
 537
 538     $c->stash->{file_id} = $c->controller('solGS::Files')->create_file_id($c);
 539     my $input_file = $c->controller('solGS::pca')->pca_data_input_files($c);
 540
 541     $c->controller('solGS::pca')->pca_scores_file($c);
 542     my $scores_file = $c->stash->{pca_scores_file};
 543
 544     my %output_details = (
 545         'pca_'
 546           . $pop_id => {
 547             'output_page' => $pca_page,
 548             'pca_pop_id'  => $pop_id,
 549             'input_file'  => $input_file,
 550             'scores_file' => $scores_file,
 551           }
 552     );
 553
 554     return \%output_details;
 555
 556 }
 557
 558 sub structure_cluster_analysis_output {
 559     my ( $self, $c ) = @_;
 560
 561     my $analysis_data = $c->stash->{analysis_profile};
 562     my $analysis_page = $analysis_data->{analysis_page};
 563
 564     my $pop_id = $c->stash->{cluster_pop_id};
 565
 566     my $base         = $c->stash->{hostname};
 567     my $cluster_page = $base . $analysis_page;
 568     $analysis_data->{analysis_page} = $cluster_page;
 569     my $cluster_type = $c->stash->{cluster_type};
 570
 571     my $input_file =
 572       $c->controller('solGS::Cluster')->cluster_data_input_files($c);
 573
 574     $c->stash->{file_id} = $c->controller('solGS::Files')->create_file_id($c);
 575     $c->controller('solGS::Cluster')->cluster_result_file($c);
 576
 577     my $result_file;
 578     if ( $cluster_type =~ /k-means/i ) {
 579         $result_file = $c->stash->{"${cluster_type}_result_file"};
 580     }
 581     else {
 582         $result_file = $c->stash->{"${cluster_type}_result_newick_file"};
 583     }
 584
 585     my %output_details = (
 586         'cluster_'
 587           . $pop_id => {
 588             'output_page'    => $cluster_page,
 589             'cluster_pop_id' => $pop_id,
 590             'input_file'     => $input_file,
 591             'result_file'    => $result_file,
 592           }
 593     );
 594
 595     return \%output_details;
 596
 597 }
 598
 599 sub structure_training_modeling_output {
 600     my ( $self, $c ) = @_;
 601
 602     my $analysis_data = $c->stash->{analysis_profile};
 603     my $analysis_page = $analysis_data->{analysis_page};
 604
 605     my $training_pop_id = $c->stash->{training_pop_id};
 606     my $protocol_id     = $c->stash->{genotyping_protocol_id};
 607
 608     my @traits_ids = @{ $c->stash->{training_traits_ids} }
 609       if $c->stash->{training_traits_ids};
 610
 611     my $referer = $c->req->referer;
 612     my $base    = $c->stash->{hostname};
 613
 614     my $url_args = {
 615         'training_pop_id'        => $training_pop_id,
 616         'genotyping_protocol_id' => $protocol_id,
 617     };
 618
 619     if (
 620         $referer =~ /solgs\/population\//
 621         || (   $referer =~ /solgs\/search\/trials\/trait\//
 622             && $analysis_page =~ m/solgs\/trait\// )
 623       )
 624     {
 625         $url_args->{data_set_type} = 'single_population';
 626     }
 627     elsif ($referer =~ /solgs\/populations\/combined\//
 628         || $analysis_page =~ /solgs\/model\/combined\/trials\// )
 629     {
 630         $url_args->{data_set_type} = 'combined_populations';
 631
 632         $c->stash->{combo_pops_id} = $training_pop_id;
 633         $c->controller('solGS::combinedTrials')->cache_combined_pops_data($c);
 634     }
 635
 636     my $multi_models_url;
 637     if ( scalar(@traits_ids) > 1 ) {
 638         my $traits_selection_id = $c->controller('solGS::Gebvs')
 639           ->create_traits_selection_id( \@traits_ids );
 640
 641         $c->controller('solGS::Gebvs')
 642           ->catalogue_traits_selection( $c, \@traits_ids );
 643
 644         $url_args->{'traits_selection_id'} = $traits_selection_id;
 645
 646         $multi_models_url =
 647           $c->controller("solGS::Path")->multi_models_page_url($url_args);
 648         $multi_models_url = $base . $multi_models_url;
 649     }
 650
 651
 652     my %output_details = ();
 653
 654     foreach my $trait_id (@traits_ids) {
 655         $url_args->{trait_id} = $trait_id;
 656
 657         $c->stash->{cache_dir} = $c->stash->{solgs_cache_dir};
 658
 659         $c->controller('solGS::Trait')->get_trait_details( $c, $trait_id );
 660         $c->controller('solGS::Files')->rrblup_training_gebvs_file($c);
 661
 662         my $trait_abbr = $c->stash->{trait_abbr};
 663
 664         my $model_page =
 665           $c->controller('solGS::Path')->model_page_url($url_args);
 666         $model_page = $base . $model_page;
 667
 668         $c->controller('solGS::Files')->model_phenodata_file($c);
 669         my $model_pheno_file = $c->stash->{model_phenodata_file};
 670
 671         $c->controller('solGS::Files')->model_genodata_file($c);
 672         my $model_geno_file = $c->stash->{model_genodata_file};
 673
 674         $output_details{ 'trait_id_' . $trait_abbr } = {
 675             'trait_id'       => $trait_id,
 676             'trait_name'     => $c->stash->{trait_name},
 677             'trait_page'     => $model_page,
 678             'gebv_file'      => $c->stash->{rrblup_training_gebvs_file},
 679             'pop_id'         => $training_pop_id,
 680             'phenotype_file' => $model_pheno_file,
 681             'genotype_file'  => $model_geno_file,
 682             'data_set_type'  => $c->stash->{data_set_type},
 683         };
 684
 685     }
 686
 687     $output_details{'multi_models_url'} = $multi_models_url;
 688
 689     return \%output_details;
 690 }
 691
 692 sub structure_training_single_pop_data_output {
 693     my ( $self, $c ) = @_;
 694
 695     my $pop_id      = $c->stash->{pop_id};
 696     my $protocol_id = $c->stash->{genotyping_protocol_id};
 697
 698     my $base = $c->stash->{hostname};
 699     my $args = {
 700         'training_pop_id'        => $pop_id,
 701         'genotyping_protocol_id' => $protocol_id,
 702         'data_set_type'          => 'single_population'
 703     };
 704
 705     my $training_pop_page =
 706       $c->controller('solGS::Path')->training_page_url($args);
 707     my $population_page = $base . $training_pop_page;
 708
 709     my $data_set_type = $c->stash->{data_set_type};
 710     my $pheno_file;
 711     my $geno_file;
 712     my $pop_name;
 713
 714     my %output_details = ();
 715
 716     if ( $pop_id =~ /list/ ) {
 717         my $files =
 718           $c->controller('solGS::List')->create_list_pop_data_files($c);
 719         $pheno_file = $files->{pheno_file};
 720         $geno_file  = $files->{geno_file};
 721
 722         $c->controller('solGS::List')
 723           ->create_list_population_metadata_file( $c, $pop_id );
 724         $c->controller('solGS::List')->list_population_summary($c);
 725         $pop_name = $c->stash->{project_name};
 726     }
 727     elsif ( $pop_id =~ /dataset/ ) {
 728         my $files = $c->controller('solGS::Dataset')
 729           ->create_dataset_pop_data_files( $c, );
 730         $pheno_file = $files->{pheno_file};
 731         $geno_file  = $files->{geno_file};
 732
 733         $c->controller('solGS::Dataset')
 734           ->create_dataset_population_metadata_file($c);
 735         $c->controller('solGS::Dataset')->dataset_population_summary($c);
 736         $pop_name = $c->stash->{project_name};
 737     }
 738     else {
 739         $c->controller('solGS::Files')->phenotype_file_name( $c, $pop_id );
 740         $c->controller('solGS::Files')
 741           ->genotype_file_name( $c, $pop_id, $protocol_id );
 742         $pheno_file = $c->stash->{phenotype_file_name};
 743         $geno_file  = $c->stash->{genotype_file_name};
 744
 745         $c->controller('solGS::Search')->get_project_details( $c, $pop_id );
 746         $pop_name = $c->stash->{project_name};
 747     }
 748
 749     $output_details{ 'population_id_' . $pop_id } = {
 750         'population_page' => $population_page,
 751         'population_id'   => $pop_id,
 752         'population_name' => $pop_name,
 753         'phenotype_file'  => $pheno_file,
 754         'genotype_file'   => $geno_file,
 755         'data_set_type'   => $data_set_type,
 756     };
 757
 758     return \%output_details;
 759 }
 760
 761 sub structure_training_combined_pops_data_output {
 762     my ( $self, $c ) = @_;
 763
 764     my $combo_pops_id = $c->stash->{combo_pops_id};
 765     my $protocol_id   = $c->stash->{genotyping_protocol_id};
 766
 767     my $base = $c->stash->{hostname};
 768     my $args = {
 769         'training_pop_id'        => $combo_pops_id,
 770         'genotyping_protocol_id' => $protocol_id,
 771         'data_set_type'          => 'combined_populations'
 772     };
 773
 774     my $training_pop_page =
 775       $c->controller('solGS::Path')->training_page_url($args);
 776
 777     my $combined_pops_page = $base . $training_pop_page;
 778     my @combined_pops_ids  = @{ $c->stash->{combo_pops_list} };
 779
 780     $c->controller('solGS::combinedTrials')
 781       ->multi_pops_pheno_files( $c, \@combined_pops_ids );
 782     $c->controller('solGS::combinedTrials')
 783       ->multi_pops_geno_files( $c, \@combined_pops_ids, $protocol_id );
 784
 785     my $multi_ph_files  = $c->stash->{multi_pops_pheno_files};
 786     my @pheno_files     = split( /\t/, $multi_ph_files );
 787     my $multi_gen_files = $c->stash->{multi_pops_geno_files};
 788     my @geno_files      = split( /\t/, $multi_gen_files );
 789     my $match_status    = $c->stash->{pops_with_no_genotype_match};
 790
 791     my %output_details = ();
 792     foreach my $pop_id (@combined_pops_ids) {
 793         $c->controller('solGS::Search')->get_project_details( $c, $pop_id );
 794         my $population_name = $c->stash->{project_name};
 795
 796         $args = {
 797             'training_pop_id'        => $pop_id,
 798             'genotyping_protocol_id' => $protocol_id,
 799             'data_set_type'          => 'single_population'
 800         };
 801
 802         my $training_pop_page =
 803           $c->controller('solGS::Path')->training_page_url($args);
 804
 805         my $population_page = $base . $training_pop_page;
 806
 807         $c->controller('solGS::Files')->phenotype_file_name( $c, $pop_id );
 808         my $pheno_file = $c->stash->{phenotype_file_name};
 809
 810         $c->controller('solGS::Files')
 811           ->phenotype_file_name( $c, $pop_id, $protocol_id );
 812         my $geno_file = $c->stash->{genotype_file_name};
 813
 814         $output_details{ 'population_id_' . $pop_id } = {
 815             'population_page' => $population_page,
 816             'population_id'   => $pop_id,
 817             'population_name' => $population_name,
 818             'combo_pops_id'   => $combo_pops_id,
 819             'phenotype_file'  => $pheno_file,
 820             'genotype_file'   => $geno_file,
 821             'data_set_type'   => $c->stash->{data_set_type},
 822         };
 823     }
 824
 825     $output_details{no_match}           = $match_status;
 826     $output_details{combined_pops_page} = $combined_pops_page;
 827
 828     return \%output_details;
 829 }
 830
 831 sub structure_selection_prediction_output {
 832     my ( $self, $c ) = @_;
 833
 834     my @traits_ids = @{ $c->stash->{training_traits_ids} }
 835       if $c->stash->{training_traits_ids};
 836     my $protocol_id = $c->stash->{genotyping_protocol_id};
 837
 838     my $referer = $c->req->referer;
 839     my $base    = $c->stash->{hostname};
 840
 841     my $data_set_type  = $c->stash->{data_set_type};
 842     my %output_details = ();
 843
 844     foreach my $trait_id (@traits_ids) {
 845         $c->controller('solGS::Trait')->get_trait_details( $c, $trait_id );
 846         my $trait_id   = $c->stash->{trait_id};
 847         my $trait_abbr = $c->stash->{trait_abbr};
 848         my $trait_name = $c->stash->{trait_name};
 849
 850         my $tr_pop_id  = $c->stash->{training_pop_id};
 851         my $sel_pop_id = $c->stash->{selection_pop_id};
 852
 853         my $tr_pop_page;
 854         my $model_page;
 855         my $sel_pop_page;
 856         my $tr_pop_name;
 857         my $sel_pop_name;
 858
 859         my $url_args = {
 860             'training_pop_id'        => $tr_pop_id,
 861             'selection_pop_id'       => $sel_pop_id,
 862             'trait_id'               => $trait_id,
 863             'genotyping_protocol_id' => $protocol_id,
 864             'data_set_type'          => $data_set_type,
 865         };
 866
 867         if ( $data_set_type =~ /combined_populations/ ) {
 868             $tr_pop_page =
 869               $c->controller('solGS::Path')->training_page_url($url_args);
 870
 871             $tr_pop_page = $base . $tr_pop_page;
 872             $tr_pop_name = 'Training population ' . $tr_pop_id;
 873             $sel_pop_page =
 874               $c->controller('solGS::Path')->selection_page_url($url_args);
 875             $sel_pop_page = $base . $sel_pop_page;
 876
 877             $model_page =
 878               $c->controller('solGS::Path')->model_page_url($url_args);
 879             $model_page = $base . $model_page;
 880         }
 881         else {
 882             my $training_pop_page =
 883               $c->controller('solGS::Path')->training_page_url($url_args);
 884
 885             $tr_pop_page = $base . $training_pop_page;
 886             if ( $tr_pop_id =~ /list/ ) {
 887                 $c->stash->{list_id} = $tr_pop_id =~ s/\w+_//r;
 888                 $c->controller('solGS::List')->list_population_summary($c);
 889                 $tr_pop_name = $c->stash->{project_name};
 890             }
 891             elsif ( $tr_pop_id =~ /dataset/ ) {
 892                 $c->stash->{dataset_id} = $tr_pop_id =~ s/\w+_//r;
 893                 $c->controller('solGS::Dataset')
 894                   ->dataset_population_summary($c);
 895                 $tr_pop_name = $c->stash->{project_name};
 896             }
 897             else {
 898                 $c->controller('solGS::Search')
 899                   ->get_project_details( $c, $tr_pop_id );
 900                 $tr_pop_name = $c->stash->{project_name};
 901             }
 902
 903             $sel_pop_page =
 904               $c->controller('solGS::Path')->selection_page_url($url_args);
 905             $sel_pop_page = $base . $sel_pop_page;
 906
 907             $model_page =
 908               $c->controller('solGS::Path')->model_page_url($url_args);
 909             $model_page = $base . $model_page;
 910         }
 911
 912         if ( $sel_pop_id =~ /list/ ) {
 913             $c->stash->{list_id} = $sel_pop_id =~ s/\w+_//r;
 914             $c->controller('solGS::List')
 915               ->list_population_summary( $c, $sel_pop_id );
 916             $c->controller('solGS::List')
 917               ->create_list_population_metadata_file( $c, $sel_pop_id );
 918
 919             $sel_pop_name = $c->stash->{selection_pop_name};
 920         }
 921         elsif ( $sel_pop_id =~ /dataset/ ) {
 922             $c->stash->{dataset_id} = $sel_pop_id =~ s/\w+_//r;
 923             $c->controller('solGS::Dataset')
 924               ->create_dataset_population_metadata_file($c);
 925             $c->controller('solGS::Dataset')->dataset_population_summary($c);
 926             $sel_pop_name = $c->stash->{selection_pop_name};
 927         }
 928         else {
 929             $c->controller('solGS::Search')
 930               ->get_project_details( $c, $sel_pop_id );
 931             $sel_pop_name = $c->stash->{project_name};
 932         }
 933
 934         $c->controller('solGS::Files')
 935           ->rrblup_selection_gebvs_file( $c, $tr_pop_id, $sel_pop_id,
 936             $trait_id );
 937         my $gebv_file = $c->stash->{rrblup_selection_gebvs_file};
 938
 939         $c->controller('solGS::Files')
 940           ->genotype_file_name( $c, $sel_pop_id, $protocol_id );
 941         my $selection_geno_file = $c->stash->{genotype_file_name};
 942
 943         $output_details{ 'trait_id_' . $trait_id } = {
 944             'training_pop_page'   => $tr_pop_page,
 945             'training_pop_id'     => $tr_pop_id,
 946             'training_pop_name'   => $tr_pop_name,
 947             'selection_pop_name'  => $sel_pop_name,
 948             'selection_pop_page'  => $sel_pop_page,
 949             'trait_name'          => $trait_name,
 950             'trait_id'            => $trait_id,
 951             'model_page'          => $model_page,
 952             'gebv_file'           => $gebv_file,
 953             'selection_geno_file' => $selection_geno_file,
 954             'data_set_type'       => $data_set_type
 955         };
 956
 957     }
 958
 959     if ( scalar(@traits_ids) > 1 ) {
 960         my $uri_base = $c->req->base;
 961         my $referer  = $c->req->referer;
 962         $referer =~ s/$uri_base//;
 963         my $base = $c->stash->{hostname};
 964
 965         $output_details{'multi_models_url'} = $base . "/" . $referer;
 966     }
 967
 968     return \%output_details;
 969
 970 }
 971
 972 sub run_analysis {
 973     my ( $self, $c ) = @_;
 974
 975     $c->stash->{background_job} = 1;
 976
 977     my $analysis_profile = $c->stash->{analysis_profile};
 978     my $analysis_page    = $analysis_profile->{analysis_page};
 979     $c->stash->{analysis_page} = $analysis_page;
 980
 981     my $base = $c->stash->{hostname};
 982     $analysis_page =~ s/$base//;
 983     my $referer = $c->req->referer;
 984
 985     my @selected_traits = @{ $c->stash->{training_traits_ids} }
 986       if $c->stash->{training_traits_ids};
 987
 988     eval {
 989         my $modeling_pages =
 990             'solgs\/traits\/all\/population\/'
 991           . '|solgs\/models\/combined\/trials\/'
 992           . '|solgs\/trait\/'
 993           . '|solgs\/model\/combined\/trials\/';
 994
 995         my $selection_pages = '/solgs\/selection\/(\d+|\w+_\d+)\/model\/'
 996           . '|solgs\/combined\/model\/(\d+|\w+_\d+)\/selection\/';
 997
 998         my $training_pages =
 999           '/solgs\/population\/' . '|solgs\/populations\/combined\/';
1000
1001         if ( $analysis_page =~ $training_pages ) {
1002             $self->create_training_data($c);
1003         }
1004         elsif ( $analysis_page =~ /$modeling_pages/ ) {
1005             $self->predict_training_traits($c);
1006         }
1007         elsif ( $analysis_page =~ /$selection_pages/ ) {
1008             $self->predict_selection_traits($c);
1009         }
1010         elsif ( $analysis_page =~ /kinship\/analysis/ ) {
1011             $self->run_kinship_analysis($c);
1012         }
1013         elsif ( $analysis_page =~ /pca\/analysis/ ) {
1014             $self->run_pca_analysis($c);
1015         }
1016         elsif ( $analysis_page =~ /cluster\/analysis/ ) {
1017             $self->run_cluster_analysis($c);
1018         }
1019         else {
1020             $c->stash->{status} = 'Error: Unknown job';
1021             print STDERR "\n Unknown job.\n";
1022         }
1023     };
1024
1025     my @error = $@;
1026
1027     if ( $error[0] ) {
1028         $c->stash->{status} =
1029 "run_analysis failed. Please try re-running the analysis and wait for it to finish. $error[0]";
1030     }
1031     else {
1032         $c->stash->{status} = 'Submitted';
1033         $self->update_analysis_progress($c);
1034     }
1035
1036 }
1037
1038 sub create_training_data {
1039     my ( $self, $c ) = @_;
1040
1041     my $analysis_page = $c->stash->{analysis_page};
1042     my $protocol_id   = $c->stash->{genotyping_protocol_id};
1043
1044     # if ($analysis_page =~ /solgs\/population\//)
1045     # {
1046     my $pop_id = $c->stash->{model_id};
1047
1048     if ( $analysis_page =~ /solgs\/population\// ) {
1049         my $pop_id = $c->stash->{model_id};
1050
1051         if ( $pop_id =~ /list/ ) {
1052             $c->controller('solGS::List')->submit_list_training_data_query($c);
1053             $c->controller('solGS::List')
1054               ->create_list_population_metadata_file( $c, $pop_id );
1055         }
1056         elsif ( $pop_id =~ /dataset/ ) {
1057             $c->controller('solGS::Dataset')
1058               ->submit_dataset_training_data_query($c);
1059             $c->controller('solGS::Dataset')
1060               ->create_dataset_population_metadata_file($c);
1061         }
1062         else {
1063             $c->controller('solGS::AsyncJob')
1064               ->submit_cluster_training_pop_data_query( $c, [$pop_id],
1065                 $protocol_id );
1066         }
1067     }
1068     elsif ( $analysis_page =~ /solgs\/populations\/combined\// ) {
1069         my $trials = $c->stash->{combo_pops_list};
1070         $c->controller('solGS::AsyncJob')
1071           ->submit_cluster_training_pop_data_query( $c, $trials, $protocol_id );
1072     }
1073
1074     # }
1075 }
1076
1077 sub predict_training_traits {
1078     my ( $self, $c ) = @_;
1079
1080     my $analysis_page   = $c->stash->{analysis_page};
1081     my $selected_traits = $c->stash->{training_traits_ids};
1082
1083     $c->stash->{training_traits_ids} = [ $c->stash->{trait_id} ]
1084       if !$c->stash->{training_traits_ids};
1085
1086     if ( $analysis_page =~ /solgs\/traits\/all\/population\/|solgs\/trait\// ) {
1087         $c->controller('solGS::solGS')->build_multiple_traits_models($c);
1088     }
1089     elsif ( $analysis_page =~
1090         /solgs\/models\/combined\/trials\/|solgs\/model\/combined\/trials\// )
1091     {
1092         if ( $c->stash->{data_set_type} =~ /combined_populations/ ) {
1093             $c->controller('solGS::combinedTrials')
1094               ->combine_data_build_multiple_traits_models($c);
1095         }
1096     }
1097
1098 }
1099
1100 sub predict_selection_traits {
1101     my ( $self, $c ) = @_;
1102
1103     $c->stash->{prerequisite_type} = 'selection_pop_download_data';
1104     my $training_pop_id  = $c->stash->{training_pop_id};
1105     my $selection_pop_id = $c->stash->{selection_pop_id};
1106
1107     if ( $selection_pop_id =~ /list/ ) {
1108         $c->stash->{list_id} = $selection_pop_id =~ s/\w+_//r;
1109         $c->controller('solGS::List')->get_genotypes_list_details($c);
1110         $c->controller('solGS::List')
1111           ->create_list_population_metadata_file( $c, $selection_pop_id );
1112     }
1113     elsif ( $selection_pop_id =~ /dataset/ ) {
1114         $c->stash->{dataset_id} = $selection_pop_id =~ s/\w+_//r;
1115         $c->controller('solGS::Dataset')
1116           ->create_dataset_population_metadata_file($c);
1117     }
1118
1119     my $referer = $c->req->referer;
1120     if ( $referer =~ /solgs\/trait\/|solgs\/traits\/all\/population\// ) {
1121         $c->controller('solGS::solGS')->predict_selection_pop_multi_traits($c);
1122     }
1123     elsif ( $referer =~ /\/combined\// ) {
1124         $c->stash->{data_set_type} = 'combined_populations';
1125         $c->controller('solGS::combinedTrials')
1126           ->predict_selection_pop_combined_pops_model($c);
1127     }
1128
1129 }
1130
1131 sub run_kinship_analysis {
1132     my ( $self, $c ) = @_;
1133
1134     my $analysis_page = $c->stash->{analysis_page};
1135
1136     if ( $analysis_page = ~/kinship\/analysis/ ) {
1137         $c->controller('solGS::Kinship')->run_kinship($c);
1138     }
1139
1140 }
1141
1142 sub run_pca_analysis {
1143     my ( $self, $c ) = @_;
1144
1145     my $analysis_page = $c->stash->{analysis_page};
1146
1147     if ( $analysis_page = ~/pca\/analysis/ ) {
1148         $c->controller('solGS::pca')->run_pca($c);
1149     }
1150
1151 }
1152
1153 sub run_cluster_analysis {
1154     my ( $self, $c ) = @_;
1155
1156     my $analysis_page = $c->stash->{analysis_page};
1157
1158     if ( $analysis_page = ~/cluster\/analysis/ ) {
1159         $c->controller('solGS::Cluster')->run_cluster($c);
1160     }
1161
1162 }
1163
1164 sub update_analysis_progress {
1165     my ( $self, $c ) = @_;
1166
1167     my $analysis_data = $c->stash->{analysis_profile};
1168     my $analysis_name = $analysis_data->{analysis_name};
1169     my $status        = $c->stash->{status};
1170
1171     $self->analysis_log_file($c);
1172     my $log_file = $c->stash->{analysis_log_file};
1173
1174     my @contents = read_file( $log_file, { binmode => ':utf8' } );
1175
1176     map {
1177             $contents[$_] =~ m/\t$analysis_name\t/
1178           ? $contents[$_] =~ s/error|submitted/$status/ig
1179           : $contents[$_]
1180     } 0 .. $#contents;
1181
1182     write_file( $log_file, { binmode => ':utf8' }, @contents );
1183
1184 }
1185
1186 sub get_user_detail {
1187     my ( $self, $c ) = @_;
1188
1189     my $user = $c->user();
1190
1191     my $contact;
1192     if ($user) {
1193         my $private_email = $user->get_private_email();
1194         my $public_email  = $user->get_contact_email();
1195
1196         my $email =
1197             $public_email
1198           ? $public_email
1199           : $private_email;
1200
1201         my $salutation = $user->get_salutation();
1202         my $first_name = $user->get_first_name();
1203         my $last_name  = $user->get_last_name();
1204         my $user_role  = $user->get_object->get_user_type();
1205         my $user_id    = $user->get_object()->get_sp_person_id();
1206         my $user_name  = $user->id();
1207
1208         $contact = {
1209             'first_name' => $first_name,
1210             'email'      => $email,
1211             'user_role'  => $user_role,
1212             'user_id'    => $user_id,
1213             'user_name'  => $user_name,
1214         };
1215
1216     }
1217
1218     return $contact;
1219
1220 }
1221
1222 sub analysis_log_file {
1223     my ( $self, $c ) = @_;
1224
1225     $self->create_analysis_log_dir($c);
1226     my $log_dir = $c->stash->{analysis_log_dir};
1227
1228     $c->stash->{cache_dir} = $log_dir;
1229
1230     my $cache_data = {
1231         key       => 'analysis_log',
1232         file      => 'analysis_log',
1233         stash_key => 'analysis_log_file'
1234     };
1235
1236     $c->controller('solGS::Files')->cache_file( $c, $cache_data );
1237
1238 }
1239
1240 sub get_user_solgs_analyses {
1241     my ( $self, $c ) = @_;
1242
1243     $self->analysis_log_file($c);
1244     my $log_file = $c->stash->{analysis_log_file};
1245
1246     my $ret = {};
1247     my @panel_data;
1248
1249     no warnings 'uninitialized';
1250
1251     if ($log_file) {
1252         my @user_analyses = grep { $_ !~ /User_name\s+/i }
1253           read_file( $log_file, { binmode => ':utf8' } );
1254
1255         $self->index_log_file_headers($c);
1256         my $header_index = $c->stash->{header_index};
1257
1258         my $json = JSON->new();
1259         foreach my $row (@user_analyses) {
1260             my @analysis = split( /\t/, $row );
1261
1262             my $arguments = $analysis[5];
1263             $arguments = $json->decode($arguments);
1264             my $analysis_type   = $arguments->{analysis_type};
1265             my $analysis_name   = $analysis[ $header_index->{'Analysis_name'} ];
1266             my $result_page     = $analysis[ $header_index->{'Analysis_page'} ];
1267             my $analysis_status = $analysis[ $header_index->{'Status'} ];
1268             my $submitted_on    = $analysis[ $header_index->{'Submitted on'} ];
1269
1270             if ( $analysis_status =~ /Failed/i ) {
1271                 $result_page = 'N/A';
1272             }
1273             elsif ( $analysis_status =~ /Submitted/i ) {
1274                 $result_page = 'In progress...';
1275             }
1276             else {
1277                 $result_page = qq |<a href=$result_page>[ View ]</a>|;
1278             }
1279
1280             my $row = [
1281                 $analysis_name,   $analysis_type, $submitted_on,
1282                 $analysis_status, $result_page
1283             ];
1284             push @panel_data, $row;
1285         }
1286     }
1287
1288     return \@panel_data;
1289 }
1290
1291 sub create_analysis_log_dir {
1292     my ( $self, $c ) = @_;
1293
1294     my $user_id = $c->user->id;
1295
1296     $c->controller('solGS::Files')->get_solgs_dirs($c);
1297
1298     my $log_dir = $c->stash->{analysis_log_dir};
1299
1300     $log_dir = catdir( $log_dir, $user_id );
1301     mkpath( $log_dir, 0, 0755 );
1302
1303     $c->stash->{analysis_log_dir} = $log_dir;
1304
1305 }
1306
1307 sub begin : Private {
1308     my ( $self, $c ) = @_;
1309
1310     $c->controller('solGS::Files')->get_solgs_dirs($c);
1311
1312 }
1313
1314 __PACKAGE__->meta->make_immutable;
1315
1316 ####
1317 1;
1318 ####