lib/SGN/Controller/AJAX/ImageAnalysis.pm

   1
   2 =head1 NAME
   3
   4 SGN::Controller::AJAX::ImageAnalysis - a REST controller class to provide image analysis including
   5 functions for necrosis image analysis https://github.com/solomonnsumba/Necrosis-_Web_Server
   6
   7 =head1 DESCRIPTION
   8
   9 =head1 AUTHOR
  10
  11 =cut
  12
  13 package SGN::Controller::AJAX::ImageAnalysis;
  14
  15 use Moose;
  16 use Data::Dumper;
  17 use LWP::UserAgent;
  18 use LWP::Simple;
  19 use JSON;
  20 use SGN::Model::Cvterm;
  21 use DateTime;
  22 use CXGN::UploadFile;
  23 use SGN::Image;
  24 use CXGN::DroneImagery::ImagesSearch;
  25 use URI::Encode qw(uri_encode uri_decode);
  26 use CXGN::Calendar;
  27 use Image::Size;
  28 use Text::CSV;
  29 use CXGN::Phenotypes::StorePhenotypes;
  30 use CXGN::Phenotypes::PhenotypeMatrix;
  31 use CXGN::BrAPI::FileResponse;
  32 use CXGN::Onto;
  33 use R::YapRI::Base;
  34 use R::YapRI::Data::Matrix;
  35 use CXGN::Tag;
  36 use CXGN::DroneImagery::ImageTypes;
  37 use Time::Piece;
  38 use POSIX;
  39 use Math::Round;
  40 use List::Util qw/sum/;
  41 use Parallel::ForkManager;
  42 use CXGN::Image::Search;
  43 use CXGN::Trait::Search;
  44 #use Inline::Python;
  45
  46 BEGIN { extends 'Catalyst::Controller::REST' }
  47
  48 __PACKAGE__->config(
  49     default   => 'application/json',
  50     stash_key => 'rest',
  51     map       => { 'application/json' => 'JSON' },
  52 );
  53
  54 sub image_analysis_submit : Path('/ajax/image_analysis/submit') : ActionClass('REST') { }
  55 sub image_analysis_submit_POST : Args(0) {
  56     my $self = shift;
  57     my $c = shift;
  58     my $schema = $c->dbic_schema("Bio::Chado::Schema");
  59     my $people_schema = $c->dbic_schema("CXGN::People::Schema");
  60     my $phenome_schema = $c->dbic_schema("CXGN::Phenome::Schema");
  61     my $image_ids = decode_json $c->req->param('selected_image_ids');
  62     my $service = $c->req->param('service');
  63     my $trait = $c->req->param('trait');
  64     my ($user_id, $user_name, $user_role) = _check_user_login($c);
  65     my $main_production_site_url = $c->config->{main_production_site_url};
  66     _log_analysis_activity($c,$image_ids,$service,$trait);
  67
  68     unless (ref($image_ids) eq 'ARRAY') { $image_ids = [$image_ids]; }
  69
  70     my ($trait_name, $db_accession) = split(/\|/, $trait);
  71     myq ($db, $accession) = split(/:/, $db_accession);
  72     my ($trait_details, $record_number) = CXGN::Trait::Search->new({
  73         bcs_schema=>$schema,
  74         ontology_db_name_list => [$db],
  75         accession_list => [$accession]
  76     })->search();
  77
  78     my $image_search = CXGN::Image::Search->new({
  79         bcs_schema=>$schema,
  80         people_schema=>$people_schema,
  81         phenome_schema=>$phenome_schema,
  82         image_id_list=>$image_ids,
  83     });
  84
  85     my ($result, $records_total) = $image_search->search();
  86
  87     my @image_urls;
  88     my @image_files;
  89     foreach (@$result) {
  90         my $image = SGN::Image->new($schema->storage->dbh, $_->{image_id}, $c);
  91         my $original_img = $main_production_site_url.$image->get_image_url("original");
  92         my $image_file = $image->get_filename('original_converted', 'full');
  93         push @image_urls, $original_img;
  94         push @image_files, $image_file;
  95     }
  96     print STDERR Dumper \@image_urls;
  97
  98     my %service_details = (
  99         'necrosis' => {
 100             server_endpoint => "http://unet.mcrops.org/api/",
 101             image_type_name => "image_analysis_necrosis_solomon_nsumba",
 102         },
 103         'whitefly_count' => {
 104             server_endpoint => "http://18.216.149.204/home/api2/",
 105             image_type_name => "image_analysis_white_fly_count_solomon_nsumba",
 106         },
 107         'count_contours' => {
 108             image_type_name => "image_analysis_contours",
 109             trait_name => "count_contours",
 110             script => 'GetContours.py',
 111             input_image => 'image_path',
 112             outfile_image => 'outfile_path',
 113             results_outfile => 'results_outfile_path',
 114         },
 115         'largest_contour_percent' => {
 116             image_type_name => 'image_analysis_largest_contour',
 117             trait_name => 'percent_largest_contour',
 118             script => 'GetLargestContour.py',
 119             input_image => 'image_path',
 120             outfile_image => 'outfile_path',
 121             results_outfile => 'results_outfile_path',
 122         },
 123         'count_sift' => {
 124             image_type_name => "image_analysis_sift",
 125             trait_name => "count_sift",
 126             script => 'ImageProcess/CalculatePhenotypeSift.py',
 127             input_image => 'image_paths',
 128             outfile_image => 'outfile_paths',
 129             results_outfile => 'results_outfile_path',
 130         }
 131     );
 132
 133     my $image_type_name = $service_details{$service}->{'image_type_name'};
 134
 135     my $linking_table_type_id = SGN::Model::Cvterm->get_cvterm_row($schema, $image_type_name, 'project_md_image')->cvterm_id();
 136
 137     my $image_tag_id = CXGN::Tag::exists_tag_named($schema->storage->dbh, $image_type_name);
 138     if (!$image_tag_id) {
 139         my $image_tag = CXGN::Tag->new($schema->storage->dbh);
 140         $image_tag->set_name($image_type_name);
 141         $image_tag->set_description('Image analysis result image: '.$image_type_name);
 142         $image_tag->set_sp_person_id($user_id);
 143         $image_tag_id = $image_tag->store();
 144     }
 145     my $image_tag = CXGN::Tag->new($schema->storage->dbh, $image_tag_id);
 146     my $ua = LWP::UserAgent->new(
 147         ssl_opts => {
 148                         verify_hostname => 0,
 149                         timeout         => 60,
 150                     }
 151     );
 152     my $it = 0;
 153
 154     foreach (@image_files) {
 155         my $dir = $c->tempfiles_subdir('/'.$image_type_name);
 156         my $archive_temp_image = $c->config->{basepath}."/".$c->tempfile( TEMPLATE => $image_type_name.'/imageXXXX');
 157         $archive_temp_image .= '.png';
 158         my %res;
 159
 160         if (defined $service_details{$service}->{'server_endpoint'}) { # submit image to external service for processing
 161             print STDERR "Using endpoint ".$service_details{$service}->{'server_endpoint'}." to analyze image\n";
 162             my $resp = $ua->post(
 163                 $service_details{$service}->{'server_endpoint'},
 164                 Content_Type => 'form-data',
 165                 Content => [
 166                     image => [ $_, $_, Content_Type => 'image/png' ],
 167                 ]
 168             );
 169             if ($resp->is_success) {
 170                 my $message = $resp->decoded_content;
 171                 my $message_hashref = decode_json $message;
 172                 my $rc = getstore($message_hashref->{image_link}, $archive_temp_image);
 173                 if (is_error($rc)) {
 174                     die "getstore of ".$message_hashref->{image_link}." failed with $rc";
 175                 }
 176                 print STDERR Dumper $message_hashref;
 177                 $res{'value'} = $message_hashref->{trait_value};
 178                 $res{'trait'} = $trait;
 179                 $res{'trait_id'} = $trait_details->[0]->{trait_id};
 180             }
 181             else {
 182                 print STDERR Dumper $resp->status_line;
 183                 $res{'error'} = $resp->status_line;
 184             }
 185         }
 186         elsif (defined $service_details{$service}->{'script'}) { # supply image to local script for processing
 187             my $script = $service_details{$service}->{'script'};
 188             print STDERR "Using script $script to analyze image\n";
 189             my $input_image = $service_details{$service}->{'input_image'};
 190             my $outfile_image = $service_details{$service}->{'outfile_image'};
 191             my $results_outfile = $service_details{$service}->{'results_outfile'};
 192             my $archive_temp_results = $c->config->{basepath}."/".$c->tempfile( TEMPLATE => $image_type_name.'/imageXXXX');
 193
 194             my $cmd = $c->config->{python_executable} . ' ' . $c->config->{rootpath} .
 195                 '/DroneImageScripts/' . $script . ' --' . $input_image . ' \'' . $_ .
 196                 '\' --' . $outfile_image . ' \'' . $archive_temp_image . '\' --' .
 197                 $results_outfile . ' \'' . $archive_temp_results . '\' ';
 198             # print STDERR Dumper $cmd;
 199             my $status = system($cmd);
 200
 201             my $csv = Text::CSV->new({ sep_char => ',' });
 202             open(my $fh, '<', $archive_temp_results)
 203                 or die "Could not open file '$archive_temp_results' $!";
 204             my $line = <$fh>;
 205             my @columns;
 206             if ($csv->parse($line)) {
 207                 @columns = $csv->fields();
 208             }
 209             $res{'value'} = $columns[0];
 210             $res{'trait'} = $service_details{$service}->{'trait_name'};
 211         }
 212
 213         $res{'original_image'} = $image_urls[$it];
 214
 215         unless (defined $res{'error'}) {
 216
 217             my $image = SGN::Image->new( $schema->storage->dbh, undef, $c );
 218             my $md5 = $image->calculate_md5sum($archive_temp_image);
 219             my $stock_id = $result->[$it]->{stock_id};
 220             my $project_id = $result->[$it]->{project_id};
 221
 222             my $project_where = ' ';
 223             my $project_join = ' ';
 224             if ($project_id) {
 225                 $project_where = " AND project_md_image.type_id = $linking_table_type_id AND project_md_image.project_id = $project_id ";
 226                 $project_join = " JOIN phenome.project_md_image AS project_md_image ON(project_md_image.image_id = md_image.image_id) ";
 227             }
 228
 229             my $q = "SELECT md_image.image_id FROM metadata.md_image AS md_image
 230                 $project_join
 231                 JOIN phenome.stock_image AS stock_image ON (stock_image.image_id = md_image.image_id)
 232                 WHERE md_image.obsolete = 'f'
 233                 $project_where
 234                 AND stock_image.stock_id = $stock_id
 235                 AND md_image.md5sum = '$md5';";
 236             my $h = $schema->storage->dbh->prepare($q);
 237             $h->execute();
 238             my ($saved_image_id) = $h->fetchrow_array();
 239             my $image_id;
 240             if ($saved_image_id) {
 241                 print STDERR Dumper "Image $archive_temp_image has already been added to the database and will not be added again.";
 242                 $image = SGN::Image->new( $schema->storage->dbh, $saved_image_id, $c );
 243                 $image_id = $image->get_image_id();
 244             }
 245             else {
 246                 $image->set_sp_person_id($user_id);
 247                 if ($project_id) {
 248                     my $ret = $image->process_image($archive_temp_image, 'project', $project_id, $linking_table_type_id);
 249                     if (!$ret ) {
 250                         return {error => "Image processing for $archive_temp_image did not work. Image not associated to stock_id $stock_id.<br/><br/>"};
 251                     }
 252                     my $stock_associate = $image->associate_stock($stock_id);
 253                 }
 254                 else {
 255                     my $ret = $image->process_image($archive_temp_image, 'stock', $stock_id);
 256                     if (!$ret ) {
 257                         return {error => "Image processing for $archive_temp_image did not work. Image not associated to stock_id $stock_id.<br/><br/>"};
 258                     }
 259                 }
 260                 print STDERR "Saved $archive_temp_image\n";
 261                 $image_id = $image->get_image_id();
 262                 my $added_image_tag_id = $image->add_tag($image_tag);
 263             }
 264
 265             $res{'analyzed_image_id'} = $image_id;
 266             $res{'image_link'} = $image->get_image_url("original");
 267         }
 268
 269         $result->[$it]->{result} = \%res;
 270         $it++;
 271     }
 272
 273     # print STDERR "Before grouping result is: ".Dumper($result);
 274
 275     $c->stash->{rest} = { success => 1, results => $result };
 276 }
 277
 278 sub image_analysis_group : Path('/ajax/image_analysis/group') : ActionClass('REST') { }
 279 sub image_analysis_group_POST : Args(0) {
 280     my $self = shift;
 281     my $c = shift;
 282     my $result = decode_json $c->req->param('result');
 283     # print STDERR Dumper($result);
 284     my %grouped_results = ();
 285     my @table_data = ();
 286
 287     my ($uniquename, $next_uniquename, $trait, $value, $results_ref, $next_results_ref);
 288     # sort result hash array by $stock_id
 289     my @sorted_result = sort {$$a{"stock_id"} <=> $$b{"stock_id"} } @{$result};
 290     # my $old_uniquename = $sorted_result[0]->{'stock_uniquename'};
 291     $grouped_results{$sorted_result[0]->{'stock_uniquename'}}{$sorted_result[0]->{'result'}->{'trait'}} = [];
 292
 293     for (my $i = 0; $i <= $#sorted_result; $i++) {
 294         $results_ref = $sorted_result[$i];
 295         # print STDERR "\n\nResults ref is ".Dumper($results_ref)."\n\n";
 296         $uniquename = $results_ref->{'stock_uniquename'};
 297         $trait = $results_ref->{'result'}->{'trait'};
 298         $value = $results_ref->{'result'}->{'value'};
 299
 300         if ($trait && $value) {
 301             print STDERR "Working on $trait for $uniquename. Saving the details \n";
 302             push @{$grouped_results{$uniquename}{$trait}}, {
 303                         stock_id => $results_ref->{'stock_id'},
 304                         collector => $results_ref->{'image_username'},
 305                         original_link => $results_ref->{'result'}->{'original_image'},
 306                         analyzed_link => $results_ref->{'result'}->{'image_link'},
 307                         image_name => $results_ref->{'image_original_filename'}.$results_ref->{'image_file_ext'},
 308                         trait_id => $results_ref->{'result'}->{'trait_id'},
 309                         value => $value + 0
 310                 };
 311         }
 312         else { # if no result returned for an image, include it with error details.
 313             print STDERR "No usable analysis data in this results_ref \n";
 314             push @{$grouped_results{$uniquename}{$trait}}, {
 315                         stock_id => $results_ref->{'stock_id'},
 316                         collector => $results_ref->{'image_username'},
 317                         original_link => $results_ref->{'result'}->{'original_image'},
 318                         analyzed_link => 'Error: ' . $results_ref->{'result'}->{'error'},
 319                         image_name => $results_ref->{'image_original_filename'}.$results_ref->{'image_file_ext'},
 320                         trait_id => $results_ref->{'result'}->{'trait_id'},
 321                         value => 'NA'
 322                 };
 323         }
 324
 325         $next_results_ref = $sorted_result[$i+1];
 326         $next_uniquename = $next_results_ref->{'stock_uniquename'};
 327
 328         if ($next_uniquename ne $uniquename) {
 329
 330             print STDERR "Calculating mean value for $uniquename\n";
 331
 332             my $uniquename_data = $grouped_results{$uniquename};
 333
 334             foreach my $trait (keys %{$uniquename_data}) {
 335                 my $details = $uniquename_data->{$trait};
 336                 my @values = map { $_->{'value'}} @{$uniquename_data->{$trait}};
 337                 @values= grep { $_ ne 'NA' } @values; # remove NAs before calculating mean
 338                 # print STDERR "\n\n\nVALUES ARE @values and length is ". scalar @values . "\n\n\n";
 339                 my $mean_value = @values ? sprintf("%.2f", sum(@values)/@values) : undef;
 340                 print STDERR "Mean value is $mean_value\n";
 341                 push @table_data, {
 342                     observationUnitDbId => $uniquename_data->{$trait}[0]->{'stock_id'},
 343                     observationUnitName => $uniquename,
 344                     collector => $uniquename_data->{$trait}[0]->{'collector'},
 345                     observationTimeStamp => localtime()->datetime,
 346                     observationVariableDbId => $uniquename_data->{$trait}[0]->{'trait_id'},
 347                     observationVariableName => $trait,
 348                     value => $mean_value,
 349                     details => $details,
 350                     numberAnalyzed => scalar @values
 351                     # Add previously observed trait value
 352                 };
 353             }
 354         }
 355     }
 356     # print STDERR "table data is ".Dumper(@table_data);
 357     $c->stash->{rest} = { success => 1, results => \@table_data };
 358 }
 359
 360 sub _check_user_login {
 361     my $c = shift;
 362     my $user_id;
 363     my $user_name;
 364     my $user_role;
 365     my $session_id = $c->req->param("sgn_session_id");
 366
 367     if ($session_id){
 368         my $dbh = $c->dbc->dbh;
 369         my @user_info = CXGN::Login->new($dbh)->query_from_cookie($session_id);
 370         if (!$user_info[0]){
 371             $c->stash->{rest} = {error=>'You must be logged in to do this!'};
 372             $c->detach();
 373         }
 374         $user_id = $user_info[0];
 375         $user_role = $user_info[1];
 376         my $p = CXGN::People::Person->new($dbh, $user_id);
 377         $user_name = $p->get_username;
 378     } else{
 379         if (!$c->user){
 380             $c->stash->{rest} = {error=>'You must be logged in to do this!'};
 381             $c->detach();
 382         }
 383         $user_id = $c->user()->get_object()->get_sp_person_id();
 384         $user_name = $c->user()->get_object()->get_username();
 385         $user_role = $c->user->get_object->get_user_type();
 386     }
 387     return ($user_id, $user_name, $user_role);
 388 }
 389
 390 sub _log_analysis_activity {
 391     my $c = shift;
 392     my $image_ids = shift;
 393     my $service = shift;
 394     my $trait = shift;
 395     my $now = DateTime->now();
 396
 397     if ($c->config->{image_analysis_log}) {
 398       my $logfile = $c->config->{image_analysis_log};
 399       open (my $F, ">> :encoding(UTF-8)", $logfile) || die "Can't open logfile $logfile\n";
 400       print $F join("\t", (
 401             $c->user->get_object->get_username(),
 402             $service,
 403             $trait,
 404             $image_ids,
 405             $now->year()."-".$now->month()."-".$now->day()." ".$now->hour().":".$now->minute()));
 406       print $F "\n";
 407       close($F);
 408       print STDERR "Analysis submission logged in $logfile\n";
 409     }
 410     else {
 411       print STDERR "Note: set config variable image_analysis_log to obtain a log and graph of image analysis activity.\n";
 412     }
 413 }
 414
 415 1;