lib/SGN/Controller/AJAX/ImageAnalysis.pm

   1
   2 =head1 NAME
   3
   4 SGN::Controller::AJAX::ImageAnalysis - a REST controller class to provide image analysis including
   5 functions for necrosis image analysis https://github.com/solomonnsumba/Necrosis-_Web_Server
   6
   7 =head1 DESCRIPTION
   8
   9 =head1 AUTHOR
  10
  11 =cut
  12
  13 package SGN::Controller::AJAX::ImageAnalysis;
  14
  15 use Moose;
  16 use Data::Dumper;
  17 use LWP::UserAgent;
  18 use LWP::Simple;
  19 use JSON;
  20 use SGN::Model::Cvterm;
  21 use DateTime;
  22 use CXGN::UploadFile;
  23 use SGN::Image;
  24 use CXGN::DroneImagery::ImagesSearch;
  25 use URI::Encode qw(uri_encode uri_decode);
  26 use CXGN::Calendar;
  27 use Image::Size;
  28 use Text::CSV;
  29 use CXGN::Phenotypes::StorePhenotypes;
  30 use CXGN::Phenotypes::PhenotypeMatrix;
  31 use CXGN::BrAPI::FileResponse;
  32 use CXGN::Onto;
  33 use R::YapRI::Base;
  34 use R::YapRI::Data::Matrix;
  35 use CXGN::Tag;
  36 use CXGN::DroneImagery::ImageTypes;
  37 use Time::Piece;
  38 use POSIX;
  39 use Math::Round;
  40 use List::Util qw/sum/;
  41 use Parallel::ForkManager;
  42 use CXGN::Image::Search;
  43 use CXGN::Trait::Search;
  44 use File::Slurp;
  45 #use Inline::Python;
  46
  47 BEGIN { extends 'Catalyst::Controller::REST' }
  48
  49 __PACKAGE__->config(
  50     default   => 'application/json',
  51     stash_key => 'rest',
  52     map       => { 'application/json' => 'JSON' },
  53 );
  54
  55 sub image_analysis_submit : Path('/ajax/image_analysis/submit') : ActionClass('REST') { }
  56 sub image_analysis_submit_POST : Args(0) {
  57     my $self = shift;
  58     my $c = shift;
  59     my $schema = $c->dbic_schema("Bio::Chado::Schema");
  60     my $people_schema = $c->dbic_schema("CXGN::People::Schema");
  61     my $phenome_schema = $c->dbic_schema("CXGN::Phenome::Schema");
  62     my $image_ids = decode_json $c->req->param('selected_image_ids');
  63     my $service = $c->req->param('service');
  64     my $trait = $c->req->param('trait');
  65     my ($user_id, $user_name, $user_role) = _check_user_login($c);
  66     my $main_production_site_url = $c->config->{main_production_site_url};
  67     _log_analysis_activity($c,$image_ids,$service,$trait);
  68
  69     unless (ref($image_ids) eq 'ARRAY') { $image_ids = [$image_ids]; }
  70
  71     my ($trait_name, $db_accession) = split(/\|/, $trait);
  72     my ($db, $accession) = split(/:/, $db_accession);
  73     my ($trait_details, $record_number) = CXGN::Trait::Search->new({
  74         bcs_schema=>$schema,
  75         ontology_db_name_list => [$db],
  76         accession_list => [$accession]
  77     })->search();
  78
  79     my $image_search = CXGN::Image::Search->new({
  80         bcs_schema=>$schema,
  81         people_schema=>$people_schema,
  82         phenome_schema=>$phenome_schema,
  83         image_id_list=>$image_ids,
  84     });
  85
  86     my ($result, $records_total) = $image_search->search();
  87
  88     my @image_urls;
  89     my @image_files;
  90     foreach (@$result) {
  91         my $image = SGN::Image->new($schema->storage->dbh, $_->{image_id}, $c);
  92         my $original_img = $main_production_site_url.$image->get_image_url("original");
  93         my $image_file = $image->get_filename('original_converted', 'full');
  94         push @image_urls, $original_img;
  95         push @image_files, $image_file;
  96     }
  97     print STDERR Dumper \@image_urls;
  98
  99     my %service_details = (
 100         'necrosis' => {
 101             server_endpoint => "http://unet.mcrops.org/api/",
 102             image_type_name => "image_analysis_necrosis_solomon_nsumba",
 103         },
 104         'whitefly_count' => {
 105             server_endpoint => "http://18.216.149.204/home/api2/",
 106             image_type_name => "image_analysis_white_fly_count_solomon_nsumba",
 107         },
 108         'count_contours' => {
 109             image_type_name => "image_analysis_contours",
 110             trait_name => "count_contours",
 111             script => 'GetContours.py',
 112             input_image => 'image_path',
 113             outfile_image => 'outfile_path',
 114             results_outfile => 'results_outfile_path',
 115         },
 116         'largest_contour_percent' => {
 117             image_type_name => 'image_analysis_largest_contour',
 118             trait_name => 'percent_largest_contour',
 119             script => 'GetLargestContour.py',
 120             input_image => 'image_path',
 121             outfile_image => 'outfile_path',
 122             results_outfile => 'results_outfile_path',
 123         },
 124         'count_sift' => {
 125             image_type_name => "image_analysis_sift",
 126             trait_name => "count_sift",
 127             script => 'ImageProcess/CalculatePhenotypeSift.py',
 128             input_image => 'image_paths',
 129             outfile_image => 'outfile_paths',
 130             results_outfile => 'results_outfile_path',
 131         }
 132     );
 133
 134     my $image_type_name = $service_details{$service}->{'image_type_name'};
 135
 136     my $linking_table_type_id = SGN::Model::Cvterm->get_cvterm_row($schema, $image_type_name, 'project_md_image')->cvterm_id();
 137
 138     my $image_tag_id = CXGN::Tag::exists_tag_named($schema->storage->dbh, $image_type_name);
 139     if (!$image_tag_id) {
 140         my $image_tag = CXGN::Tag->new($schema->storage->dbh);
 141         $image_tag->set_name($image_type_name);
 142         $image_tag->set_description('Image analysis result image: '.$image_type_name);
 143         $image_tag->set_sp_person_id($user_id);
 144         $image_tag_id = $image_tag->store();
 145     }
 146     my $image_tag = CXGN::Tag->new($schema->storage->dbh, $image_tag_id);
 147     my $ua = LWP::UserAgent->new(
 148         ssl_opts => {
 149                         verify_hostname => 0,
 150                         timeout         => 60,
 151                     }
 152     );
 153     my $it = 0;
 154
 155     foreach (@image_files) {
 156         my $dir = $c->tempfiles_subdir('/'.$image_type_name);
 157         my $archive_temp_image = $c->config->{basepath}."/".$c->tempfile( TEMPLATE => $image_type_name.'/imageXXXX');
 158         $archive_temp_image .= '.png';
 159         my %res;
 160
 161         if (defined $service_details{$service}->{'server_endpoint'}) { # submit image to external service for processing
 162             print STDERR "Using endpoint ".$service_details{$service}->{'server_endpoint'}." to analyze image\n";
 163             my $resp = $ua->post(
 164                 $service_details{$service}->{'server_endpoint'},
 165                 Content_Type => 'form-data',
 166                 Content => [
 167                     image => [ $_, $_, Content_Type => 'image/png' ],
 168                 ]
 169             );
 170             if ($resp->is_success) {
 171                 my $message = $resp->decoded_content;
 172                 my $message_hashref = decode_json $message;
 173                 my $rc = getstore($message_hashref->{image_link}, $archive_temp_image);
 174                 if (is_error($rc)) {
 175                     die "getstore of ".$message_hashref->{image_link}." failed with $rc";
 176                 }
 177                 print STDERR Dumper $message_hashref;
 178                 $res{'value'} = $message_hashref->{trait_value};
 179                 $res{'analysis_info'} = $message_hashref->{info};
 180                 $res{'trait'} = $trait;
 181                 $res{'trait_id'} = $trait_details->[0]->{trait_id};
 182             }
 183             else {
 184                 print STDERR Dumper $resp->status_line;
 185                 $res{'error'} = $resp->status_line;
 186             }
 187         }
 188         elsif (defined $service_details{$service}->{'script'}) { # supply image to local script for processing
 189             my $script = $service_details{$service}->{'script'};
 190             print STDERR "Using script $script to analyze image\n";
 191             my $input_image = $service_details{$service}->{'input_image'};
 192             my $outfile_image = $service_details{$service}->{'outfile_image'};
 193             my $results_outfile = $service_details{$service}->{'results_outfile'};
 194             my $archive_temp_results = $c->config->{basepath}."/".$c->tempfile( TEMPLATE => $image_type_name.'/imageXXXX');
 195
 196             my $cmd = $c->config->{python_executable} . ' ' . $c->config->{rootpath} .
 197                 '/DroneImageScripts/' . $script . ' --' . $input_image . ' \'' . $_ .
 198                 '\' --' . $outfile_image . ' \'' . $archive_temp_image . '\' --' .
 199                 $results_outfile . ' \'' . $archive_temp_results . '\' ';
 200             # print STDERR Dumper $cmd;
 201             my $status = system($cmd);
 202
 203             my $csv = Text::CSV->new({ sep_char => ',' });
 204             open(my $fh, '<', $archive_temp_results)
 205                 or die "Could not open file '$archive_temp_results' $!";
 206             my $line = <$fh>;
 207             my @columns;
 208             if ($csv->parse($line)) {
 209                 @columns = $csv->fields();
 210             }
 211             $res{'value'} = $columns[0];
 212             $res{'trait'} = $service_details{$service}->{'trait_name'};
 213         }
 214
 215         $res{'original_image'} = $image_urls[$it];
 216
 217         unless (defined $res{'error'}) {
 218
 219             my $image = SGN::Image->new( $schema->storage->dbh, undef, $c );
 220             my $md5 = $image->calculate_md5sum($archive_temp_image);
 221             my $stock_id = $result->[$it]->{stock_id};
 222             my $project_id = $result->[$it]->{project_id};
 223
 224             my $project_where = ' ';
 225             my $project_join = ' ';
 226             if ($project_id) {
 227                 $project_where = " AND project_md_image.type_id = $linking_table_type_id AND project_md_image.project_id = $project_id ";
 228                 $project_join = " JOIN phenome.project_md_image AS project_md_image ON(project_md_image.image_id = md_image.image_id) ";
 229             }
 230
 231             my $q = "SELECT md_image.image_id FROM metadata.md_image AS md_image
 232                 $project_join
 233                 JOIN phenome.stock_image AS stock_image ON (stock_image.image_id = md_image.image_id)
 234                 WHERE md_image.obsolete = 'f'
 235                 $project_where
 236                 AND stock_image.stock_id = $stock_id
 237                 AND md_image.md5sum = '$md5';";
 238             my $h = $schema->storage->dbh->prepare($q);
 239             $h->execute();
 240             my ($saved_image_id) = $h->fetchrow_array();
 241             my $image_id;
 242             if ($saved_image_id) {
 243                 print STDERR Dumper "Image $archive_temp_image has already been added to the database and will not be added again.";
 244                 $image = SGN::Image->new( $schema->storage->dbh, $saved_image_id, $c );
 245                 $image_id = $image->get_image_id();
 246             }
 247             else {
 248                 $image->set_sp_person_id($user_id);
 249                 if ($project_id) {
 250                     my $ret = $image->process_image($archive_temp_image, 'project', $project_id, $linking_table_type_id);
 251                     if (!$ret ) {
 252                         return {error => "Image processing for $archive_temp_image did not work. Image not associated to stock_id $stock_id.<br/><br/>"};
 253                     }
 254                     my $stock_associate = $image->associate_stock($stock_id);
 255                 }
 256                 else {
 257                     my $ret = $image->process_image($archive_temp_image, 'stock', $stock_id);
 258                     if (!$ret ) {
 259                         return {error => "Image processing for $archive_temp_image did not work. Image not associated to stock_id $stock_id.<br/><br/>"};
 260                     }
 261                 }
 262                 print STDERR "Saved $archive_temp_image\n";
 263                 $image_id = $image->get_image_id();
 264                 my $added_image_tag_id = $image->add_tag($image_tag);
 265             }
 266
 267             $res{'analyzed_image_id'} = $image_id;
 268             $res{'image_link'} = $image->get_image_url("original");
 269         }
 270
 271         $result->[$it]->{result} = \%res;
 272         $it++;
 273     }
 274
 275     # print STDERR "Before grouping result is: ".Dumper($result);
 276
 277     $c->stash->{rest} = { success => 1, results => $result };
 278 }
 279
 280 sub image_analysis_group : Path('/ajax/image_analysis/group') : ActionClass('REST') { }
 281 sub image_analysis_group_POST : Args(0) {
 282     my $self = shift;
 283     my $c = shift;
 284     my $result = decode_json $c->req->param('result');
 285     # print STDERR Dumper($result);
 286     my %grouped_results = ();
 287     my @table_data = ();
 288
 289     my ($uniquename, $next_uniquename, $trait, $value, $results_ref, $next_results_ref);
 290     # sort result hash array by $stock_id
 291     my @sorted_result = sort {$$a{"stock_id"} <=> $$b{"stock_id"} } @{$result};
 292     # my $old_uniquename = $sorted_result[0]->{'stock_uniquename'};
 293     $grouped_results{$sorted_result[0]->{'stock_uniquename'}}{$sorted_result[0]->{'result'}->{'trait'}} = [];
 294
 295     for (my $i = 0; $i <= $#sorted_result; $i++) {
 296         $results_ref = $sorted_result[$i];
 297         # print STDERR "\n\nResults ref is ".Dumper($results_ref)."\n\n";
 298         $uniquename = $results_ref->{'stock_uniquename'};
 299         $trait = $results_ref->{'result'}->{'trait'};
 300         $value = $results_ref->{'result'}->{'value'};
 301
 302         if ($trait && $value) {
 303             print STDERR "Working on $trait for $uniquename. Saving the details \n";
 304             push @{$grouped_results{$uniquename}{$trait}}, {
 305                         stock_id => $results_ref->{'stock_id'},
 306                         collector => $results_ref->{'image_username'},
 307                         original_link => $results_ref->{'result'}->{'original_image'},
 308                         analyzed_link => $results_ref->{'result'}->{'image_link'},
 309                         image_name => $results_ref->{'image_original_filename'}.$results_ref->{'image_file_ext'},
 310                         trait_id => $results_ref->{'result'}->{'trait_id'},
 311                         value => $value + 0
 312                 };
 313         }
 314         else { # if no result returned for an image, include it with error details.
 315             print STDERR "No usable analysis data in this results_ref \n";
 316             push @{$grouped_results{$uniquename}{$trait}}, {
 317                         stock_id => $results_ref->{'stock_id'},
 318                         collector => $results_ref->{'image_username'},
 319                         original_link => $results_ref->{'result'}->{'original_image'},
 320                         analyzed_link => 'Error: ' . $results_ref->{'result'}->{'error'},
 321                         image_name => $results_ref->{'image_original_filename'}.$results_ref->{'image_file_ext'},
 322                         trait_id => $results_ref->{'result'}->{'trait_id'},
 323                         value => 'NA'
 324                 };
 325         }
 326
 327         $next_results_ref = $sorted_result[$i+1];
 328         $next_uniquename = $next_results_ref->{'stock_uniquename'};
 329
 330         if ($next_uniquename ne $uniquename) {
 331
 332             print STDERR "Calculating mean value for $uniquename\n";
 333
 334             my $uniquename_data = $grouped_results{$uniquename};
 335
 336             foreach my $trait (keys %{$uniquename_data}) {
 337                 my $details = $uniquename_data->{$trait};
 338                 my @values = map { $_->{'value'}} @{$uniquename_data->{$trait}};
 339                 @values= grep { $_ ne 'NA' } @values; # remove NAs before calculating mean
 340                 # print STDERR "\n\n\nVALUES ARE @values and length is ". scalar @values . "\n\n\n";
 341                 my $mean_value = @values ? sprintf("%.2f", sum(@values)/@values) : undef;
 342                 print STDERR "Mean value is $mean_value\n";
 343                 push @table_data, {
 344                     observationUnitDbId => $uniquename_data->{$trait}[0]->{'stock_id'},
 345                     observationUnitName => $uniquename,
 346                     collector => $uniquename_data->{$trait}[0]->{'collector'},
 347                     observationTimeStamp => localtime()->datetime,
 348                     observationVariableDbId => $uniquename_data->{$trait}[0]->{'trait_id'},
 349                     observationVariableName => $trait,
 350                     value => $mean_value,
 351                     details => $details,
 352                     numberAnalyzed => scalar @values
 353                     # Add previously observed trait value
 354                 };
 355             }
 356         }
 357     }
 358     # print STDERR "table data is ".Dumper(@table_data);
 359     $c->stash->{rest} = { success => 1, results => \@table_data };
 360 }
 361
 362 sub get_activity_data : Path('/ajax/image_analysis/activity') Args(0) {
 363   my $self = shift;
 364   my $c = shift;
 365
 366   my @activity;
 367   my $logfile = $c->config->{image_analysis_log};
 368   if (-e $logfile) {
 369         my @file_data = read_file($logfile, chomp => 1);
 370         foreach my $line (@file_data) {
 371             my @values = split("\t", $line);
 372             my @ts_parts = split(" ", $values[0]);
 373             push @activity, { date => $ts_parts[0]};
 374         }
 375     } else {
 376         $c->stash->{rest} = {error=>'No activity log set up.'};
 377         $c->detach();
 378     }
 379
 380   my $json = JSON->new();
 381   $c->stash->{rest} = { activity => $json->encode(\@activity)};
 382
 383 }
 384
 385 sub _check_user_login {
 386     my $c = shift;
 387     my $user_id;
 388     my $user_name;
 389     my $user_role;
 390     my $session_id = $c->req->param("sgn_session_id");
 391
 392     if ($session_id){
 393         my $dbh = $c->dbc->dbh;
 394         my @user_info = CXGN::Login->new($dbh)->query_from_cookie($session_id);
 395         if (!$user_info[0]){
 396             $c->stash->{rest} = {error=>'You must be logged in to do this!'};
 397             $c->detach();
 398         }
 399         $user_id = $user_info[0];
 400         $user_role = $user_info[1];
 401         my $p = CXGN::People::Person->new($dbh, $user_id);
 402         $user_name = $p->get_username;
 403     } else{
 404         if (!$c->user){
 405             $c->stash->{rest} = {error=>'You must be logged in to do this!'};
 406             $c->detach();
 407         }
 408         $user_id = $c->user()->get_object()->get_sp_person_id();
 409         $user_name = $c->user()->get_object()->get_username();
 410         $user_role = $c->user->get_object->get_user_type();
 411     }
 412     return ($user_id, $user_name, $user_role);
 413 }
 414
 415 sub _log_analysis_activity {
 416     my $c = shift;
 417     my $image_ids = shift;
 418     my $service = shift;
 419     my $trait = shift;
 420     my $now = DateTime->now();
 421
 422     if ($c->config->{image_analysis_log}) {
 423       my $logfile = $c->config->{image_analysis_log};
 424       open (my $F, ">> :encoding(UTF-8)", $logfile) || die "Can't open logfile $logfile\n";
 425       print $F join("\t", (
 426             $now->year()."-".$now->month()."-".$now->day()." ".$now->hour().":".$now->minute(),
 427             $c->user->get_object->get_username(),
 428             $service,
 429             $trait,
 430             $image_ids
 431             ));
 432       print $F "\n";
 433       close($F);
 434       print STDERR "Analysis submission logged in $logfile\n";
 435     }
 436     else {
 437       print STDERR "Note: set config variable image_analysis_log to obtain a log and graph of image analysis activity.\n";
 438     }
 439 }
 440
 441 1;