bin/load_images.pl

   1 #!/usr/bin/perl
   2
   3 =head1 NAME
   4
   5 load_images.pl
   6
   7 =head1 SYNOPSYS
   8
   9 load_images.pl -D database_name -H hostname -i dirname -r chado table name [script will load image ids into ChadoTableprop ]
  10
  11 =head1 DESCRIPTION
  12
  13 Loads  images  into the SGN database, using the SGN::Image framework.
  14 Then link the loaded image with the user-supplied chado objects (e.g. stock, nd_experiment)
  15
  16 Requires the following parameters:
  17
  18 =over 8
  19
  20 =item -D
  21
  22 database name
  23
  24 =item -H
  25
  26 host name
  27
  28 =item -m
  29
  30 map file. If provided links between stock names - image file name , is read from a mapping file.
  31 Row labels are expected to be unique file names, column header for the associated stocks is 'name'
  32
  33 =item -i
  34
  35 a dirname that contains image filenames or subdirectories named after database accessions, containing one or more images (see option -d) .
  36
  37 =item -u
  38
  39 use name - from sgn_people.sp_person.
  40
  41 =item -b
  42
  43 the dir where the database stores the images (the concatenated values from image_path and image_dir from sgn_local.conf or sgn.conf)
  44
  45 =item -d
  46
  47 files are stored in sub directories named after database accessions
  48
  49 =item -e
  50
  51 image file extension. Defaults to 'jpg'
  52
  53 =item -t
  54
  55 trial mode . Nothing will be stored.
  56
  57 =back
  58
  59 Errors and messages are output on STDERR.
  60
  61 =head1 AUTHOR(S)
  62
  63 Naama Menda (nm249@cornell.edu) October 2010.
  64
  65 Tweaks and move to sgn/bin: Lukas Mueller (lam87@cornell.edu) December 2023.
  66
  67 =cut
  68
  69 use strict;
  70
  71 use CXGN::Metadata::Schema;
  72 use CXGN::Metadata::Metadbdata;
  73 use CXGN::DB::InsertDBH;
  74 use CXGN::Image;
  75 use Bio::Chado::Schema;
  76 use CXGN::People::Person;
  77 use Carp qw /croak/;
  78 use Data::Dumper qw / Dumper /;
  79
  80 use File::Basename;
  81 use SGN::Context;
  82 use Getopt::Std;
  83
  84 use CXGN::Tools::File::Spreadsheet;
  85 use File::Glob qw | bsd_glob |;
  86
  87 our ($opt_H, $opt_D, $opt_t, $opt_i, $opt_u, $opt_r, $opt_d, $opt_e, $opt_m, $opt_b);
  88 getopts('H:D:u:i:e:f:tdr:m:b:');
  89
  90 my $dbhost = $opt_H;
  91 my $dbname = $opt_D;
  92 my $dirname = $opt_i;
  93 my $sp_person=$opt_u;
  94 my $db_image_dir = $opt_b;
  95 my $chado_table = $opt_r;
  96 my $ext = $opt_e || 'jpg';
  97
  98 if (!$dbhost && !$dbname) {
  99     print "dbhost = $dbhost , dbname = $dbname\n";
 100     print "opt_t = $opt_t, opt_u = $opt_u, opt_r = $chado_table, opt_i = $dirname\n";
 101     usage();
 102 }
 103
 104 if (!$dirname) { print "dirname = $dirname\n" ; usage(); }
 105
 106 my $dbh = CXGN::DB::InsertDBH->new( { dbhost=>$dbhost,
 107                                       dbname=>$dbname,
 108                                     } );
 109
 110 my $schema= Bio::Chado::Schema->connect(  sub { $dbh->get_actual_dbh() } ,  { on_connect_do => ['SET search_path TO  public;'] }
 111     );
 112
 113
 114 print STDERR "Generate metadata_id... ";
 115 my $metadata_schema = CXGN::Metadata::Schema->connect("dbi:Pg:database=$dbname;host=".$dbh->dbhost(), "postgres", $dbh->dbpass(), {on_connect_do => "SET search_path TO 'metadata', 'public'", });
 116
 117 my $sp_person_id= CXGN::People::Person->get_person_by_username($dbh, $sp_person);
 118 my %name2id = ();
 119
 120
 121 #my $ch = SGN::Context->new();
 122 print "PLEASE VERIFY:\n";
 123 print "Using dbhost: $dbhost. DB name: $dbname. \n";
 124 print "Path to image is: $db_image_dir\n";
 125 print "CONTINUE? ";
 126 my $a = (<STDIN>);
 127 if ($a !~ /[yY]/) { exit(); }
 128
 129 my %image_hash = ();  # used to retrieve images that are already loaded
 130 my %connections = (); # keep track of object -- image connections that have already been made.
 131
 132 print STDERR "Caching stock table...\n";
 133 my $object_rs = $schema->resultset("Stock::Stock")->search( { } ) ;
 134 while (my $object = $object_rs->next ) {
 135     my $id = $object->stock_id;
 136     my $name = $object->uniquename;
 137     $name2id{lc($name)} = $id;
 138 }
 139
 140 # cache image chado object - image links to prevent reloading of the
 141 # same data
 142 #
 143 print "Caching image $chado_table links...\n";
 144
 145 my $q = "SELECT * FROM phenome.stock_image";
 146 my $sth = $dbh->prepare($q);
 147 $sth->execute();
 148 while ( my $hashref = $sth->fetchrow_hashref() ) {
 149     my $image_id = $hashref->{image_id};
 150     my $chado_table_id = $hashref->{stock_id};  ##### table specific
 151
 152     if ($chado_table_id % 10000 == 0) {
 153         print STDERR "CACHING $chado_table_id\n";
 154     }
 155
 156     my $i = CXGN::Image->new(dbh=>$dbh, image_id=>$image_id, image_dir=>$db_image_dir); # SGN::Image...$ch
 157     my $original_filename = $i->get_original_filename();
 158     $image_hash{$original_filename} = $i; # this doesn't have the file extension
 159     $connections{$image_id."-".$chado_table_id}++;
 160 }
 161
 162 #open (ERR, ">load_bcs_images.err") || die "Can't open error file\n";
 163
 164 my @files;
 165 if (! $opt_d) {
 166     @files = bsd_glob "$dirname/*.$ext";
 167 }
 168 else {
 169     @files = bsd_glob "$dirname/*" if $opt_d ;
 170 }
 171
 172 print STDERR "DIRS = ".(join("\n", @files))."\n";
 173
 174 my @sub_files;
 175
 176 my $new_image_count = 0;
 177
 178 my $metadata = CXGN::Metadata::Metadbdata->new($metadata_schema, $sp_person);
 179 my $metadata_id = $metadata->store()->get_metadata_id();
 180
 181 #read from spreadsheet:
 182 my $map_file = $opt_m; #
 183 my %name_map;
 184
 185 if ($opt_m) {
 186     my $s = CXGN::Tools::File::Spreadsheet->new($map_file); #
 187     my @rows = $s->row_labels(); #
 188     foreach my $file_name (@rows) { #
 189         my $stock_name = $s->value_at($file_name, 'name'); #
 190         $name_map{$file_name} = $stock_name;
 191     }
 192 }
 193
 194 print STDERR "Starting to process ".scalar(@files)." images...\n";
 195
 196 foreach my $file (@files) {
 197     eval {
 198         chomp($file);
 199         @sub_files = ($file);
 200         @sub_files =  bsd_glob "$file/*"; # if $opt_d;
 201
 202         print STDERR "FILES FOR $file: ".Dumper(\@sub_files)."\n";
 203
 204         my $object =  basename($file, ".$ext" );
 205
 206 #       if (!$plot) { die "File $file has no object name in it!"; }
 207         my $stock = $schema->resultset("Stock::Stock")->find( {
 208             stock_id => $name2id{ lc($object) }  } );
 209         foreach my $filename (@sub_files) {
 210
 211             chomp $filename;
 212
 213             print STDERR "FILENAME NOW: $filename\n";
 214             my $image_base = basename($filename);
 215             my ($object_name, $description, $extension);
 216             if ($opt_m) {
 217                 $object_name = $name_map{$object . "." . $ext } ;
 218             }
 219
 220             print STDERR "OBJECT = $object...\n";
 221 #           if ($image_base =~ /(.*?)\_(.*?)(\..*?)?$/) {
 222             if ($image_base =~ m/(.*)(\.$ext)/i) {
 223                 $extension = $2;
 224                 $image_base = $1;
 225             }
 226             if ($image_base =~ m/(.*)\_(.*)/)  {
 227                 $object_name = $1;
 228                 $description = $2;
 229
 230             }
 231             else {
 232                 $object_name = $image_base;
 233             }
 234             print STDERR "Object: $object OBJECT NAME: $object_name DESCRPTION: $description EXTENSIO: $extension\n";
 235
 236
 237             print STDOUT "Processing file $file...\n";
 238             print STDOUT "Loading $object_name, image $filename\n";
 239             print STDERR "Loading $object_name, image $filename\n";
 240             my $image_id; # this will be set later, depending if the image is new or not
 241             if (! -e $filename) {
 242                 warn "The specified file $filename does not exist! Skipping...\n";
 243                 next();
 244             }
 245
 246             if (!exists($name2id{lc($object)})) {
 247                 message ("$object does not exist in the database...\n");
 248             }
 249
 250             else {
 251                 print STDERR "Adding $filename...\n";
 252                 if (exists($image_hash{$filename})) {
 253                     print STDERR "$filename is already loaded into the database...\n";
 254                     $image_id = $image_hash{$filename}->get_image_id();
 255                     $connections{$image_id."-".$name2id{lc($object)}}++;
 256                     if ($connections{$image_id."-".$name2id{lc($object)}} > 1) {
 257                         print STDERR "The connection between $object and image $filename has already been made. Skipping...\n";
 258                     }
 259                     elsif ($image_hash{$filename}) {
 260                         print STDERR qq  { Associating $chado_table $name2id{lc($object)} with already loaded image $filename...\n };
 261                     }
 262                 }
 263                 else {
 264                     print STDERR qq { Generating new image object for image $filename and associating it with $chado_table $object, id $name2id{lc($object) } ...\n };
 265
 266                     if ($opt_t)  {
 267                         print STDOUT qq { Would associate file $filename to $chado_table $object_name, id $name2id{lc($object)}\n };
 268                         $new_image_count++;
 269                     }
 270                     else {
 271                         my $image = CXGN::Image->new(dbh=>$dbh, image_dir=>$db_image_dir);
 272                         $image_hash{$filename}=$image;
 273
 274                         my $error;
 275                         ($image_id, $error) = $image->process_image("$filename", $chado_table , $name2id{lc($object)}, 1);
 276
 277                         print STDERR "IMAGE ID $image_id, ERROR: $error\n";
 278
 279                         if ($error eq "ok") {
 280
 281
 282                             $image->set_description("$description");
 283                             $image->set_name(basename($filename , ".$ext"));
 284                             $image->set_sp_person_id($sp_person_id);
 285                             $image->set_obsolete("f");
 286                             $image_id = $image->store();
 287                             #link the image with the BCS object
 288                             $new_image_count++;
 289                             my $image_subpath = $image->image_subpath();
 290                             print STDERR "FINAL IMAGE PATH = $db_image_dir/$image_subpath\n";
 291                         }
 292                     }
 293                 }
 294             }
 295
 296
 297             print STDERR "Connecting image $filename and id $image_id with stock ".$stock->stock_id()."\n";
 298             #store the image_id - stock_id link
 299             my $q = "INSERT INTO phenome.stock_image (stock_id, image_id, metadata_id) VALUES (?,?,?)";
 300             my $sth  = $dbh->prepare($q);
 301             $sth->execute($stock->stock_id, $image_id, $metadata_id);
 302         }
 303     };
 304     if ($@) {
 305         print STDERR "ERROR OCCURRED WHILE SAVING NEW INFORMATION. $@\n";
 306         $dbh->rollback();
 307     }
 308     else {
 309         $dbh->commit();
 310     }
 311 }
 312
 313
 314
 315
 316 #close(ERR);
 317 close(F);
 318
 319
 320
 321
 322 print STDERR "Inserted  $new_image_count images.\n";
 323 print STDERR "Done. \n";
 324
 325 sub usage {
 326     print "Usage: load_images.pl -D dbname [ cxgn | sandbox ]  -H dbhost -t [trial mode ] -i input dir -r chado table name for the object to link with the image \n";
 327     exit();
 328 }
 329
 330 sub message {
 331     my $message=shift;
 332     print STDERR $message;
 333 }