bin/load_images.pl

   1 #!/usr/bin/perl
   2
   3 =head1 NAME
   4
   5 load_images.pl
   6
   7 =head1 SYNOPSYS
   8
   9 load_images.pl -D [ sandbox | cxgn | trial ] -H hostname -i dirname -r chado table name [script will load image ids into ChadoTableprop ]
  10
  11 =head1 DESCRIPTION
  12
  13 Loads  images  into the SGN database, using the SGN::Image framework.
  14 Then link the loaded image with the user-supplied chado objects (e.g. stock, nd_experiment)
  15
  16
  17 Requires the following parameters:
  18
  19 =over 8
  20
  21 =item -D
  22
  23 a database parameter, which can either be "cxgn", "sandbox", or "trial". "cxgn" and "sandbox" will cause the script to connect to the respective databases; "trial" will connect to sandbox, but not perform any of the database modifications.
  24
  25 =item -H
  26
  27 host name
  28
  29 =item -m
  30
  31 map file. If provided links between stock names - image file name , is read from a mapping file.
  32 Row labels are expected to be unique file names, column header for the associated stocks is 'name'
  33
  34 =item -i
  35
  36 a dirname that contains image filenames or subdirectories named after database accessions, containing one or more images (see option -d) .
  37
  38 =item -u
  39
  40 use name - from sgn_people.sp_person.
  41
  42 =item -b
  43
  44 the dir where the database stores the images (the concatenated values from image_path and image_dir from sgn_local.conf or sgn.conf)
  45
  46 =item -d
  47
  48 files are stored in sub directories named after database accessions
  49
  50 =item -e
  51
  52 image file extension . Defaults to 'jpg'
  53
  54
  55 =item -t
  56
  57 trial mode . Nothing will be stored.
  58
  59
  60 =back
  61
  62 The script will generate an error file, named like the filename supplied, with the extension .err.
  63
  64 =head1 AUTHOR(S)
  65
  66 Naama Menda (nm249@cornell.edu) October 2010.
  67 Tweaks and move to sgn/bin: Lukas Mueller (lam87@cornell.edu) December 2023.
  68
  69 =cut
  70
  71 use strict;
  72
  73 use CXGN::Metadata::Schema;
  74 use CXGN::Metadata::Metadbdata;
  75 use CXGN::DB::InsertDBH;
  76 use CXGN::Image;
  77 use Bio::Chado::Schema;
  78 use CXGN::People::Person;
  79 use Carp qw /croak/;
  80 use Data::Dumper qw / Dumper /;
  81
  82 use File::Basename;
  83 use SGN::Context;
  84 use Getopt::Std;
  85
  86 use CXGN::Tools::File::Spreadsheet;
  87 use File::Glob qw | bsd_glob |;
  88
  89 our ($opt_H, $opt_D, $opt_t, $opt_i, $opt_u, $opt_r, $opt_d, $opt_e, $opt_m, $opt_b);
  90 getopts('H:D:u:i:e:f:tdr:m:b:');
  91
  92 my $dbhost = $opt_H;
  93 my $dbname = $opt_D;
  94 my $dirname = $opt_i;
  95 my $sp_person=$opt_u;
  96 my $db_image_dir = $opt_b;
  97 my $chado_table = $opt_r;
  98 my $ext = $opt_e || 'jpg';
  99
 100 if (!$dbhost && !$dbname) {
 101     print "dbhost = $dbhost , dbname = $dbname\n";
 102     print "opt_t = $opt_t, opt_u = $opt_u, opt_r = $chado_table, opt_i = $dirname\n";
 103     usage();
 104 }
 105
 106 if (!$dirname) { print "dirname = $dirname\n" ; usage(); }
 107
 108 my $dbh = CXGN::DB::InsertDBH->new( { dbhost=>$dbhost,
 109                                       dbname=>$dbname,
 110                                     } );
 111
 112 my $schema= Bio::Chado::Schema->connect(  sub { $dbh->get_actual_dbh() } ,  { on_connect_do => ['SET search_path TO  public;'] }
 113     );
 114
 115
 116 print STDERR "Generate metadata_id... ";
 117 my $metadata_schema = CXGN::Metadata::Schema->connect("dbi:Pg:database=$dbname;host=".$dbh->dbhost(), "postgres", $dbh->dbpass(), {on_connect_do => "SET search_path TO 'metadata', 'public'", });
 118
 119 my $sp_person_id= CXGN::People::Person->get_person_by_username($dbh, $sp_person);
 120 my %name2id = ();
 121
 122
 123 #my $ch = SGN::Context->new();
 124 print "PLEASE VERIFY:\n";
 125 print "Using dbhost: $dbhost. DB name: $dbname. \n";
 126 print "Path to image is: $db_image_dir\n";
 127 print "CONTINUE? ";
 128 my $a = (<STDIN>);
 129 if ($a !~ /[yY]/) { exit(); }
 130
 131 my %image_hash = ();  # used to retrieve images that are already loaded
 132 my %connections = (); # keep track of object -- image connections that have already been made.
 133
 134 print STDERR "Caching stock table...\n";
 135 my $object_rs = $schema->resultset("Stock::Stock")->search( { } ) ;
 136 while (my $object = $object_rs->next ) {
 137     my $id = $object->stock_id;
 138     my $name = $object->uniquename;
 139     $name2id{lc($name)} = $id;
 140 }
 141
 142 # cache image chado object - image links to prevent reloading of the
 143 # same data
 144 #
 145 print "Caching image $chado_table links...\n";
 146
 147 my $q = "SELECT * FROM phenome.stock_image";
 148 my $sth = $dbh->prepare($q);
 149 $sth->execute();
 150 while ( my $hashref = $sth->fetchrow_hashref() ) {
 151     my $image_id = $hashref->{image_id};
 152     my $chado_table_id = $hashref->{stock_id};  ##### table specific
 153
 154     print STDERR "\n\nCHADO TABLE ID = $chado_table_id\n\n";
 155
 156     my $i = CXGN::Image->new(dbh=>$dbh, image_id=>$image_id, image_dir=>$db_image_dir); # SGN::Image...$ch
 157     my $original_filename = $i->get_original_filename();
 158     $image_hash{$original_filename} = $i; # this doesn't have the file extension
 159     $connections{$image_id."-".$chado_table_id}++;
 160 }
 161
 162 #open (ERR, ">load_bcs_images.err") || die "Can't open error file\n";
 163
 164 my @files;
 165 if (! $opt_d) {
 166     @files = bsd_glob "$dirname/*.$ext";
 167 }
 168 else {
 169     @files = bsd_glob "$dirname/*" if $opt_d ;
 170 }
 171
 172 print STDERR "DIRS = ".(join("\n", @files))."\n";
 173
 174 my @sub_files;
 175
 176 my $new_image_count = 0;
 177
 178 my $metadata = CXGN::Metadata::Metadbdata->new($metadata_schema, $sp_person);
 179 my $metadata_id = $metadata->store()->get_metadata_id();
 180
 181 #read from spreadsheet:
 182 my $map_file = $opt_m; #
 183 my %name_map;
 184
 185 if ($opt_m) {
 186     my $s = CXGN::Tools::File::Spreadsheet->new($map_file); #
 187     my @rows = $s->row_labels(); #
 188     foreach my $file_name (@rows) { #
 189         my $stock_name = $s->value_at($file_name, 'name'); #
 190         $name_map{$file_name} = $stock_name;
 191     }
 192 }
 193
 194 print STDERR "Starting to process ".scalar(@files)." images...\n";
 195
 196 foreach my $file (@files) {
 197     eval {
 198         chomp($file);
 199         @sub_files = ($file);
 200         @sub_files =  bsd_glob "$file/*"; # if $opt_d;
 201
 202         print STDERR "FILES FOR $file: ".Dumper(\@sub_files)."\n";
 203
 204         my $object =  basename($file, ".$ext" );
 205
 206 #       if (!$plot) { die "File $file has no object name in it!"; }
 207         my $stock = $schema->resultset("Stock::Stock")->find( {
 208             stock_id => $name2id{ lc($object) }  } );
 209         foreach my $filename (@sub_files) {
 210
 211             chomp $filename;
 212
 213             print STDERR "FILENAME NOW: $filename\n";
 214             my $image_base = basename($filename);
 215             my ($object_name, $description, $extension);
 216             if ($opt_m) {
 217                 $object_name = $name_map{$object . "." . $ext } ;
 218             }
 219
 220             print STDERR "OBJECT = $object...\n";
 221 #           if ($image_base =~ /(.*?)\_(.*?)(\..*?)?$/) {
 222             if ($image_base =~ m/(.*)(\.$ext)/i) {
 223                 $extension = $2;
 224                 $image_base = $1;
 225             }
 226             if ($image_base =~ m/(.*)\_(.*)/)  {
 227                 $object_name = $1;
 228                 $description = $2;
 229
 230             }
 231             else {
 232                 $object_name = $image_base;
 233             }
 234             print STDERR "Object: $object OBJECT NAME: $object_name DESCRPTION: $description EXTENSIO: $extension\n";
 235
 236
 237             print STDOUT "Processing file $file...\n";
 238             print STDOUT "Loading $object_name, image $filename\n";
 239             print STDERR "Loading $object_name, image $filename\n";
 240             my $image_id; # this will be set later, depending if the image is new or not
 241             if (! -e $filename) {
 242                 warn "The specified file $filename does not exist! Skipping...\n";
 243                 next();
 244             }
 245
 246             if (!exists($name2id{lc($object)})) {
 247                 message ("$object does not exist in the database...\n");
 248             }
 249
 250             else {
 251                 print STDERR "Adding $filename...\n";
 252                 if (exists($image_hash{$filename})) {
 253                     print STDERR "$filename is already loaded into the database...\n";
 254                     $image_id = $image_hash{$filename}->get_image_id();
 255                     $connections{$image_id."-".$name2id{lc($object)}}++;
 256                     if ($connections{$image_id."-".$name2id{lc($object)}} > 1) {
 257                         print STDERR "The connection between $object and image $filename has already been made. Skipping...\n";
 258                     }
 259                     elsif ($image_hash{$filename}) {
 260                         print STDERR qq  { Associating $chado_table $name2id{lc($object)} with already loaded image $filename...\n };
 261                     }
 262                 }
 263                 else {
 264                     print STDERR qq { Generating new image object for image $filename and associating it with $chado_table $object, id $name2id{lc($object) } ...\n };
 265
 266                     if ($opt_t)  {
 267                         print STDOUT qq { Would associate file $filename to $chado_table $object_name, id $name2id{lc($object)}\n };
 268                         $new_image_count++;
 269                     }
 270                     else {
 271                         my $image = CXGN::Image->new(dbh=>$dbh, image_dir=>$db_image_dir);
 272                         $image_hash{$filename}=$image;
 273
 274                         my $error;
 275                         ($image_id, $error) = $image->process_image("$filename", $chado_table , $name2id{lc($object)}, 1);
 276
 277                         print STDERR "IMAGE ID $image_id, ERROR: $error\n";
 278
 279                         if ($error eq "ok") {
 280
 281
 282                             $image->set_description("$description");
 283                             $image->set_name(basename($filename , ".$ext"));
 284                             $image->set_sp_person_id($sp_person_id);
 285                             $image->set_obsolete("f");
 286                             $image_id = $image->store();
 287                             #link the image with the BCS object
 288                             $new_image_count++;
 289                             my $image_subpath = $image->image_subpath();
 290                             print STDERR "FINAL IMAGE PATH = $db_image_dir/$image_subpath\n";
 291                         }
 292                     }
 293                 }
 294             }
 295
 296
 297             print STDERR "Connecting image $filename and id $image_id with stock ".$stock->stock_id()."\n";
 298             #store the image_id - stock_id link
 299             my $q = "INSERT INTO phenome.stock_image (stock_id, image_id, metadata_id) VALUES (?,?,?)";
 300             my $sth  = $dbh->prepare($q);
 301             $sth->execute($stock->stock_id, $image_id, $metadata_id);
 302         }
 303     };
 304     if ($@) {
 305         print STDERR "ERROR OCCURRED WHILE SAVING NEW INFORMATION. $@\n";
 306         $dbh->rollback();
 307     }
 308     else {
 309         $dbh->commit();
 310     }
 311 }
 312
 313
 314
 315
 316 #close(ERR);
 317 close(F);
 318
 319
 320
 321
 322 print STDERR "Inserted  $new_image_count images.\n";
 323 print STDERR "Done. \n";
 324
 325 sub usage {
 326     print "Usage: load_images.pl -D dbname [ cxgn | sandbox ]  -H dbhost -t [trial mode ] -i input dir -r chado table name for the object to link with the image \n";
 327     exit();
 328 }
 329
 330 sub message {
 331     my $message=shift;
 332     print STDERR $message;
 333 }