lib/data_subs.pm

   1 This file contains the code for the subroutines in
   2 Perl-speaks-NONMEMs data module. It is not functional by itself.
   3 The code should be transferred to the module autogenerated by dia2code
   4 using the fill_diacode.pl script.
   5
   6 TODO: Fix the synced attribute so that all methods that change the state of the
   7 data object sets synced=0
   8
   9 # {{{ include
  10
  11 start include statements
  12 use Digest::MD5 'md5_hex';
  13 use OSspecific;
  14 use File::Copy "cp";
  15 use Carp;
  16 use Carp qw(cluck);
  17 use Config;
  18 use Math::Random;
  19 use Storable;
  20 use debug;
  21 use ui;
  22 use status_bar;
  23 use Data::Dumper;
  24 use Time::HiRes qw(gettimeofday);
  25 my @primary_column_names = ('ID', 'DATE', 'DAT1', 'DAT2', 'DAT3' ,'L1', 'L2', 'DV', 'MDV', 'RAW_', 'MRG_', 'RPT_', 'TIME', 'DROP', 'SKIP', 'EVID', 'AMT', 'RATE', 'SS', 'II', 'ADDL', 'CMT', 'PCMT', 'CALL');
  26 end include
  27
  28 # }}} include statements
  29
  30 # {{{ description
  31
  32 start description
  33     # The structure of the data class is subject-centric, recognising that
  34     # the subjects included in a study often can be regarded as
  35     # independent. A class for the subject level exists within PsN and is
  36     # called the individual class. A data object consists of at least one
  37     # but probably many individual objects plus optional comments.
  38 end description
  39
  40 # }}} description
  41
  42 # {{{ synopsis
  43
  44 start synopsis
  45     #   use data;
  46     #
  47     #   my $data_obj = data -> new ( filename => 'test040314.dta' );
  48     #
  49     #   $data_obj -> renumber_ascending;
  50     #
  51     #   my $subsets_ref = $data_obj -> case_deletion( bins => 10 );
  52     #
  53     #   my @subsets = @{$subsets_ref};
  54 end synopsis
  55
  56 # }}} synopsis
  57
  58 # {{{ see_also
  59
  60 start see_also
  61     # =begin html
  62     #
  63     # <a HREF="model.html">model</a>, <a HREF="output.html">output</a>,
  64     # <a HREF="tool/modelfit.html">tool::modelfit</a>,
  65     # <a HREF="tool.html">tool</a>
  66     #
  67     # =end html
  68     #
  69     # =begin man
  70     #
  71     # model, output, tool::modelfit, tool
  72     #
  73     # =end man
  74 end see_also
  75
  76 # }}} see_also
  77
  78 # {{{ new
  79
  80 start new
  81       {
  82         # If the column holding the subject identifier is not the
  83         # first, it can be specified using the I<idcolumn> attribute
  84         #
  85         # I<ignoresign> determines which rows that are regarded as
  86         # comments. Corresponds to the IGNORE= option in the $DATA
  87         # record in a NONMEM model file.
  88
  89         $this -> {'use_data_table'} = 0;
  90
  91         ( $this -> {'directory'},
  92           $this -> {'filename'} ) = OSspecific::absolute_path( $this -> {'directory'},
  93                                                                $this->{'filename'} );
  94
  95         debug -> warn( level => 2,
  96                        message => "data -> new: Data object initialized from file: ".
  97                        $this -> full_name );
  98
  99 #       sub register_in_database {
 100 #         my $this = shift;
 101 #         # Backslashes messes up the sql syntax
 102 #         my $file_str = $this->{'filename'};
 103 #         my $dir_str = $this->{'directory'};
 104 #         $file_str =~ s/\\/\//g;
 105 #         $dir_str =~ s/\\/\//g;
 106
 107 #         # md5sum
 108 #         my $md5sum = md5_hex(OSspecific::slurp_file($this-> full_name ));
 109 #         my $dbh = DBI ->
 110 #             connect("DBI:mysql:host=".$PsN::config -> {'_'} -> {'database_server'}.
 111 #                     ";databse=".$PsN::config -> {'_'} -> {'project'},
 112 #                     $PsN::config -> {'_'} -> {'user'},
 113 #                     $PsN::config -> {'_'} -> {'password'},
 114 #                     {'RaiseError' => 1});
 115 #         my $sth;
 116 #         my $sth = $dbh -> prepare( "SELECT data_id FROM ".$PsN::config -> {'_'} -> {'project'}.
 117 #                              ".data ".
 118 #                                    "WHERE filename = '$file_str' AND ".
 119 #                                    "directory = '$dir_str' AND ".
 120 #                                    "md5sum = '".$md5sum."'" );
 121 #         $sth -> execute or debug -> die( message => $sth->errstr ) ;
 122 #         my $select_arr = $sth -> fetchall_arrayref;
 123 #         if ( scalar @{$select_arr} > 0 ) {
 124 #           debug -> warn( level   => 1,
 125 #                          message => "Found an old entry in the database matching the ".
 126 #                          "current data file" );
 127 #           if ( scalar @{$select_arr} > 1 ) {
 128 #             debug -> warn( level   => 1,
 129 #                            message => "Found more than one matching entry in database".
 130 #                            ", using the first" );
 131 #           }
 132 #           $this -> {'data_id'} = $select_arr->[0][0];
 133 #         } else {
 134 #           my ( $date_str, $time_str );
 135 #           if ( $Config{osname} eq 'MSWin32' ) {
 136 #             $date_str = `date /T`;
 137 #             $time_str = ' '.`time /T`;
 138 #           } else {
 139 #             # Assuming UNIX
 140 #             $date_str = `date`;
 141 #           }
 142 #           chomp($date_str);
 143 #           chomp($time_str);
 144 #           my $date_time = $date_str.$time_str;
 145 #           $sth = $dbh -> prepare("INSERT INTO ".$PsN::config -> {'_'} -> {'project'}.
 146 #                              ".data (filename,date,directory,md5sum) ".
 147 #                                  "VALUES ('$file_str', '$date_time', '$dir_str','".
 148 #                                  $md5sum."' )");
 149 #           $sth -> execute;
 150 #           $this -> {'data_id'} = $sth->{'mysql_insertid'};
 151 #         }
 152 #         $sth -> finish;
 153 #         $dbh -> disconnect;
 154 #         }
 155
 156
 157         unless ( ( defined $this -> {'header'} and
 158                    scalar @{$this -> {'header'}} > 0 ) or
 159                  ( defined $this -> {'individuals'} and
 160                    scalar @{$this -> {'individuals'}} > 0 ) ) {
 161           if ( -e $this -> full_name ) {
 162             if ( $this -> {'target'} eq 'mem' ) {
 163 #             &register_in_database( $this ) if ( $PsN::config -> {'_'} -> {'use_database'} and
 164 #                                                 $this -> {'use_data_table'} );
 165               $this -> _read_header;
 166               $this -> _read_individuals;
 167               $this -> {'synced'} = 1;
 168             } else {
 169               $this -> {'synced'} = 0;
 170             }
 171           } else {
 172             debug -> die(message => "No header, individuals, and no file " . $this -> full_name . " on disk.")
 173               unless $this -> {'ignore_missing_files'};
 174             $this ->  {'synced'} = 0;
 175           }
 176         } else {
 177           if ( $this -> {'target'} eq 'mem') {
 178             if ( -e $this -> {'filename'} ) {
 179               $this -> _read_header;
 180 #             &register_in_database if ( $PsN::config -> {'_'} -> {'use_database'} and
 181 #                                        $this -> {'use_data_table'} );
 182               $this -> _read_individuals;
 183               $this -> {'synced'} = 1;
 184             } else {
 185               debug -> die(message => "No file:".$this->{'filename'}." on disk" )
 186                   unless $this -> {'ignore_missing_files'};
 187               $this -> {'synced'} = 0;
 188             }
 189           } else {
 190             $this -> flush;
 191           }
 192         }
 193
 194         if ( $this -> {'synced'} ) {
 195           my $i = 1;
 196           foreach my $head ( @{$this -> {'header'}} ) {
 197             $this -> {'column_head_indices'} -> {$head} = $i;
 198             $i++;
 199           }
 200         }
 201 #       $Data::Dumper::Maxdepth = 3;
 202 #       die Dumper $this -> {'individuals'};
 203       }
 204 end new
 205
 206 # }}} new
 207
 208 # {{{ register_in_database
 209 start register_in_database
 210     if ( $PsN::config -> {'_'} -> {'use_database'} ) {
 211       # Backslashes messes up the sql syntax
 212       my $file_str = $self->{'filename'};
 213       my $dir_str = $self->{'directory'};
 214       $file_str =~ s/\\/\//g;
 215       $dir_str =~ s/\\/\//g;
 216
 217       my $project = $PsN::config -> {'_'} -> {'project'};
 218       # md5sum
 219       my $md5sum = md5_hex(OSspecific::slurp_file($self-> full_name ));
 220
 221       my $dbh = DBI -> connect("DBI:mysql:host=".$PsN::config -> {'_'} -> {'database_server'}.
 222                                ";databse=".$project,
 223                                $PsN::config -> {'_'} -> {'user'},
 224                                $PsN::config -> {'_'} -> {'password'},
 225                                {'RaiseError' => 1});
 226
 227       my $sth;
 228
 229       my $select_arr = [];
 230
 231       if ( not $force ) {
 232         my $sth = $dbh -> prepare( "SELECT data_id FROM ".$project.
 233                                    ".data ".
 234                                    "WHERE filename = '$file_str' AND ".
 235                                    "directory = '$dir_str' AND ".
 236                                    "md5sum = '".$md5sum."'" );
 237         $sth -> execute or debug -> die( message => $sth->errstr ) ;
 238         $select_arr = $sth -> fetchall_arrayref;
 239       }
 240
 241       if ( scalar @{$select_arr} > 0 ) {
 242         'debug' -> warn( level   => 1,
 243                          message => "Found an old entry in the database matching the ".
 244                          "current data file" );
 245         if ( scalar @{$select_arr} > 1 ) {
 246           'debug' -> warn( level   => 1,
 247                            message => "Found more than one data matching entry in database".
 248                            ", using the first" );
 249         }
 250         $self -> {'data_id'} = $select_arr->[0][0];
 251         # Find the id's
 252         my $sth = $dbh -> prepare( "SELECT individual_id FROM ".$project.".data_individual ".
 253                                    "WHERE data_id = '".$self -> {'data_id'}."'" );
 254         $sth -> execute or debug -> die( message => $sth->errstr ) ;
 255         my $id_arr = $sth -> fetchall_arrayref;
 256         map( $_ = $_ -> [0], @{$id_arr} );
 257         $self -> {'individual_ids'} = $id_arr;
 258       } elsif ( defined $self -> {'individuals'} ) {
 259         my ( $date_str, $time_str );
 260         if( $Config{osname} eq 'MSWin32' ){
 261           $date_str = `date /T`;
 262           $time_str = ' '.`time /T`;
 263         } else {
 264           # Assuming UNIX
 265           $date_str = `date`;
 266         }
 267         chomp($date_str);
 268         chomp($time_str);
 269         my $date_time = $date_str.$time_str;
 270         my ( $columns, $values );
 271         my $res_str = $resampled ? '1' : '0';
 272         if ( defined $model_id ) {
 273           $columns = '(model_id, filename, date, directory, md5sum, resampled)';
 274           $values = "('$model_id', '$file_str', '$date_time', '$dir_str','".
 275               $md5sum."', '$res_str' )";
 276         } else {
 277           $columns = '(filename, date, directory, md5sum, resampled)';
 278           $values = "('$file_str', '$date_time', '$dir_str','".$md5sum."', '$res_str' )";
 279         }
 280         $sth = $dbh -> prepare("INSERT INTO ".$PsN::config -> {'_'} -> {'project'}.
 281                                ".data $columns VALUES $values");
 282         $sth -> execute;
 283         $self -> {'data_id'} = $sth->{'mysql_insertid'};
 284
 285         if ( defined $self -> {'data_id'} ) {
 286           my $values;
 287           my $columns = "( id_key, id )";
 288           if( $#individual_ids >= 0 ) {
 289             $self -> register_di_relation( individual_ids => \@individual_ids );
 290           } else {
 291             my $inds = scalar @{$self -> {'individuals'}};
 292             $dbh -> do( "LOCK TABLES ".$PsN::config -> {'_'} -> {'project'}.
 293                         ".individual WRITE" );
 294 #           $sth = $dbh -> prepare( "SELECT MAX(individual_id)".
 295 #                                   " FROM ".$PsN::config -> {'_'} -> {'project'}.
 296 #                                   ".individual" );
 297             $dbh -> do( 'USE '.$PsN::config -> {'_'} -> {'project'} );
 298             $sth = $dbh -> prepare( "SHOW TABLE STATUS LIKE 'individual'" );
 299             $sth -> execute or debug -> die( message => $sth->errstr ) ;
 300             my $select_arr = $sth -> fetchall_arrayref;
 301             my $first_id_id = $select_arr -> [0][10] ?
 302                 $select_arr -> [0][10] : 0;
 303 #           my $first_id_id = $select_arr -> [0][0] ? ($select_arr -> [0][0] + 1) : 0;
 304             my $last_id_id = $first_id_id + $inds - 1;
 305             for( my $i = 0; $i < $inds; $i++ ) {
 306               if( defined $self -> {'individuals'}[$i] ) {
 307                 my $id_id = $self -> {'individuals'}[$i] -> idnumber;
 308                 $values = $values."," if ( defined $values );
 309                 $values = $values."( $i, $id_id )";
 310               }
 311             }
 312             $sth = $dbh -> prepare( "INSERT INTO ".$PsN::config -> {'_'} -> {'project'}.
 313                                     ".individual $columns VALUES $values" );
 314             $sth -> execute;
 315             $dbh -> do( "UNLOCK TABLES" );
 316             @individual_ids = ($first_id_id .. $last_id_id);
 317             $self -> register_di_relation( individual_ids => \@individual_ids );
 318           }
 319           $self -> {'individual_ids'} = \@individual_ids;
 320         }
 321         $sth -> finish;
 322         $dbh -> disconnect;
 323       }
 324       $data_id = $self -> {'data_id'}; # return the data_id
 325     }
 326 end register_in_database
 327 # }}} register_in_database
 328
 329 # {{{ register_di_relation
 330 start register_di_relation
 331   if ( $PsN::config -> {'_'} -> {'use_database'} and
 332        defined $self -> {'data_id'} and $#individual_ids >= 0 ) {
 333     my $dbh = DBI -> connect("DBI:mysql:host=".$PsN::config -> {'_'} -> {'database_server'}.
 334                              ";databse=".$PsN::config -> {'_'} -> {'project'},
 335                              $PsN::config -> {'_'} -> {'user'},
 336                              $PsN::config -> {'_'} -> {'password'},
 337                              {'raiseerror' => 1});
 338     my $sth;
 339     my $values;
 340     my $columns = "( data_id, individual_id )";
 341     foreach my $individual_id ( @individual_ids ) {
 342       if ( defined $individual_id ) {
 343         $values = $values."," if ( defined $values );
 344         $values = $values."(".$self -> {'data_id'}.", $individual_id )";
 345       }
 346     }
 347     $sth = $dbh -> prepare( "INSERT INTO ".$PsN::config -> {'_'} -> {'project'}.
 348                                ".data_individual $columns VALUES $values" );
 349     $sth -> execute;
 350     $sth -> finish if ( defined $sth );
 351     $dbh -> disconnect;
 352   }
 353 end register_di_relation
 354 # }}} register_di_relation
 355
 356 # {{{ full_name
 357
 358 start full_name
 359       {
 360         $full_name = $self -> {'directory'} . $self -> {'filename'};
 361       }
 362 end full_name
 363
 364 # }}}
 365
 366 # {{{ bootstrap
 367
 368 start bootstrap
 369       {
 370         # The bootstrap method draws I<samples> number of boostrap
 371         # samples from the data set. The I<subjects> arguments
 372         # determines the size of each sample (default equals to the
 373         # number of individuals in the original data set). The method
 374         # returns references to three arrays: I<boot_samples_ref>,
 375         # which holds the bootstrap data sets, I<incl_individuals_ref>
 376         # which holds arrays containing the subject identifiers (ID's)
 377         # for the included individuals of each bootstrap data set and
 378         # I<included_keys_ref> which holds the key or index of the
 379         # included individuals. The key or index is an integer
 380         # starting at 1 for the first individual in the original data
 381         # set and increasing by one for each following.
 382         $self -> synchronize;
 383         my @header      = @{$self -> {'header'}};
 384         my $individuals = $self -> {'individuals'};
 385         my $key_ref;
 386
 387         my $status_bar = status_bar -> new( steps => $samples );
 388         ui -> print( category => 'bootstrap',
 389                      message => $status_bar -> print_step,
 390                      newline => 0);
 391
 392         for ( my $i = 1; $i <= $samples; $i++ ) {
 393           my $new_name = defined $name_stub ? $name_stub."_$i.dta" : "bs$i.dta";
 394           $new_name = $directory.'/'.$new_name;
 395           my ( $boot, $incl_ind_ref, $incl_key_ref ) =
 396             $self -> resample( subjects    => $subjects,
 397                                resume      => $resume,
 398                                new_name    => $new_name,
 399                                target      => $target,
 400                                stratify_on => $stratify_on,
 401                                model_id    => $model_ids[$i-1] );
 402           push( @included_keys, $incl_key_ref );
 403           push( @incl_individuals, $incl_ind_ref );
 404 #         $boot -> renumber_ascending;
 405           push( @boot_samples, $boot );
 406 #          $boot -> synchronize;
 407 #          $boot -> flush;
 408           if( $status_bar -> tick() ){
 409             ui -> print( category => 'bootstrap',
 410                          message => $status_bar -> print_step,
 411                          newline => 0,
 412                          wrap => 0);
 413           }
 414         }
 415         ui -> print( category => 'bootstrap',
 416                      message => ' ... done' );
 417       }
 418 end bootstrap
 419
 420 # }}} bootstrap
 421
 422 # {{{ resample
 423
 424 start resample
 425       {
 426         $self -> synchronize;
 427         my ( @header, $individuals, @bs_inds, $key_ref, @id_ids, @bs_id_ids );
 428         @id_ids = @{$self -> {'individual_ids'}} if( defined $self -> {'individual_ids'} );
 429         if ( defined $stratify_on ) {
 430           unless ( $resume and -e $new_name ) {
 431             @header = @{$self -> {'header'}};
 432             $individuals = $self -> {'individuals'};
 433             my %strata;
 434             if( $stratify_on =~ /\D/ ){
 435               %strata = %{$self -> factors( column_head => $stratify_on )};
 436               if ( $strata{'Non-unique values found'} eq '1' ) {
 437                 debug -> die( message => "Individuals were found to have multiple values in the $stratify_on column. ".
 438                               "The column $stratify_on cannot be used for stratification of the resampling." );
 439               }
 440             } else {
 441               %strata = %{$self -> factors( column => $stratify_on )};
 442               if ( $strata{'Non-unique values found'} eq '1' ) {
 443                 debug -> die( message => "Individuals were found to have multiple values in column number $stratify_on. ".
 444                               "Column $stratify_on cannot be used for stratification of the resampling." );
 445               }
 446             }
 447
 448             while( my ( $factor, $key_list ) = each %strata ) {
 449               my $keys = scalar @{$key_list};
 450               for ( my $i = 0; $i < $keys; $i++ ) {
 451                 my $list_ref = random_uniform_integer(1,0,$keys-1);
 452                 push( @bs_inds, $individuals ->
 453                       [ $key_list -> [$list_ref] ] -> copy );
 454                 push( @included_keys, $key_list -> [$list_ref] );
 455                 push( @incl_individuals, $individuals ->
 456                       [ $key_list -> [$list_ref] ] -> idnumber );
 457                 push( @bs_id_ids, $id_ids[ $key_list -> [$list_ref] ] );
 458               }
 459             }
 460
 461             $boot = data -> new( header      => \@header,
 462                                  idcolumn    => $self -> {'idcolumn'},
 463                                  ignoresign  => $self -> {'ignoresign'},
 464                                  individuals => \@bs_inds,
 465                                  filename    => $new_name,
 466                                  ignore_missing_files => 1,
 467                                  target      => 'mem' );
 468             $boot -> renumber_ascending;
 469             $boot -> _write;
 470             $boot -> flush;
 471             #$boot -> target( $target );
 472           }
 473         } else {
 474           unless ( $resume and -e $new_name ) {
 475             @header = @{$self -> {'header'}};
 476             $individuals = $self -> {'individuals'};
 477             for ( my $i = 1; $i <= $subjects; $i++ ) {
 478               $key_ref = random_uniform_integer(1,0,scalar @{$individuals}-1);
 479               push( @bs_inds, $individuals -> [ $key_ref ] -> copy );
 480               push( @included_keys, $key_ref );
 481               push( @incl_individuals, $individuals -> [ $key_ref ] -> idnumber );
 482               push( @bs_id_ids, $id_ids[ $key_ref ] );
 483             }
 484
 485             # MUST FIX: If a file already exists with the same name,
 486             # the created bs data set will be appended to this. IT
 487             # MUST BE OVERWRITTEN!
 488             $boot = data -> new( header      => \@header,
 489                                  idcolumn    => $self -> {'idcolumn'},
 490                                  ignoresign  => $self -> {'ignoresign'},
 491                                  individuals => \@bs_inds,
 492                                  filename    => $new_name,
 493                                  ignore_missing_files => 1,
 494                                  target      => 'mem' );
 495             $boot -> renumber_ascending;
 496             $boot -> _write;
 497             $boot -> target( $target );
 498           } else {
 499             # If we are resuming, we still need to generate the
 500             # pseudo-random sequence and initiate a data object
 501             for ( my $i = 1; $i <= $subjects; $i++ ) {
 502               random_uniform_integer(1,0,scalar @{$individuals}-1)
 503             }
 504             $boot = data -> new( idcolumn    => $self -> {'idcolumn'},
 505                                  ignoresign  => $self -> {'ignoresign'},
 506                                  filename    => $new_name,
 507                                  ignore_missing_files => 1,
 508                                  target      => $target );
 509             $boot -> _write;
 510             $boot -> flush;
 511           }
 512           if( $target eq 'disk'){
 513             $boot -> flush;
 514           }
 515         }
 516         $boot -> register_in_database( individual_ids => \@bs_id_ids,
 517                                        resampled      => 1,
 518                                        model_id       => $model_id );
 519       }
 520 end resample
 521
 522 # }}} resample
 523
 524 # {{{ case_deletion
 525
 526 start case_deletion
 527       {
 528         # case_deletion creates subsets of the data. The number of
 529         # subsets is specified by the bins argument. The individuals
 530         # of each subset is selected randomly or in ascending
 531         # numerical order depending on the selection argument that can
 532         # be either 'consecutive' or 'random'. case_column must be
 533         # specified to give the method something to base the selection
 534         # on. Valid case_column values are either the column number
 535         # (pure digits) or the name of the column in the (optional)
 536         # header row.
 537         $self -> synchronize;
 538         my @header    = @{$self -> {'header'}};
 539         if ( not defined $case_column ) {
 540           debug -> die( message => "case_column must be specified" );
 541         } else {
 542           if ( not $case_column =~ /^\d/ ) {
 543             for ( my $i = 0; $i <= $#header; $i++ ) {
 544               $case_column = $i+1 if ( $header[$i] eq $case_column );
 545             }
 546           }
 547         }
 548         $bins = defined $bins ? $bins :
 549           scalar keys %{$self -> factors( column => $case_column)};
 550         my %factors   = %{$self -> factors( column => $case_column )};
 551         if ( $factors{'Non-unique values found'} eq '1' ) {
 552           debug -> die( message => "Individuals were found to have multiple values in column number $case_column. ".
 553                         "Column $case_column cannot be used for case deletion." );
 554         }
 555
 556         my $maxbins   = scalar keys %factors;
 557         my @ftrs      = sort { $a <=> $b } keys %factors;
 558         my $individuals = $self -> {'individuals'};
 559         my $maxkey    = scalar @{$individuals} - 1;
 560
 561         my ( @tmp_ftrs, @binsize ) =
 562             ((),());
 563         my ( $k, $j, $i ) = ( 0, 0, 0 );
 564         # Create the binsizes
 565         for ( $j = 0; $j < $maxbins; $j++ ) {
 566           $binsize[ $k++ ]++;
 567           $k = 0 if( $k >= $bins );
 568         }
 569         $self -> _fisher_yates_shuffle( array => \@ftrs ) if( $selection eq 'random' );
 570         for ( $k = 0; $k < $bins; $k++ ) {
 571           for ( $j = 0; $j < $binsize[ $k ]; $j++ ) {
 572 #           print "SK: ",$skipped_keys[ $k ]," F: ",$factors{ $ftrs[ $i ] },"\n";
 573             push( @{$skipped_keys[ $k ]}, @{$factors{ $ftrs[ $i ] }} );
 574             push( @{$skipped_values[ $k ]}, $ftrs[ $i++ ] );
 575           }
 576         }
 577
 578         for ( $k = 0; $k < $bins; $k++ ) {
 579           my @cd_inds = ();
 580           my @del_inds = ();
 581         SELKEYS: foreach my $key ( 0..$maxkey ) {
 582           foreach my $skipped ( @{$skipped_keys[ $k ]} ) {
 583             if ( $key == $skipped ) {
 584               push( @{$skipped_ids[ $k ]}, $individuals ->
 585                     [ $skipped ] -> idnumber );
 586               push( @del_inds, $individuals -> [ $key ] -> copy );
 587               next SELKEYS;
 588             }
 589           }
 590           push( @cd_inds, $individuals -> [ $key ] -> copy );
 591         }
 592           # Set ignore_missing_files = 1 to make it possible to get the result
 593           # in memory only
 594           my $newdata = data ->
 595               new ( header      => \@header,
 596                     ignoresign  => $self -> {'ignoresign'},
 597                     idcolumn    => $self -> {'idcolumn'},
 598                     individuals => \@cd_inds,
 599                     target      => $target,
 600                     filename    => 'cd'.$k+1 .'.dta',
 601                     ignore_missing_files => 1 );
 602           my $deldata = data ->
 603               new ( header      => \@header,
 604                     ignoresign  => $self -> {'ignoresign'},
 605                     idcolumn    => $self -> {'idcolumn'},
 606                     individuals => \@del_inds,
 607                     target      => $target,
 608                     filename    => 'del'.$k+1 .'.dta',
 609                     ignore_missing_files => 1 );
 610           push( @subsets, $newdata );
 611           push( @remainders, $deldata );
 612         }
 613       }
 614 end case_deletion
 615
 616 # }}} case_deletion
 617
 618 # {{{ copy
 619 start copy
 620       {
 621         # filename: new data file name.
 622         #
 623         # target: keep the copy in memory ('mem') or write it to disk and flush the memory ('disk').
 624
 625         ($directory, $filename) = OSspecific::absolute_path( $directory, $filename );
 626
 627         # Clone self into new data object. Why don't the individuals get cloned too?
 628         # strange. need to set synced to 0 AND set the {'individuals'} to undef.
 629         cp($self -> full_name, $directory.$filename );
 630         $new_data = Storable::dclone( $self );
 631         $new_data -> {'synced'} = 0;
 632         $new_data -> {'individuals'} = undef;
 633         $new_data -> synchronize;
 634
 635         # Set the new file name for the copy
 636         $new_data -> directory( $directory );
 637         $new_data -> filename( $filename );
 638       }
 639 end copy
 640
 641 # }}} copy
 642
 643 # {{{ count_ind
 644
 645 start count_ind
 646       {
 647         # Returns the number of individuals in the data set.
 648         $self -> synchronize;
 649         $num = scalar @{$self -> {'individuals'}};
 650       }
 651 end count_ind
 652
 653 # }}} count_ind
 654
 655 # {{{ diff
 656 start diff
 657 {
 658   $self -> synchronize;
 659
 660   my $first_id = $self -> {'individuals'}[0];
 661
 662   debug -> die( message => "No individuals defined in data object based on ".
 663                 $self -> full_name ) unless ( defined $first_id );
 664
 665   # Check if $column(-index) is defined and valid, else try to find index
 666   # using column_head
 667
 668   my @data_row = split( /,/, $first_id -> subject_data -> [0] );
 669   if( $#columns >= 0 ) {
 670     foreach my $column ( @columns ) {
 671       unless ( defined $column && defined( $data_row[$column-1] ) ) {
 672         debug -> die( message => "Error in data -> factors: ".
 673                       "invalid column number: \"$column\"\n".
 674                       "Valid column numbers are 1 to ".
 675                       scalar @{$first_id -> subject_data ->[0]}."\n" );
 676       }
 677     }
 678   } elsif ( $#column_heads >= 0 ) {
 679     foreach my $column_head ( @column_heads ) {
 680       unless (defined($column_head) && defined($self -> {'column_head_indices'}{$column_head})) {
 681         debug -> die( message => "Error in data -> factors: unknown column: \"$column_head\" ".
 682                       "Valid column headers are (in no particular order):\n".
 683                       join(', ',keys(%{$self -> {'column_head_indices'}})) );
 684       } else {
 685         my $column = $self -> {'column_head_indices'}{$column_head};
 686         push( @columns, $column );
 687         debug -> warn( level   => 2,
 688                        message => "$column_head is in column number $column" );
 689       }
 690     }
 691   } else {
 692     debug -> die( message => "No column or column_head defined" );
 693   }
 694
 695   if( $global_largest or $global_smallest or
 696       $largest_per_individual or $smallest_per_individual ) {
 697     if( not scalar @{$self -> {'individuals'}} == scalar @{$against_data -> individuals} ) {
 698       debug -> die( message => "Both data object must hold the same number of individuals ".
 699                     "and observations when calling data -> diff" );
 700     }
 701     for( my $i = 0; $i < scalar @{$self -> {'individuals'}}; $i++ ) {
 702       my %id_diffs = %{$self -> {'individuals'}[$i] ->
 703                            diff( against_individual => $against_data -> individuals -> [$i],
 704                                  columns            => \@columns,
 705                                  absolute_diff      => $absolute_diff,
 706                                  diff_as_fraction   => $diff_as_fraction,
 707                                  largest            => ( $global_largest or $largest_per_individual ),
 708                                  smallest           => ( $global_smallest or $smallest_per_individual ) )};
 709       if( $global_largest ) {
 710         for( my $j = 0; $j <= $#columns; $j++ ) {
 711           my $label = defined $column_heads[$j] ? $column_heads[$j] : $columns[$j];
 712           if( not defined $diff_results{$label} or not defined $diff_results{$label}{'diff'} or
 713               $id_diffs{$columns[$j]}{'diff'} > $diff_results{$label}{'diff'} ) {
 714             $diff_results{$label}{'diff'} = $id_diffs{$columns[$j]}{'diff'};
 715             $diff_results{$label}{'self'} = $id_diffs{$columns[$j]}{'self'};
 716             $diff_results{$label}{'test'} = $id_diffs{$columns[$j]}{'test'};
 717           }
 718         }
 719       }
 720     }
 721   } else {
 722     die "data -> diff is only implemented for finding the largest difference at any observation at this point\n";
 723   }
 724 }
 725 end diff
 726 # }}} diff
 727
 728 # {{{ filename
 729 start filename
 730       {
 731         if ( defined $parm and $parm ne $self -> {'filename'} ) {
 732           $self -> {'filename'} = $parm;
 733           $self -> {'data_id'} = undef;
 734 #         $self -> _write;
 735         }
 736       }
 737 end filename
 738 # }}} filename
 739
 740 # {{{ fractions
 741
 742 start fractions
 743       {
 744         my %factors = $self -> factors( 'return_occurences' => 1,
 745                                         'unique_in_individual' => $unique_in_individual,
 746                                         'column_head' => $column_head,
 747                                         'column' => $column);
 748
 749         my $sum = 0;
 750         while (my ($factor, $amount) = each %factors) {
 751           if ( $factor == $self -> {'missing_data'} && $ignore_missing ) {
 752             next;
 753           } else {
 754             $sum += $amount;
 755           }
 756         }
 757         while (my ($factor, $amount) = each %factors) {
 758           if ( $factor == $self -> {'missing_data'} && $ignore_missing ) {
 759             next;
 760           } else {
 761             $fractions{$factor} = $amount/$sum;
 762           }
 763         }
 764       }
 765 end fractions
 766
 767 # }}} fractions
 768
 769 # {{{ factors
 770
 771 start factors
 772       {
 773         # Either column (number, starting at 1) or column_head must be specified.
 774         #
 775         # The default behaviour is to return a hash with the factors as keys
 776         # and as values references to arrays with the order numbers (not the ID numbers)
 777         # of the individuals that contain this factor
 778         #
 779         # If unique_in_individual is true (1), the returned hash will contain
 780         # an element with key 'Non-unique values found' and value 1 if any
 781         # individual contain more than one value in the specified column.
 782         #
 783         # Return occurences will calculate the occurence of each
 784         # factor value. Several occurences in one individual counts as
 785         # one occurence. The elements of the returned hash will have the factors
 786         # as keys and the number of occurences as values.
 787         #
 788
 789         $self -> synchronize;
 790
 791         # Check if $column(-index) is defined and valid, else try to find index
 792         # using column_head
 793         my $first_id = $self -> {'individuals'}[0];
 794
 795         debug -> die( message => "No individuals defined in data object based on ".
 796                       $self -> full_name ) unless ( defined $first_id );
 797
 798         my @data_row = split( /,/, $first_id -> subject_data -> [0] );
 799         unless ( defined $column && defined( $data_row[$column-1] ) ) {
 800           unless (defined($column_head) && defined($self -> {'column_head_indices'}{$column_head})) {
 801             debug -> die( message => "Error in data -> factors: unknown column: \"$column_head\" ".
 802                           "or invalid column number: \"$column\".\n".
 803                           "Valid column numbers are 1 to ".scalar @data_row ."\n".
 804                           "Valid column headers are (in no particular order):\n".
 805                           join(', ',keys(%{$self -> {'column_head_indices'}})) );
 806           } else {
 807             $column = $self -> {'column_head_indices'}{$column_head};
 808             debug -> warn( level   => 2,
 809                            message => "$column_head is in column number $column" );
 810           }
 811         }
 812
 813         my $key = 0;
 814         foreach my $individual ( @{$self -> {'individuals'}} ) {
 815           my @ifactors = keys %{$individual -> factors( column => $column )};
 816           if ( scalar @ifactors > 1 and $unique_in_individual ) {
 817             %factors = ( 'Non-unique values found' => 1 );
 818             last;
 819           }
 820           debug -> die( message => "No value found in column $column in individual ".
 821                         $individual -> idnumber ) if ( scalar @ifactors == 0 );
 822
 823           # Return occurences will calculate the occurence of each
 824           # factor value. Several occurences in one individual counts as
 825           # one occurence.
 826
 827           if ( $return_occurences ) {
 828             foreach my $ifactor ( @ifactors ) {
 829               $factors{$ifactor}++;
 830             }
 831           } else {
 832             foreach my $ifactor ( @ifactors ) {
 833               push( @{$factors{$ifactor}}, $key );
 834             }
 835           }
 836           $key++;
 837         }
 838       }
 839 end factors
 840
 841 # }}} factors
 842
 843 # {{{ find_individual
 844
 845 # start find_individual
 846 #         foreach my $tmp_ind ( @{$self -> individuals} ) {
 847 #         if ( $tmp_ind -> key == $key ) {
 848 #           $individual = $tmp_ind;
 849 #           last;
 850 #         }
 851 #       }
 852 #         if ( defined $individual ) {
 853 #         if ( $copy ) {
 854 #           $individual = $individual -> copy;
 855 #         }
 856 #       } else {
 857 #         print "No individual with key $key found in call to ".
 858 #           "data -> find_individual\n" if ( $self -> debug );
 859 #         }
 860 # end find_individual
 861
 862 # }}}
 863
 864 # {{{ format_data
 865
 866 start format_data
 867       {
 868         my $header = $self -> {'header'};
 869
 870         # format the data for NONMEM (simple comma-separated layout)
 871         if ( defined $self -> {'comment'} ) {
 872           my @comment   = @{$self -> {'comment'}};
 873           for ( @comment ) {
 874             push( @form_data );
 875           }
 876         }
 877
 878         my $wrap = ( defined $self -> {'wrap_column'} and
 879                      defined $self -> {'cont_column'} );
 880
 881         my @primary_columns = defined $self -> {'primary_columns'} ?
 882           @{$self -> {'primary_columns'}} : ();
 883         my @secondary_columns = defined $self -> {'secondary_columns'} ?
 884           @{$self -> {'secondary_columns'}} : ();
 885         if ( defined $header and defined $self -> {'ignoresign'} ) {
 886           my $istr;
 887           if ( $self -> {'ignoresign'} ne '@' ) {
 888             $istr = $self -> {'ignoresign'};
 889           }
 890           if ( $wrap ) {
 891             my @h_data;
 892             for ( my $i = 0; $i <= $#secondary_columns ; $i++ ) {
 893               my $sstr = $istr;
 894               for ( my $j = 0; $j < scalar @{$secondary_columns[$i]} ; $j++ ) {
 895                 my $jstr = $j == 0 ? '' : ',';
 896                 $sstr = $sstr.$jstr.$secondary_columns[$i][$j][0];
 897               }
 898               push( @h_data, $sstr."\n" );
 899             }
 900             push( @form_data, @h_data );
 901             my $pstr = $istr;
 902             for ( my $i = 0; $i <= $#primary_columns ; $i++ ) {
 903               my $jstr = $i == 0 ? '' : ',';
 904               $pstr = $pstr.$jstr.$primary_columns[$i][0];
 905             }
 906             push( @form_data, $pstr."\n" );
 907           } else {
 908             push( @form_data, $istr.join(',',@{$self -> {'header'}})."\n" );
 909           }
 910         }
 911         if ( $wrap ) {
 912           foreach my $individual ( @{$self -> {'individuals'}} ) {
 913             foreach my $row ( @{$individual -> subject_data} ) {
 914               my @r_data;
 915               for ( my $i = 0; $i <= $#secondary_columns ; $i++ ) {
 916                 my $sstr = '';
 917                 for ( my $j = 0; $j < scalar @{$secondary_columns[$i]} ; $j++ ) {
 918                   my $jstr = $j == 0 ? '' : ',';
 919                   if ( $secondary_columns[$i][$j][0] eq 'CONT' ) {
 920                     $sstr = $sstr.$jstr.'1';
 921                   } else {
 922                     my @data_row = split( /,/, $row );
 923                     $sstr = $sstr.$jstr.$data_row[$secondary_columns[$i][$j][1]];
 924                   }
 925                 }
 926                 push( @r_data, $sstr."\n" );
 927               }
 928               push( @form_data, @r_data );
 929               my $pstr = '';
 930               for ( my $i = 0; $i <= $#primary_columns ; $i++ ) {
 931                 my $jstr = $i == 0 ? '' : ',';
 932                 if ( $primary_columns[$i][0] eq 'CONT' ) {
 933                   $pstr = $pstr.$jstr.'0';
 934                 } else {
 935                   my @data_row = split( /,/, $row );
 936                   $pstr = $pstr.$jstr.$data_row[$primary_columns[$i][1]];
 937                 }
 938               }
 939               push( @form_data, $pstr."\n" );
 940             }
 941           }
 942         } else {
 943           foreach my $individual ( @{$self -> {'individuals'}} ) {
 944             foreach my $row ( @{$individual -> subject_data} ) {
 945               push( @form_data, $row ."\n" );
 946             }
 947           }
 948         }
 949       }
 950 end format_data
 951
 952 # }}} format_data
 953
 954 # {{{ drop_dropped
 955
 956 start drop_dropped
 957       {
 958         # This method removes columns that has '=DROP' value in the
 959         # model header as given by $INPUT. The model header must be
 960         # transfered to this method through the model_header
 961         # argument. The model_header argument should be a
 962         # two-dimensional array where each position in the first
 963         # dimension should be a reference to a 1*2 array holding the
 964         # column name and value. Any ignore-sign must be removed.
 965
 966         debug -> die( message => 'model header must be defined' )
 967           if ( $#model_header < 0 );
 968         # Important that the drop_dropped method of the model::problem
 969         # class is in sync with this method.
 970         $self -> synchronize;
 971
 972         $self -> {'header'} = [];
 973         my @drop;
 974         my $counter = 1;
 975         for( my $i = 0; $i <= $#model_header; $i++ ) {
 976           $self -> {'idcolumn'} = $counter if ( $model_header[$i][0] eq 'ID' );
 977           if( $model_header[$i][1] eq 'DROP' and
 978               not $model_header[$i][0] =~ /DAT(E|1|2|3)/ ) {
 979             push( @drop, 1 );
 980           } else {
 981             $counter++;
 982             push( @drop, 0 );
 983             push( @{$self -> {'header'}}, $model_header[$i][0] );
 984           }
 985         }
 986
 987         foreach my $individual ( @{$self -> {'individuals'}} ) {
 988           $individual -> drop_columns( drop => \@drop );
 989         }
 990
 991         $self -> {'synced'} = 0;
 992 #       $Data::Dumper::Maxdepth = 2;
 993 #       die Dumper $self;
 994 #       die Dumper $self -> {'individuals'};
 995       }
 996 end drop_dropped
 997
 998 # }}} drop_dropped
 999
1000 # {{{ wrap
1001 start wrap
1002       {
1003         $self -> synchronize;
1004         $self -> cont_column( $cont_column ) if ( defined $cont_column );
1005         $self -> wrap_column( $wrap_column ) if ( defined $wrap_column );
1006         $self -> prepare_wrap( model_header => \@model_header );
1007         @secondary_columns = @{$self -> {'secondary_columns'}}
1008           if ( defined $self -> {'secondary_columns'} );
1009         @primary_columns = @{$self -> {'primary_columns'}}
1010           if ( defined $self -> {'primary_columns'} );
1011       }
1012 end wrap
1013 # }}} wrap
1014
1015 # {{{ unwrap
1016 start unwrap
1017       {
1018         $self -> {'cont_column'} = undef;
1019         $self -> {'wrap_column'} = undef;
1020         $self -> {'secondary_columns'} = undef;
1021         $self -> {'primary_columns'} = undef;
1022       }
1023 end unwrap
1024 # }}} unwrap
1025
1026 # {{{ prepare_wrap
1027
1028 start prepare_wrap
1029       {
1030         my $cont_column = $self -> {'cont_column'};
1031         my $wrap_column = $self -> {'wrap_column'};
1032         debug -> die( message => 'cont_column ('.$cont_column.') must be less or equal '.
1033                       'to the requested number of columns in each row ('.
1034                       ($wrap_column).')' )
1035           if ( $cont_column > $wrap_column );
1036         my @header;
1037         if ( scalar @model_header > 0 ) {
1038           @header = @model_header;
1039         } else {
1040           @header = @{$self -> {'header'}};
1041         }
1042
1043         my ( @primary, @secondary, @date_columns );
1044
1045         for ( my $i = 0; $i <= $#header; $i++ ) {
1046           my $name  = ref( $header[$i] ) eq 'ARRAY' ? $header[$i][0] : $header[$i];
1047           my $value = ref( $header[$i] ) eq 'ARRAY' ? $header[$i][1] : undef;
1048           next if ( $name eq 'ID' );
1049           my $found = 0;
1050           foreach my $prim ( @primary_column_names ) {
1051             if ( $name eq $prim or $value eq $prim ) {
1052               push( @primary, [$name, $i, $value] );
1053               $found = 1;
1054               my $col = ($#primary+2)>= $cont_column ? ($#primary+3) : ($#primary+2);
1055               push( @date_columns, $col ) if ( $name =~ /DAT(E|1|2|3)/ );
1056             }
1057           }
1058           push( @secondary, [$name, $i, $value] ) if ( not $found );
1059         }
1060
1061         my $prim_num = scalar @primary;
1062         debug -> die( message => 'The number of primary columns (that need to '.
1063                       'be part of the row with CONT=0) ('.($prim_num+1).
1064                       ') is larger than the required number of columns (wrap_column='.
1065                       $wrap_column.') - 1' )
1066           if ( scalar $prim_num > ($wrap_column-2) );
1067
1068         my ( $i, $dum ) = ( 0, 1 );
1069         my @tmp;
1070         for ( my $j = 1; $j <= $wrap_column; $j++ ) {
1071           if( $j == 1 ) {
1072             push( @tmp, ['ID', $self -> {'idcolumn'}-1] );
1073           } elsif ( $j == $wrap_column ) {
1074             if ( $j == $cont_column ) {
1075               push( @tmp, ['CONT', undef] );
1076             } else {
1077               my $val;
1078               if ( defined $primary[$i] ) {
1079                 $val = $primary[$i];
1080               } elsif ( defined $secondary[0] ) {
1081                 $val = shift(@secondary);
1082               } else {
1083                 $val = ['XX'.$dum++,$self -> {'idcolumn'}-1];
1084               }
1085               push( @tmp, $val );
1086               $i++;
1087             }
1088             push( @{$self -> {'primary_columns'}}, @tmp );
1089           } else {
1090             if ( $j == $cont_column ) {
1091               push( @tmp, ['CONT', undef] );
1092             } else {
1093               if ( $i <= $#primary ) {
1094                 push( @tmp, $primary[$i] );
1095                 $i++;
1096               } else {
1097                 my $val = defined $secondary[0] ? shift(@secondary) :
1098                   ['XX'.$dum++,$self -> {'idcolumn'}-1];
1099                 push( @tmp, $val );
1100               }
1101             }
1102           }
1103         }
1104
1105         my $i = 0;
1106         while ( $i <= $#secondary ) {
1107           my @tmp;
1108           for ( my $j = 1; $j <= $wrap_column; $j++ ) {
1109             if( $j == 1 ) {
1110               push( @tmp, ['ID', $self -> {'idcolumn'}-1] );
1111             } elsif ( $j == $wrap_column ) {
1112               if ( $j == $cont_column ) {
1113                 push( @tmp, ['CONT', undef] );
1114               } else {
1115                 my $val = defined $secondary[$i] ? $secondary[$i] :
1116                   ['XX'.$dum++,$self -> {'idcolumn'}-1];
1117                 push( @tmp, $val );
1118                 $i++;
1119               }
1120               unshift( @{$self -> {'secondary_columns'}}, \@tmp );
1121             } else {
1122               if ( $j == $cont_column ) {
1123                 push( @tmp, ['CONT', undef] );
1124               } else {
1125                 my $isdate = 0;
1126                 if ( $#date_columns >= 0 ) {
1127                   foreach my $col ( @date_columns ) {
1128                     # This is a date column which may have to be dropped
1129                     # and thus will not appear as a secondary
1130                     # column. Nothing should be pushed. The indexes in
1131                     # model::problem::pk::_format_record will be ok.
1132                     $isdate = 1 if ( $col == $j ) ;
1133                   }
1134                 }
1135                 if ( $isdate ) {
1136                   push( @tmp, ['XX'.$dum++,$self -> {'idcolumn'}-1] );
1137                 } else {
1138                   if ( $i <= $#secondary ) {
1139                     push( @tmp, $secondary[$i] );
1140                     $i++;
1141                   } else {
1142                     push( @tmp, ['XX'.$dum++,$self -> {'idcolumn'}-1] );
1143                   }
1144                 }
1145               }
1146             }
1147           }
1148         }
1149       }
1150 end prepare_wrap
1151
1152 # }}} prepare_wrap
1153
1154 # {{{ have_missing_data
1155 start have_missing_data
1156       {
1157         # Either I<column> or I<column_head> must be specified.
1158         #
1159         # This method looks through the data column with index I<column> or
1160         # (optional) header name I<column_head> and returns O if no missing
1161         # data indicator was found or 1 otherwise.
1162
1163         $self -> synchronize;
1164         my $first_id = $self -> {'individuals'}[0];
1165         debug -> die( message => "No individuals defined in data object based on ".
1166                       $self -> full_name ) unless ( defined $first_id );
1167         my @data_row = split( /,/ , $first_id -> subject_data -> [0] );
1168         unless ( defined $column  && defined( $data_row[$column-1] ) ) {
1169           unless(defined($column_head) && defined($self -> {'column_head_indices'}{$column_head})){
1170             die "Error in data -> have_missing_data: unknown column: \"$column_head\" or invalid column number: \"$column\"\n";
1171           } else {
1172             $column = $self -> {'column_head_indices'}{$column_head};
1173           }
1174         }
1175         $self -> flush if ( $self -> {'target'} eq 'disk' );
1176
1177         # In case anyone wonders, the ternary statment ( bool ? true :
1178         # false ) below will possibly make a minuscle memory
1179         # optimization. But hey, why not :)
1180
1181         $return_value = defined $self -> {'have_missing_data'} ? $self -> {'have_missing_data'} -> {$column} : 0;
1182       }
1183 end have_missing_data
1184 # }}} have_missing_data
1185
1186 # {{{ merge
1187 start merge
1188       {
1189         #$self -> synchronize;
1190         push( @{$self -> {'individuals'}}, @{$mergeobj -> individuals} );
1191       }
1192 end merge
1193 # }}} merge
1194
1195 # {{{ max
1196
1197 start max
1198       {
1199         # Either column or column_head must be specified. Column_head must be a string that
1200         # identifies a column in the (optional ) data file header.
1201
1202 # The if-statement below used to be a cache of allready calculated
1203 # means. But since individuals can be accessed in so many ways, we
1204 # don't know when this cache should be updated. Its easier to
1205 # recalculate the max. Maybe we can include this optimization in the
1206 # future, if it turns out to be a bottleneck
1207 #       my $tmp_column = $self -> {'column_head_indices'}{$column_head};
1208 #       if ( defined $self -> {'max'}[$tmp_column] ) {
1209 #         $return_value = $self -> {'max'}[$tmp_column] ;
1210 #       } else {
1211           $self -> synchronize;
1212           my $first_id = $self -> {'individuals'}[0];
1213           debug -> die( message => "data -> max: No individuals defined in data object based on " .
1214                         $self -> full_name ) unless defined $first_id;
1215
1216           my @data_row = split( /,/ , $first_id -> subject_data ->[0] );
1217
1218           unless ( defined $column  && defined( $data_row[$column-1] ) ) {
1219             unless (defined($column_head) && defined($self -> {'column_head_indices'}{$column_head})) {
1220               die "Error in data -> max: unknown column: \"$column_head\" or invalid column number: \"$column\"\n";
1221             } else {
1222               $column = $self -> {'column_head_indices'}{$column_head};
1223             }
1224           }
1225           foreach my $individual ( @{$self -> {'individuals'}} ) {
1226             my $ifactors = $individual -> factors( 'column' => $column );
1227             foreach ( keys %{$ifactors} ) {
1228               next if ( $_ == $self -> {'missing_data_token'} );
1229               if ( defined ($return_value) ) {
1230                 $return_value = $_ > $return_value ? $_ : $return_value;
1231               } else {
1232                 $return_value = $_;
1233               }
1234             }
1235           }
1236
1237 #         $self -> {'max'}[$column] = $return_value;
1238           $self -> flush if ( $self -> {'target'} eq 'disk' );
1239 #       }
1240       }
1241 end max
1242
1243 # }}} max
1244
1245 # {{{ min
1246
1247 start min
1248       {
1249         # See L</max>.
1250         my $tmp_column = $self -> {'column_head_indices'}{$column_head};
1251
1252 # The if-statement below used to be a cache of allready calculated
1253 # means. But since individuals can be accessed in so many ways, we
1254 # don't know when this cache should be updated. Its easier to
1255 # recalculate the min. Maybe we can include this optimization in the
1256 # future, if it turns out to be a bottleneck
1257 #       if ( defined $self -> {'min'}[$tmp_column] ) {
1258 #         $return_value = $self -> {'min'}[$tmp_column] ;
1259 #       } else {
1260           $self -> synchronize;
1261           my $first_id = $self -> {'individuals'}[0];
1262           die "data -> min: No individuals defined in data object based on ",
1263             $self -> full_name,"\n" unless defined $first_id;
1264
1265           my @data_row = split( /,/ , $first_id -> subject_data ->[0] );
1266
1267           unless ( defined $column  && defined( $data_row[$column-1] ) ) {
1268             unless (defined($column_head) && defined($self -> {'column_head_indices'}{$column_head})) {
1269               die "Error in data -> min: unknown column: \"$column_head\" or invalid column number: \"$column\"\n";
1270             } else {
1271               $column = $self -> {'column_head_indices'}{$column_head};
1272             }
1273           }
1274           foreach my $individual ( @{$self -> {'individuals'}} ) {
1275             my $ifactors = $individual -> factors( 'column' => $column );
1276             foreach ( keys %{$ifactors} ) {
1277               next if ( $_ == $self -> {'missing_data_token'} );
1278               if ( defined ($return_value) ) {
1279                 $return_value = $_ < $return_value ? $_ : $return_value;
1280               } else {
1281                 $return_value = $_;
1282               }
1283             }
1284           }
1285 #         $self -> {'min'}[$column] = $return_value;
1286           $self -> flush if ( $self -> {'target'} eq 'disk' );
1287 #       }
1288       }
1289 end min
1290
1291 # }}} min
1292
1293 # {{{ median
1294
1295 start median
1296       {
1297           # See L</max>.
1298         $self -> synchronize;
1299         my $first_id = $self -> {'individuals'}[0];
1300         die "data -> median: No individuals defined in data object based on ",
1301           $self -> full_name,"\n" unless defined $first_id;
1302
1303         my @data_row = split( /,/ , $first_id -> subject_data ->[0] );
1304
1305         unless ( defined $column  && defined( $data_row[$column-1] ) ) {
1306           unless(defined($column_head) && defined($self -> {'column_head_indices'}{$column_head})){
1307             die "Error in data -> median: unknown column: \"$column_head\" or invalid column number: \"$column\"\n";
1308           } else {
1309             $column = $self -> {'column_head_indices'}{$column_head};
1310           }
1311         }
1312
1313         if( defined $self -> {'median'}[$column] ){
1314           return $self -> {'median'}[$column];
1315         }
1316
1317         my @median_array;
1318
1319         foreach my $individual ( @{$self -> {'individuals'}} ) {
1320           if( $unique_in_individual ){
1321             my $ifactors = $individual -> factors( 'column' => $column );
1322
1323             foreach ( keys %{$ifactors} ) {
1324               next if ( $_ == $self -> {'missing_data_token'} );
1325               push( @median_array, $_ );
1326             }
1327           } else {
1328             my $ifactors = $individual -> subject_data;
1329
1330             for(my $i=0; $i<=$#{$ifactors}; $i++ ) {
1331               my @data_row = split( /,/ , $ifactors -> [$i] );
1332               next if ( $data_row[$column-1] == $self -> {'missing_data_token'} );
1333               push(@median_array, $data_row[$column-1]);
1334             }
1335           }
1336         }
1337         @median_array = sort {$a <=> $b} @median_array ;
1338
1339         if( @median_array % 2 ){
1340           $return_value = $median_array[$#median_array / 2];
1341         } else {
1342           $return_value = ( $median_array[@median_array / 2] +
1343                             $median_array[(@median_array - 2) / 2] ) / 2;
1344         }
1345
1346         $self -> {'median'}[$column] = $return_value;
1347       }
1348 end median
1349
1350 # }}} median
1351
1352 # {{{ mean
1353
1354 start mean
1355   {
1356      # Returns mean value of a column
1357      # If a individual contains more then 1 value (i.e. if an
1358      # individual has different values in different samples a mean
1359      # value of all individuals if calculate first, then the mean
1360      # value of the column If hi_cutoff is defined the mean function
1361      # will cut all value below the cutoff, and set their value to
1362      # 0. It's used to calculate the HI-mean/LOW-mean of a column for
1363      # e.g. Hockey-stick covariates If both hi_cutoff and low_cutoff
1364      # are defined only the hi_cutoff will be used.  See L</max>.
1365      my $tmp_column = $self -> {'column_head_indices'}{$column_head};
1366      $self -> synchronize;
1367      my $first_id = $self -> {'individuals'}[0];
1368        die "data -> mean: No individuals defined in data object based on ",
1369         $self -> full_name,"\n" unless defined $first_id;
1370
1371      my @data_row = split( /,/, $first_id -> subject_data ->[0] );
1372
1373      unless ( defined $column  && defined( $data_row[$column-1] ) ) {
1374        unless (defined($column_head) && defined($self -> {'column_head_indices'}{$column_head})) {
1375          die "Error in data -> mean: unknown column: \"$column_head\" or invalid column number: \"$column\"\n";
1376        } else {
1377               $column = $self -> {'column_head_indices'}{$column_head};
1378          }
1379      }
1380
1381         ## Here the calculation starts
1382         my $num_individuals = 0;
1383         my $sum = 0;
1384
1385      my $all_data_rows=0;
1386         foreach my $individual ( @{$self ->{'individuals'}} ) {
1387
1388           my $ifactors = $individual -> subject_data;
1389           my $individual_sum = 0;
1390           my $data_rows = 0;
1391           for(my $i=0; $i<=$#{$ifactors}; $i++ ) {
1392
1393             # data is stored in strings. We need to split them into an
1394             # array.
1395
1396             my @data_row = split( /,/, $ifactors -> [$i] );
1397             if ( $data_row[$column-1] == $self -> {'missing_data_token'} ) {
1398 #             print "Skipping row with missing data\n";
1399               next;
1400             }
1401
1402             if( defined $subset_column and not eval ( $data_row[$subset_column-1].$subset_syntax ) ) {
1403 #             print "Skipping row outside subset: syntax: ".($subset_column-1)." $subset_syntax\n";
1404               next;
1405             }
1406
1407             if (defined $hi_cutoff) {
1408               if ($data_row[$column-1]>$hi_cutoff) {
1409                 $individual_sum += $data_row[$column-1]-$hi_cutoff;
1410               }
1411             }
1412             else {
1413               if (defined $low_cutoff) {
1414                 if ($data_row[$column-1]<$low_cutoff) {
1415                   $individual_sum += $low_cutoff - $data_row[$column-1];
1416                 }
1417               }
1418               else {
1419                 $individual_sum += $data_row[$column-1];
1420               }
1421             }
1422             $data_rows++;
1423           }
1424           if( $global_mean ) {
1425             $sum += $individual_sum;
1426             $num_individuals += $data_rows;
1427           } else {
1428             if( $data_rows != 0 ) {
1429               $sum += $individual_sum/$data_rows;
1430             }
1431             $num_individuals ++;
1432           }
1433           $all_data_rows += $data_rows;
1434         }
1435      if( $num_individuals != 0 ) {
1436        $return_value = $sum / $num_individuals;
1437      }
1438 #     print "DR: $all_data_rows\n";
1439 #     print "\nNIM: $num_individuals $return_value\n";
1440 }
1441
1442 end mean
1443
1444 # }}} mean
1445
1446 # {{{ sd
1447
1448 start sd
1449   {
1450     # This sub returns standard deviation for a specific column
1451     # If there are more than one sample/individual the value used for that specific
1452     # individual is the mean value of its samples.
1453     # The cut-offs are for hockey stick variables. I.e. If one individual value is
1454     # lower than the hi-cutoff the individual value will be zero.
1455     # HI_cutoff is used to calculate the HI-mean of a column.
1456     # If cut_off is undef it won't be used
1457     # See L</max>.
1458      my $tmp_column = $self -> {'column_head_indices'}{$column_head};
1459      $self -> synchronize;
1460      my $first_id = $self -> {'individuals'}[0];
1461      debug -> die( message => "No individuals defined in data object based on ".
1462                    $self -> full_name ) unless defined $first_id;
1463
1464      my @data_row = split( /,/ , $first_id -> subject_data ->[0] );
1465
1466      unless ( defined $column  && defined( $data_row[$column-1] ) ) {
1467        unless (defined($column_head) && defined($self -> {'column_head_indices'}{$column_head})) {
1468          debug -> die( message => "Unknown column: \"$column_head\" or "
1469                        ."invalid column number: \"$column\"" );
1470        } else {
1471          $column = $self -> {'column_head_indices'}{$column_head};
1472        }
1473      }
1474
1475      ## Here the calculation starts
1476      my $num_individuals = 0;
1477      my $sum = 0;
1478      my $mean;
1479      if (defined $hi_cutoff) {
1480        $mean = $self->mean(column   => $column,
1481                            hi_cutoff => $hi_cutoff,
1482                            global_mean => $global_sd );
1483      } elsif (defined $low_cutoff) {
1484        $mean = $self->mean(column   => $column,
1485                            low_cutoff => $low_cutoff,
1486                            global_mean => $global_sd );
1487      } else {
1488        $mean = $self->mean( column        => $column,
1489                             subset_column => $subset_column,
1490                             subset_syntax => $subset_syntax,
1491                             global_mean => $global_sd );
1492      }
1493
1494      foreach my $individual ( @{$self -> {'individuals'}} ) {
1495        my $ifactors = $individual -> subject_data;
1496        my $individual_sum = 0;
1497        my $data_rows = 0;
1498        for(my $i=0; $i<=$#{$ifactors}; $i++ ) {
1499
1500          # data is stored in strings. We need to split them into an
1501          # array.
1502
1503          my @data_row = split( /,/, $ifactors -> [$i] );
1504
1505          if ( $data_row[$column-1] == $self -> {'missing_data_token'} ) {
1506 #          print "Skipping row with missing data\n";
1507            next;
1508          }
1509
1510          if( defined $subset_column and not eval ( $data_row[$subset_column-1].$subset_syntax ) ) {
1511 #          print "Skipping row outside subset: syntax: ".($subset_column-1)." $subset_syntax\n";
1512            next;
1513          }
1514
1515          if (defined $hi_cutoff) {
1516            if ($ifactors->[$i]->[$column-1]>$hi_cutoff) {
1517              if( $global_sd ) {
1518                $individual_sum += ($data_row[$column-1] - $hi_cutoff - $mean) ** 2;
1519              } else {
1520                $individual_sum += $data_row[$column-1]-$hi_cutoff;
1521              }
1522            }
1523          } else {
1524            if (defined $low_cutoff) {
1525              if ($ifactors->[$i]->[$column-1]<$low_cutoff) {
1526                if( $global_sd ) {
1527                  $individual_sum += ($low_cutoff - $data_row[$column-1] - $mean) ** 2;
1528                } else {
1529                  $individual_sum += $low_cutoff - $data_row[$column-1];
1530                }
1531              }
1532            } else {
1533              if( $global_sd ) {
1534                $individual_sum += ($data_row[$column-1] - $mean) ** 2;
1535              } else {
1536                $individual_sum += $data_row[$column-1];
1537              }
1538            }
1539          }
1540          $data_rows++;
1541        }
1542        if( $global_sd ) {
1543          $sum += $individual_sum;
1544          $num_individuals += $data_rows;
1545        } else {
1546          if( $data_rows != 0 ) {
1547            $sum += ($individual_sum/$data_rows - $mean) ** 2;
1548          }
1549          $num_individuals++;
1550        }
1551      }
1552      if( $num_individuals < 2 ) {
1553        $return_value = 0;
1554      } else {
1555        if( $num_individuals != 0 ) {
1556          $return_value = (1/($num_individuals-1)*$sum) ** 0.5;
1557        }
1558      }
1559    }
1560
1561 end sd
1562
1563 # }}} sd
1564
1565 # {{{ range
1566 start range
1567       {
1568         # See L</max>.
1569         my $tmp_column = $self -> {'column_head_indices'}{$column_head};
1570         if ( defined $self -> {'range'}[$tmp_column] ) {
1571           $return_value = $self -> {'range'}[$tmp_column];
1572         } else {
1573           my $old_target = $self -> {'target'};
1574           $self -> {'target'} = 'mem';
1575           $self -> synchronize;
1576           $return_value = $self -> max( column      => $column,
1577                                         column_head => $column_head ) -
1578                                           $self -> min( column      => $column,
1579                                                         column_head => $column_head );
1580           $self -> {'range'}[$column] = $return_value;
1581           if ( $old_target eq 'disk' ) {
1582             $self -> flush if ( $self -> {'target'} eq 'disk' );
1583             $self -> {'target'} = 'disk';
1584           }
1585         }
1586       }
1587 end range
1588 # }}} range
1589
1590 # {{{ recalc_column
1591 start recalc_column
1592       {
1593           # Recalculates a column based on expression. Also, see L</max>.
1594         $self -> synchronize;
1595
1596         # Check if $column(-index) is defined and valid, else try to find index using column_head
1597         my $first_id = $self -> {'individuals'}[0];
1598         die "data -> recalc_column: No individuals defined in data object based on ",
1599           $self -> full_name,"\n" unless defined $first_id;
1600
1601         my @data_row = split( /,/ , $first_id -> subject_data ->[0] );
1602
1603         unless ( defined $column  && defined( $data_row[$column-1] ) ) {
1604           if(defined($column_head) && defined($self -> {'column_head_indices'}{$column_head})){
1605             die "Error in data -> recalc_column: unknown column: \"$column_head\" or column number: \"$column\"\n";
1606           } else {
1607             $column = $self -> {'column_head_indices'}{$column_head};
1608           }
1609         }
1610
1611         for my $individual ( @{$self -> {'individuals'}} ) {
1612           $individual -> recalc_column( column     => $column,
1613                                         expression => $expression );
1614         }
1615       }
1616 end recalc_column
1617 # }}} recalc_column
1618
1619 # {{{ renumber_ascending
1620
1621 start renumber_ascending
1622       {
1623 # Renumbers the individuals (changes the subject identifiers) so that
1624 # all have unique integer numbers starting with start_at and
1625 # ascending. The primary use of this
1626 # method is not to order the individuals after their identifiers but to
1627 # ensure that all individuals have unique identifiers.
1628
1629         $self -> synchronize;
1630         foreach my $individual ( @{$self -> {'individuals'}} ) {
1631           $individual -> idnumber ( $start_at++ );
1632         }
1633         $self -> {'synced'} = 0;
1634       }
1635 end renumber_ascending
1636
1637 # }}} renumber_ascending
1638
1639 # {{{ renumber_descending
1640
1641 start renumber_descending
1642       {
1643           # See L</renumber_ascending>.
1644         $self -> synchronize;
1645         foreach my $individual ( @{$self -> {'individuals'}} ) {
1646           $individual -> idnumber ( $start_at-- );
1647         }
1648         $self -> {'synced'} = 0;
1649       }
1650 end renumber_descending
1651
1652 # }}} renumber_descending
1653
1654 # {{{ single_valued_data
1655
1656 start single_valued_data
1657       {
1658         # Usage:
1659         #
1660         # ($single_value_data_set, $remainder, $column_indexes) =
1661         #      $data_object -> single_valued_data( subset_name         => 'subset.dta',
1662         #                                          remainder_name      => 'remainder.dta',
1663         #                                          target              => 'disk',
1664         #                                          do_not_test_columns => [1..18,24,26];
1665         #
1666         # my $single_value_column_indexes = $column_indexes -> [0];
1667         # my $all_other_column_indexes    = $column_indexes -> [1];
1668         #
1669         # Analyses the content of each column, based on the
1670         # ID column, and returns two new data objects: One
1671         # that contains all columns that is has only one value per
1672         # individual and one that contains the
1673         # remainding data. This is useful for creating compact 'extra'
1674         # data sets that can be read in via user-defined sub-routines
1675         # when the number of columns needed exceeds the maximum that
1676         # NONMEM allows (e.g. 20 in NONMEM version V).
1677         #
1678         # The I<do_not_test_columns> argument specifies on which columns
1679         # to skip the single value test
1680
1681         my @multi_value_flags;
1682         my @individuals = @{$self -> {'individuals'}};
1683         # Initiate the flags:
1684         if ( defined $individuals[0] ) {
1685           my @data = @{$individuals[0] -> {'subject_data'}};
1686           my @data_row = split( /,/ , $data[0] );
1687           for ( my $i = 0; $i < scalar @data_row; $i++ ) {
1688             my $dnt_flag = 0;
1689             foreach my $dntc ( @do_not_test_columns ) {
1690               $dnt_flag = 1 if ( $i == $dntc - 1 );
1691             }
1692             $multi_value_flags[$i] = $dnt_flag;
1693           }
1694         } else {
1695           die "data -> single_valued_data: No data in ID number 1\n";
1696         }
1697         # Collect the stats
1698         for ( my $id = 0; $id <= $#individuals; $id++ ) {
1699           my @data = @{$individuals[$id] -> {'subject_data'}};
1700           my @data_row = split( /,/, $data[0] );
1701           for ( my $j = 0; $j < scalar @data_row; $j++ ) {
1702             my %col_unique;
1703             for ( my $i = 0; $i <= $#data; $i++ ) {
1704               my @data_row = split( /,/ , $data[$i] );
1705               $col_unique{$data_row[$j]}++;
1706             }
1707             my $factors = scalar keys %col_unique;
1708             $multi_value_flags[$j]++ if ( $factors > 1 );
1709           }
1710         }
1711         for ( my $i = 0; $i <= $#multi_value_flags; $i++ ) {
1712           if ( $multi_value_flags[$i] ) {
1713             push ( @{$column_indexes[1]}, $i + 1);
1714           } else {
1715             push ( @{$column_indexes[0]}, $i + 1);
1716           }
1717         }
1718         ( $single_value_data_set, $remainder ) =
1719           $self -> subset_vertically( column_indexes      => $column_indexes[0],
1720                                       subset_name         => $subset_name,
1721                                       return_remainder    => 1,
1722                                       remainder_name      => $remainder_name,
1723                                       target              => $target,
1724                                       keep_first_row_only => 1);
1725       }
1726 end single_valued_data
1727
1728 # }}}
1729
1730 # {{{ subset_vertically
1731
1732 start subset_vertically
1733       {
1734         # Usage:
1735         #
1736         # $subset = $data_object -> subset_vertically ( column_indexes => [1,2,6],
1737         #                                               subset_name    => 'subset.dta' );
1738         #
1739         # This basic usage returns a new data object containing
1740         # columns 1,2 and 6 from the original data plus the
1741         # idcolumn. The new data object will be associated with the
1742         # file 'subset.dta'.
1743         #
1744         # You get the remaining data, i.e. the original data minus
1745         # the created subset by specifying
1746         #
1747         # ( $subset, $remainder ) =
1748         #      $data_object -> subset_vertically ( column_indexes   => [1,2,6],
1749         #                                          subset_name      => 'subset.dta',
1750         #                                          return_remainder => 1,
1751         #                                          remainder_name   => 'remainder.dta' );
1752         #
1753         # If you would like to flush the created data sets to disk and
1754         # save memory, set the I<target> argument to 'disk'. The
1755         # default value 'mem' will keep the whole data object in RAM.
1756         #
1757         # The I<keep_first_row_only> argument can be used to reduce
1758         # the size of the subset data obejct by excluding all but the
1759         # first row of data from each individual.
1760         #
1761         my @individuals = @{$self -> {'individuals'}};
1762         # Create remainder index array if necessary
1763         my @remainder_indexes;
1764         if ( defined $individuals[0] ) {
1765           my @data = @{$individuals[0] -> {'subject_data'}};
1766           my $idcolumn = $individuals[0] -> {'idcolumn'};
1767 #         print "IC: $idcolumn\n";
1768           my $id_flag = 0;
1769           foreach my $use_index ( @column_indexes ) {
1770             $id_flag = 1 if ( $use_index == $idcolumn );
1771           }
1772           if ( $return_remainder ) {
1773 #           @remainder_indexes = ( $idcolumn );
1774             for ( my $i = 0; $i < scalar split(/,/,$data[0]); $i++ ) {
1775               my $rem_flag = 1;
1776               foreach my $use_index ( @column_indexes ) {
1777                 $rem_flag = 0 if ( $i == $use_index -1 );
1778 # or
1779 #                                  $i == $idcolumn -1 );
1780               }
1781               push( @remainder_indexes, $i + 1 ) if ( $rem_flag );
1782             }
1783             unshift( @remainder_indexes, $idcolumn ) if ( $id_flag );
1784           }
1785           unshift( @column_indexes, $idcolumn ) unless ( $id_flag );
1786         } else {
1787           die "data -> single_valued_data: No data in ID number 1\n";
1788         }
1789
1790 #       print "SS: @column_indexes\n";
1791 #       print "R : @remainder_indexes\n";
1792
1793         my @new_ids;
1794         my @new_ids_2;
1795         for ( my $id = 0; $id <= $#individuals; $id++ ) {
1796           my $idnumber = $individuals[$id] -> idnumber;
1797           my $idcolumn = $individuals[$id] -> idcolumn;
1798           my @data = @{$individuals[$id] -> {'subject_data'}};
1799           my @new_data;
1800           my @new_data_2;
1801           my $use_rows = $keep_first_row_only ? 0 : $#data;
1802           for ( my $i = 0; $i <= $use_rows; $i++ ) {
1803             my @new_row;
1804             my @data_row = split( /,/, $data[$i] );
1805             foreach my $use_index ( @column_indexes ) {
1806               push( @new_row, $data_row[$use_index-1] );
1807             }
1808 #           print "@new_row $#new_row\n";
1809             push( @new_data, join( ',', @new_row ) );
1810           }
1811           for ( my $i = 0; $i <= $#data; $i++ ) {
1812             if ( $return_remainder ) {
1813               my @new_row_2;
1814               my @data_row = split( /,/, $data[$i] );
1815               foreach my $use_index ( @remainder_indexes ) {
1816                 push( @new_row_2, $data_row[$use_index-1] );
1817               }
1818 #             print "@new_row_2 $#new_row_2\n";
1819               push( @new_data_2, join( ',' , @new_row_2 ) );
1820             }
1821           }
1822           my $new_id = data::individual -> new( idnumber     => $idnumber,
1823                                                 idcolumn     => $idcolumn,
1824                                                 subject_data => \@new_data );
1825           push( @new_ids, $new_id );
1826           if ( $return_remainder ) {
1827             my $new_id_2;
1828             $new_id_2 = data::individual -> new( idnumber     => $idnumber,
1829                                                  idcolumn     => $idcolumn,
1830                                                  subject_data => \@new_data_2 );
1831           push( @new_ids_2, $new_id_2 );
1832           }
1833         }
1834         my @header = @{$self -> {'header'}};
1835         my @new_header;
1836         foreach my $use_index ( @column_indexes ) {
1837           push( @new_header, @header[$use_index-1] );
1838         }
1839         my $comment;
1840         if( defined $self -> {'comment'} ){
1841           my @comment = @{$self -> {'comment'}};
1842           $comment = \@comment;
1843         }
1844         $subset = data -> new ( filename             => $subset_name,
1845                                 directory            => $self -> {'directory'},
1846                                 ignoresign           => $self -> {'ignoresign'},
1847                                 header               => \@new_header,
1848                                 comment              => $comment,
1849                                 individuals          => \@new_ids,
1850                                 target               => $target,
1851                                 ignore_missing_files => 1 );
1852         if ( $return_remainder ) {
1853           my @new_header_2;
1854           foreach my $use_index ( @remainder_indexes ) {
1855             push( @new_header_2, @header[$use_index-1] );
1856           }
1857           $remainder = data -> new ( filename             => $remainder_name,
1858                                      directory            => $self -> {'directory'},
1859                                      ignoresign           => $self -> {'ignoresign'},
1860                                      header               => \@new_header_2,
1861                                      comment              => $comment,
1862                                      individuals          => \@new_ids_2,
1863                                      target               => $target,
1864                                      ignore_missing_files => 1 );
1865         }
1866       }
1867 end subset_vertically
1868
1869 # }}}
1870
1871 # {{{ subsets
1872
1873 start subsets
1874       {
1875 #       if ( defined $expression and defined $bins ) {
1876 #         die "data -> subset: expression and bins may not both be specified\n";
1877 #       }
1878 #       if ( not ( defined $expression or defined $bins ) ) {
1879 #         die "data -> subset: expression or bins must be specified\n";
1880 #       }
1881         $self -> synchronize;
1882         my @header  = @{$self -> {'header'}};
1883         my @comment = defined $self -> {'comment'} ? @{$self -> {'comment'}} : ();
1884         my @subset_ids= ();
1885         my %rnd_ids;
1886         my $key = 0;
1887         my @ids = @{$self -> {'individuals'}};
1888         if ( defined $stratify_on ) {
1889           my $work_data = $self -> copy( filename => 'work_data.dta',
1890                                          target   => 'mem' );
1891           my %strata = %{$work_data -> factors( column => $stratify_on )};
1892 #         $Data::Dumper::Maxdepth = 1;
1893 #         print Dumper \%strata;
1894
1895           while ( my ( $factor, $keys ) = each %strata ) {
1896               foreach my $key ( @{$keys} ) {
1897                   my $rnd_num = rand;
1898                   while ( defined $rnd_ids{$factor}{$rnd_num} ) {
1899                       $rnd_num = rand;
1900                   }
1901                   $rnd_ids{$factor}{$rnd_num} = $ids[$key];
1902               }
1903           }
1904           my $first = 1;
1905           while ( my ( $factor, $rnd_nums ) = each %rnd_ids ) {
1906               my @sort_rnd_nums = sort { $a <=> $b } keys %{$rnd_nums};
1907               for ( my $i = 0; $i <= $#sort_rnd_nums; $i ) {
1908                   for ( my $j = 0; $j < $bins; $j++ ) {
1909                       if ( $first ) {
1910                           push( @subset_ids, [$rnd_ids{$factor}{$sort_rnd_nums[$i]} -> copy] );
1911                           push( @incl_ids, [$rnd_ids{$factor}{$sort_rnd_nums[$i]} -> idnumber] );
1912                       } else {
1913                           push( @{$subset_ids[$j]}, $rnd_ids{$factor}{$sort_rnd_nums[$i]} -> copy );
1914                           push( @{$incl_ids[$j]}, $rnd_ids{$factor}{$sort_rnd_nums[$i]} -> idnumber );
1915                       }
1916                       $i++;
1917                       last if $i > $#sort_rnd_nums;
1918                   }
1919                   $first = 0;
1920               }
1921           }
1922           for ( my $j = 0; $j < $bins; $j++ ) {
1923             my $sdata = data -> new ( header               => \@header,
1924                                       comment              => \@comment,
1925                                       ignoresign           => $self -> {'ignoresign'},
1926                                       individuals          => $subset_ids[$j],
1927                                       ignore_missing_files => 1,
1928                                       target               => 'disk',
1929                                       idcolumn             => $self -> {'idcolumn'},
1930                                       filename             => "subset_$j.dta" );
1931             #$sdata -> _write;
1932             push( @subsets, $sdata );
1933           }
1934         } else {
1935           for ( my $i = 0; $i <= $#ids; $i++ ) {
1936             my $rnd_num = rand;
1937             while ( defined $rnd_ids{$rnd_num} ) {
1938               $rnd_num = rand;
1939             }
1940             $rnd_ids{$rnd_num} = $ids[$i];
1941           }
1942           my @keys = sort { $a <=> $b } keys %rnd_ids;
1943           my $first = 1;
1944           for ( my $i = 0; $i <= $#keys; $i ) {
1945             for ( my $j = 0; $j < $bins; $j++ ) {
1946               if ( $first ) {
1947                 push( @subset_ids, [$rnd_ids{$keys[$i]} -> copy] );
1948                 push( @incl_ids, [$rnd_ids{$keys[$i]} -> idnumber] );
1949               } else {
1950                 push( @{$subset_ids[$j]}, $rnd_ids{$keys[$i]} -> copy );
1951                 push( @{$incl_ids[$j]}, $rnd_ids{$keys[$i]} -> idnumber );
1952               }
1953               $i++;
1954               last if $i > $#keys;
1955             }
1956             $first = 0;
1957           }
1958           for ( my $j = 0; $j < $bins; $j++ ) {
1959             my $sdata = data -> new ( header               => \@header,
1960                                       comment              => \@comment,
1961                                       ignoresign           => $self -> {'ignoresign'},
1962                                       individuals          => $subset_ids[$j],
1963                                       ignore_missing_files => 1,
1964                                       target               => $target,
1965                                       idcolumn             => $self -> {'idcolumn'},
1966                                       filename             => "subset_$j.dta" );
1967             #$sdata -> _write;
1968             push( @subsets, $sdata );
1969           }
1970         }
1971       }
1972 end subsets
1973
1974 # }}} subsets
1975
1976 # {{{ subset
1977
1978 start subset
1979       {
1980         $self -> synchronize;
1981         my @header  = @{$self -> {'header'}};
1982         my @comment = defined $self -> {'comment'} ? @{$self -> {'comment'}} : ();
1983         my @subset_inds = ();
1984         my $key = 0;
1985         foreach my $individual ( @{$self -> {'individuals'}} ) {
1986           if ( $individual -> evaluate_expression( column     => $based_on,
1987                                                    expression => $expression ) ) {
1988             push( @subset_inds, $individual -> copy );
1989             push( @incl_individuals, $individual -> idnumber );
1990             push( @included_keys, $key );
1991           }
1992           $key++;
1993         }
1994         $subset = data -> new ( header      => \@header,
1995                                 comment     => \@comment,
1996                                 ignoresign  => $self -> {'ignoresign'},
1997                                 individuals => \@subset_inds,
1998                                 idcolumn    => $self -> {'idcolumn'},
1999                                 filename    => "subset.dta" );
2000       }
2001 end subset
2002
2003 # }}} subset
2004
2005 # {{{ target
2006
2007 start target
2008       {
2009         if ( $parm eq 'disk' and $self -> {'target'} eq 'mem' ) {
2010           $self -> {'target'} = 'disk';
2011           $self -> flush;
2012         } elsif ( $parm eq 'mem' and $self -> {'target'} eq 'disk' ) {
2013           $self -> {'target'} = 'mem';
2014           $self -> synchronize;
2015         }
2016       }
2017 end target
2018
2019 # }}}
2020
2021 # {{{ _write
2022
2023 start _write
2024     {
2025       die "ERROR: data -> _write: No filename set in data object.\n"
2026           if( $filename eq '' );
2027
2028 #      $Data::Dumper::Maxdepth = 2;
2029 #      die Dumper $self -> {'individuals'};
2030
2031       if( not defined $self -> {'individuals'} ){
2032
2033         # If we don't have any individuals and write to a new
2034         # filename, we must first read individuals from the old
2035         # file. A call to synchronize will do that. There is no risk
2036         # of a infinite loop here since synchronize allways writes to
2037         # "full_name".
2038
2039         unless( $filename eq $self -> full_name ){
2040           $self -> synchronize;
2041         }
2042       }
2043
2044       open(FILE,">$filename") ||
2045           die "Could not create $filename\n";
2046       my $data_ref = $self -> format_data;
2047       my @data = @{$data_ref};
2048       for ( @data ) {
2049         print ( FILE );
2050       }
2051       close(FILE);
2052
2053 #       if ( $PsN::config -> {'_'} -> {'use_database'} and
2054 #          $self -> {'use_data_table'} ) {
2055 #       # Backslashes messes up the sql syntax
2056 #       my $file_str = $self->{'filename'};
2057 #       my $dir_str = $self->{'directory'};
2058 #       $file_str =~ s/\\/\//g;
2059 #       $dir_str =~ s/\\/\//g;
2060
2061 #       # md5sum
2062 #       my $md5sum = md5_hex(OSspecific::slurp_file($self-> full_name ));
2063 #       my ( $date_str, $time_str );
2064 #       if ( $Config{osname} eq 'MSWin32' ) {
2065 #         $date_str = `date /T`;
2066 #         $time_str = ' '.`time /T`;
2067 #       } else {
2068 #         # Assuming UNIX
2069 #         $date_str = `date`;
2070 #       }
2071 #       chomp($date_str);
2072 #       chomp($time_str);
2073 #       my $date_time = $date_str.$time_str;
2074 #       my $dbh = DBI -> connect("DBI:mysql:host=".$PsN::config -> {'_'} -> {'database_server'}.
2075 #                              ";databse=".$PsN::config -> {'_'} -> {'project'},
2076 #                                $PsN::config -> {'_'} -> {'user'},
2077 #                               $PsN::config -> {'_'} -> {'password'},
2078 #                                {
2079 #                                 'RaiseError' => 1});
2080 #       my $sth;
2081 #       if ( defined $self -> {'data_id'} ) {
2082 #         $sth = $dbh -> prepare( "UPDATE ".$PsN::config -> {'_'} -> {'project'}.
2083 #                              ".data ".
2084 #                                 "SET filename='$file_str',date='$date_time',".
2085 #                                 "directory='$dir_str',md5sum='$md5sum' ".
2086 #                                 "WHERE data_id='".$self -> {'data_id'}."'" );
2087 #         $sth -> execute or debug -> die( message => $sth->errstr ) ;
2088 #       } else {
2089 #         $sth = $dbh -> prepare("INSERT INTO ".$PsN::config -> {'_'} -> {'project'}.
2090 #                              ".data (filename,date,directory,md5sum) ".
2091 #                                "VALUES ('$file_str', '$date_time', '$dir_str','".
2092 #                                $md5sum."' )");
2093 #         $sth -> execute;
2094 #         $self -> {'data_id'} = $sth->{'mysql_insertid'};
2095 #       }
2096 #       $sth -> finish;
2097 #       $dbh -> disconnect;
2098 #       }
2099     }
2100 end _write
2101
2102 # }}} _write
2103
2104 # {{{ flush
2105 start flush
2106     {
2107       # synchronizes the object with the file on disk and empties
2108       # most of the objects attributes to save memory.
2109       if( defined $self -> {'individuals'} and
2110           ( !$self -> {'synced'} or $force ) ) {
2111         $self -> _write;
2112       }
2113 #      $self -> {'header'} = undef;
2114       $self -> {'comment'} = undef;
2115       $self -> {'individuals'} = undef;
2116       $self -> {'synced'} = 0;
2117       $self -> {'column_head_indices'} = undef;
2118       $self -> {'have_missing_data'} = undef;
2119     }
2120 end flush
2121 # }}} flush
2122
2123 # {{{ synchronize
2124
2125 start synchronize
2126     {
2127       # synchronizes the object with the file on disk
2128       unless( $self -> {'synced'} ){
2129         if( defined $self -> {'individuals'} and
2130             scalar @{$self -> {'individuals'}} > 0 ){
2131           # We should not read new data from file if we
2132           # have an individuals defined?
2133           # Perhaps there should be an attribute
2134           # 'from_file' that overrides this and reads in
2135           # the data from the file specified in filename
2136           # and overwrites whatever the object already
2137           # contains?
2138 #           if( -e $self -> {'filename'} ){
2139 #             $self -> _read_header;
2140 #             $self -> _read_individuals;
2141 #           }
2142           $self -> _write;
2143         } else {
2144           if( -e $self -> full_name ){
2145             unless( defined $self -> {'header'} and scalar @{$self -> {'header'}} > 0 ){
2146               $self -> _read_header;
2147             }
2148             $self -> _read_individuals;
2149           } else {
2150             debug -> die( message => "Fatal error: datafile: " . $self -> full_name . " does not exist." );
2151             return;
2152           }
2153         }
2154       }
2155       my $i = 1;
2156       foreach my $head ( @{$self -> {'header'}} ){
2157         $self -> {'column_head_indices'} -> {$head} = $i;
2158         $i++;
2159       }
2160       $self -> {'synced'} = 1;
2161     }
2162 end synchronize
2163
2164 # }}} synchronize
2165
2166 # {{{ _fisher_yates_shuffle
2167
2168 start _fisher_yates_shuffle
2169       {
2170         my $arr_ref = $parm{'array'};
2171         debug -> warn( level   => 1,
2172                        message => "Array of zero length received" )
2173           if ( scalar @{$arr_ref} < 1 );
2174         my $i;
2175         for ($i = @$arr_ref; --$i; ) {
2176           my $j = random_uniform_integer(1,0,$i);
2177 #          my $j = int rand ($i+1);
2178 #         print "$j $j_new\n";
2179           @$arr_ref[$i,$j] = @$arr_ref[$j,$i];
2180         }
2181       }
2182 end _fisher_yates_shuffle
2183
2184 # }}} _fisher_yates_shuffle
2185
2186 # {{{ _read_header
2187
2188 start _read_header
2189       {
2190         my $filename   = $self -> full_name;
2191         my $ignoresign = $self -> ignoresign;
2192         my ( @data, @new_record, $row, $tmp_row, @header, $hdrstring );
2193
2194         open(DATAFILE,"$filename") ||
2195           die "Could not open $filename for reading";
2196         my $columns;
2197         while (<DATAFILE>) {
2198           $tmp_row    = $_;
2199           #     @new_record = split(/\,|\s+/,$_);
2200           if ( ! (/^\s*\d+|^\s*\./) ) {
2201             $data[$row] = $tmp_row;
2202             $row++;
2203           } else {
2204             # We have reached the first data-row, return.
2205             $columns = scalar split(/\,\s*|\s+/);
2206             last;
2207           }
2208         }
2209         close(DATAFILE);
2210
2211         if ( defined $self -> {'cont_column'} and not $self -> {'table_file'} ) {
2212           my $data_len = $#data;
2213           for ( my $i = $data_len; $i >= 0; $i-- ) {
2214             my @arr = split(/\,\s*|\s+/,$data[$i]);
2215             if ( $arr[$self -> {'cont_column'}-1] eq 'CONT' ) {
2216               my $start = $i == $data_len ? 0 : 1;
2217               for ( my $j = $start; $j <= $#arr; $j++ ) {
2218                 if ( $j != ($self -> {'cont_column'}-1) ) {
2219                   push( @header, $arr[$j] );
2220                 }
2221               }
2222               pop( @data );
2223             }
2224           }
2225           $header[0] =~ s/$ignoresign//
2226               if ( defined $self->ignoresign );
2227           shift( @header ) if ( $header[0] eq "" );
2228         } else {
2229           chomp( $hdrstring = pop(@data));
2230           @header = split(/\,\s*|\s+/,$hdrstring);
2231           $header[0] =~ s/$ignoresign//
2232               if ( defined $self->ignoresign );
2233           shift( @header ) if ( $header[0] eq "" );
2234           if( $self -> {'table_file'} ) {
2235             my @new_header;
2236             for( my $i = 1; $i <= scalar @header; $i++ ) {
2237               if( $header[$i-1] eq 'CONT' ) {
2238                 if ( defined $self -> {'cont_column'} and not $i == $self -> {'cont_column'} ) {
2239                   debug -> warn( level   => 1,
2240                                  message => "The supplied columns for the CONT data item (".
2241                                  $self -> {'cont_column'}.") does not match the column where the CONT ".
2242                                  "header was found ($i), using $i" );
2243                 }
2244                 $self -> {'cont_column'} = $i;
2245               } else {
2246                 push( @new_header, $header[$i-1] );
2247               }
2248             }
2249             @header = @new_header;
2250             for( my $i = 1; $i <= scalar @header; $i++ ) {
2251               if( $header[$i-1] eq 'ID' ) {
2252                 if ( defined $self -> {'idcolumn'} and not $i == $self -> {'idcolumn'} ) {
2253                   debug -> warn( level   => 1,
2254                                  message => "The supplied columns for the ID data item (".
2255                                  $self -> {'idcolumn'}.") does not match the column where the CONT ".
2256                                  "header was found ($i), using $i" );
2257                 }
2258                 $self -> {'idcolumn'} = $i;
2259               }
2260             }
2261           }
2262         }
2263
2264 # I'm not certain on how to deal with this conflict. I'm leaving it commented because I believe this code should not be here.
2265 #
2266 #<<<<<<< data_subs.pm
2267 #       $header[0] =~ s/$ignoresign//
2268 #         if ( defined $self->ignoresign );
2269 #       shift( @header ) if ( $header[0] eq "" );
2270 #=======
2271 #>>>>>>> 1.28
2272
2273 # It is ok with data sets without a header.
2274 #       unless( scalar @header > 0 ){ debug -> die( message => 'Datafile ' . $self -> full_name . ' is empty.' ); }
2275
2276         $self -> {'header'} = \@header;
2277         $self -> {'comment'} = \@data;
2278 #       if ( $PsN::config -> {'_'} -> {'use_database'} and
2279 #            $self -> {'use_data_table'} ) {
2280 #         my $dbh = DBI -> connect("DBI:mysql:host=".$PsN::config -> {'_'} -> {'database_server'}.
2281 #                              ";databse=".$PsN::config -> {'_'} -> {'project'},
2282 #                                  $PsN::config -> {'_'} -> {'user'},
2283 #                               $PsN::config -> {'_'} -> {'password'},
2284 #                                  {'RaiseError' => 1});
2285 #         if ( scalar @header < 1 ) {
2286 #           for ( my $i = 1; $i <= $columns; $i++ ) {
2287 #             push( @header, $i );
2288 #           }
2289 #         }
2290 #         for ( my $i = 0; $i <= $#header; $i++ ) {
2291 #           my $sth = $dbh -> prepare("INSERT INTO ".$PsN::config -> {'_'} -> {'project'}.
2292 #                              ".data_column ".
2293 #                                     "(name,number,data_id) ".
2294 #                                     "VALUES ('".$header[$i]."', '".($i+1).
2295 #                                     "', '".$self -> {'data_id'}."' )");
2296 #           $sth -> execute;
2297 #           push( @{$self -> {'data_column_ids'}}, $sth->{'mysql_insertid'} );
2298 #           $sth -> finish;
2299 #         }
2300 #         $dbh -> disconnect;
2301 #       }
2302       }
2303 end _read_header
2304
2305 # }}} _read_header
2306
2307 # {{{ _read_individuals
2308
2309 start _read_individuals
2310       {
2311         my $idcol        = $self -> idcolumn;
2312         my $filename     = $self -> full_name;
2313         debug -> warn( level   => 1,
2314                        message => "Building array of individuals from file " . $self -> {'filename'} );
2315         open(DATAFILE,"$filename") ||
2316           die "Could not open $filename for reading";
2317         my ( @new_row, $new_ID, $old_ID, @init_data );
2318         my $buffer;
2319         my $lines = 0;
2320         while (sysread DATAFILE, $buffer, 4096) {
2321           $lines += ($buffer =~ tr/\n//);
2322         }
2323         seek( DATAFILE, 0,0 );
2324
2325         # For status bar:
2326         my $status_bar = status_bar -> new( steps => $lines );
2327
2328         ui -> print( category => 'scm',
2329                      message  => "Reading data file: ".$self -> filename );
2330         ui -> print( category => 'scm',
2331                      message => $status_bar -> print_step(),
2332                      newline => 0);
2333
2334         my ( $sth, $dbh, $first_row_id, $first_value_id );
2335         my $insert = 1;
2336 #       if ( $PsN::config -> {'_'} -> {'use_database'} and
2337 #            $self -> {'use_data_table'} ) {
2338 #         $dbh = DBI -> connect("DBI:mysql:host=".$PsN::config -> {'_'} -> {'database_server'}.
2339 #                              ";databse=".$PsN::config -> {'_'} -> {'project'},
2340 #                               $PsN::config -> {'_'} -> {'user'},
2341 #                               $PsN::config -> {'_'} -> {'password'},
2342 #                               {'RaiseError' => 1});
2343 #         my $sth = $dbh -> prepare( "SELECT data_row_id FROM ".$PsN::config -> {'_'} -> {'project'}.
2344 #                              ".data_row ".
2345 #                                    "WHERE data_id='".$self -> {'data_id'}."'" );
2346 #         $sth -> execute or debug -> die( message => $sth->errstr ) ;
2347 #         my $select_arr = $sth -> fetchall_arrayref;
2348 #         if ( scalar @{$select_arr} > 0 ) {
2349 #           for ( my $i = 0; $i < scalar @{$select_arr}; $i++ ) {
2350 #             push( @{$self -> {'data_row_ids'}}, $select_arr->[$i][0] );
2351 #           }
2352 #           $sth = $dbh -> prepare( "SELECT data_value_id FROM ".$PsN::config -> {'_'} -> {'project'}.
2353 #                              ".data_value ".
2354 #                                   "WHERE data_id='".$self -> {'data_id'}."'" );
2355 #           $sth -> execute or debug -> die( message => $sth->errstr ) ;
2356 #           my $select_val = $sth -> fetchall_arrayref;
2357 #           for ( my $i = 0; $i < scalar @{$select_val}; $i++ ) {
2358 #             push( @{$self -> {'data_value_ids'}}, $select_val->[$i][0] );
2359 #           }
2360 #           $insert = 0;
2361 #           $dbh -> disconnect;
2362 #         } else {
2363 #           $dbh -> do( "LOCK TABLES ".$PsN::config -> {'_'} -> {'project'}.
2364 #                              ".data_row WRITE, ".$PsN::config -> {'_'} -> {'project'}.
2365 #                              ".data_value WRITE" );
2366 #           $sth = $dbh -> prepare( "SELECT MAX(data_row_id) FROM ".$PsN::config -> {'_'} -> {'project'}.
2367 #                              ".data_row" );
2368 #           $sth -> execute or debug -> die( message => $sth->errstr ) ;
2369 #           my $select_arr = $sth -> fetchall_arrayref;
2370 #           $first_row_id = defined $select_arr -> [0][0] ? $select_arr -> [0][0] : 0;
2371 #           $sth = $dbh -> prepare( "SELECT MAX(data_value_id) FROM ".$PsN::config -> {'_'} -> {'project'}.
2372 #                              ".data_value" );
2373 #           $sth -> execute or debug -> die( message => $sth->errstr ) ;
2374 #           my $select_arr = $sth -> fetchall_arrayref;
2375 #           $first_value_id = defined $select_arr -> [0][0] ? $select_arr -> [0][0] : 0;
2376 #         }
2377 #         $sth -> finish;
2378 #       }
2379
2380         my $insert_rows;
2381         my $insert_values;
2382         my $row_counter = 0;
2383         my $full_row;
2384       ROW: while ( <DATAFILE> ) {
2385           s/^ *//;
2386           my @new_row   = split(/\,\s*|\s+/);
2387           # This regexp check is not time consuming.
2388           if ( /^\s*\d+|^\s*\./ ) {
2389             if ( defined $self -> {'cont_column'} ) {
2390               if ( $new_row[$self -> {'cont_column'} - 1] == 1 ) {
2391                 if ( not $self -> {'table_file'} ) { # Skip the CONT=1 rows if this is a table file
2392                   for ( my $i = $#new_row; $i > 0; $i-- ) {
2393                     if ( $i != ($self -> {'cont_column'} - 1) ) {
2394                       unshift( @{$full_row}, $new_row[$i] );
2395                     }
2396                   }
2397                 }
2398                 next ROW;
2399               } else {
2400                 for ( my $i = $#new_row; $i >= 0; $i-- ) {
2401 #                 if ( $i != ($self -> {'cont_column'} - 1) or $self -> {'table_file'} ) {
2402                   if ( $i != ($self -> {'cont_column'} - 1) ) {
2403                     unshift( @{$full_row}, $new_row[$i] );
2404                   }
2405                 }
2406               }
2407             } else {
2408               @{$full_row} = @new_row;
2409             }
2410             $new_ID = $full_row -> [$idcol-1]; # index starts at 0
2411             $old_ID = $new_ID if ( not defined $old_ID );
2412
2413             # Check if column miss data at some row (This adds about 30% of init time)
2414             my $mdt = $self -> {'missing_data_token'};
2415             for( my $i = 0; $i <= $#{$full_row}; $i++ ){
2416                 $self -> {'have_missing_data'} -> {$i+1} = 1
2417                   if( $full_row -> [$i] == $mdt ); # == is slower but safer than eq
2418             }
2419 #           if ( $PsN::config -> {'_'} -> {'use_database'} and
2420 #                $self -> {'use_data_table'} and $insert ) {
2421 #             $row_counter++;
2422 #             $insert_rows = $insert_rows."," if ( defined $insert_rows );
2423 #             $insert_rows = $insert_rows.
2424 #               "('$row_counter', '".$self -> {'data_id'}."' )";
2425 #             for ( my $j = 0; $j <= $#{$full_row}; $j++ ) {
2426 #               $insert_values = $insert_values."," if ( defined $insert_values );
2427 #               $insert_values = $insert_values.
2428 #                 "('".$full_row -> [$j]."', '".
2429 #                   ($first_row_id+$row_counter)."', '".
2430 #                     $self -> {'data_column_ids'}->[$j].
2431 #                       "', '".$self -> {'data_id'}."' )";
2432 #             }
2433 #           }
2434
2435             if ( $new_ID != $old_ID ) {
2436               my @subject_data = @init_data;
2437               my $id = data::individual -> new ( idcolumn     => $idcol,
2438                                                  subject_data => \@subject_data,
2439                                                  data_id      => $self -> {'data_id'} );
2440               push( @{$self -> {'individuals'}}, $id );
2441               @init_data =(join( ",", @{$full_row}));
2442             } else {
2443               push( @init_data, join( ",", @{$full_row}) );
2444             }
2445             $old_ID = $new_ID;
2446             $full_row = undef;
2447           }
2448           if ( $status_bar -> tick() ) {
2449             ui -> print( category => 'scm',
2450                          message  => $status_bar -> print_step(),
2451                          wrap     => 0,
2452                          newline  => 0 );
2453           }
2454         }
2455
2456 #       if ( $PsN::config -> {'_'} -> {'use_database'} and
2457 #            $self -> {'use_data_table'} and $insert ) {
2458 #         $dbh -> do("INSERT INTO ".$PsN::config -> {'_'} -> {'project'}.
2459 #                    ".data_row ".
2460 #                    "(number,data_id) ".
2461 #                    "VALUES ".$insert_rows);
2462 #         push( @{$self -> {'data_row_ids'}}, ($first_row_id..$first_row_id+$row_counter) );
2463 #         $dbh -> do( "INSERT INTO ".$PsN::config -> {'_'} -> {'project'}.
2464 #                     ".data_value ".
2465 #                     "(value,data_row_id,data_column_id,data_id) ".
2466 #                     "VALUES ".$insert_values );
2467 #         push( @{$self -> {'data_value_ids'}},
2468 #               ($first_value_id..$first_value_id+($row_counter*
2469 #                                                  scalar @{$self->{'data_column_ids'}})));
2470 #         $dbh -> do( "UNLOCK TABLES" );
2471 #         $dbh -> disconnect;
2472 #       }
2473
2474         if ( $#init_data >= 0 ) {
2475           push( @{$self -> {'individuals'}},
2476                 data::individual -> new ( idcolumn     => $idcol,
2477                                           subject_data => \@init_data ) );
2478         }
2479         ui -> print( category => 'scm',
2480                      message  => " ... done" );
2481         close(DATAFILE);
2482 #       $self -> _write( filename => 'test.dta' );
2483       }
2484 end _read_individuals
2485
2486 # }}} _read_individuals
2487