Mem optimizations on outputfile started
[PsN.git] / lib / output / problem_subs.pm
bloba45fdce98bceee6e8cc1d96486821524075fa521
1 # {{{ include
3 start include statements
4 # No brackets!
5 use Data::Dumper;
6 my $nrec_exp = '^\s*NO. OF DATA RECS IN DATA SET:\s*(\d*)|^\s*TOT. NO. OF DATA RECS:\s*(\d*)';
7 my $nobs_exp = ' TOT. NO. OF OBS RECS:\s*(\d*)';
8 my $nind_exp = ' TOT. NO. OF INDIVIDUALS:\s*(\d*)';
9 my $subprob_exp = '^ PROBLEM NO\.:\s*\d+\s*SUBPROBLEM NO\.:\s*(\d+)';
10 use Config;
11 end include
13 # }}} include statements
15 # {{{ new
17 start new
19 # Read Global data
20 $this -> _read_arbitrary( regexp => $nrec_exp,
21 member => 'nrecs' );
22 $this -> _read_arbitrary( regexp => $nobs_exp,
23 member => 'nobs' );
24 $this -> _read_arbitrary( 'regexp' => $nind_exp,
25 'member' => 'nind' );
26 $this -> _read_block_structures;
28 $this -> _read_inits;
29 $this -> _read_covstep;
31 # Read SubProblems
32 my $subproblem_start;
33 while ( $_ = @{$this -> {'lstfile'}}[ $this -> {'lstfile_pos'}++ ] ) {
34 if( /$subprob_exp/ or $this -> {'lstfile_pos'} > $#{$this -> {'lstfile'}} ){
35 if( defined $subproblem_start ){
36 my @subproblem_lstfile =
37 @{$this -> {'lstfile'}}[$subproblem_start .. $this -> {'lstfile_pos'} - 2];
38 my $subproblems;
39 if( $this -> {'lstfile_pos'} > $#{$this -> {'lstfile'}} ) {
40 $subproblems = $2;
41 } else {
42 $subproblems = $2 - 1; # Assuming problems come in order
44 $this -> add_subproblem
45 ( 'init_data' => {lstfile => \@subproblem_lstfile,
46 covstep => $this -> {'covstep'},
47 omega_block_structure_type => $this -> {'omega_block_structure_type'},
48 sigma_block_structure_type => $this -> {'sigma_block_structure_type'},
49 omega_block_structure => $this -> {'omega_block_structure'},
50 sigma_block_structure => $this -> {'sigma_block_structure'},
51 omega_block_sets => $this -> {'omega_block_sets'},
52 sigma_block_sets => $this -> {'sigma_block_sets'},
53 # lower_theta_bounds => $this -> {'lower_theta_bounds'},
54 # upper_theta_bounds => $this -> {'upper_theta_bounds'},
55 tablename => @{$this -> {'tablenames'}}[$subproblems],
56 tableidcolumn => @{$this -> {'tableidcolumns'}}[$subproblems],
57 model_id => $this -> {'model_id'},
58 problem_id => $this -> {'problem_id'},
59 output_id => $this -> {'output_id'} });
61 unless ( $this -> {'lstfile_pos'} > $#{$this -> {'lstfile'}} ){
62 $subproblem_start = $this -> {'lstfile_pos'};
67 unless( defined $subproblem_start ) { # No subproblems. Try to make one from the whole file.
68 $this -> add_subproblem
69 ( 'init_data' => {lstfile => $this -> {'lstfile'},
70 covstep => $this -> {'covstep'},
71 omega_block_structure_type => $this -> {'omega_block_structure_type'},
72 sigma_block_structure_type => $this -> {'sigma_block_structure_type'},
73 omega_block_structure => $this -> {'omega_block_structure'},
74 sigma_block_structure => $this -> {'sigma_block_structure'},
75 omega_block_sets => $this -> {'omega_block_sets'},
76 sigma_block_sets => $this -> {'sigma_block_sets'},
77 # lower_theta_bounds => $this -> {'lower_theta_bounds'},
78 # upper_theta_bounds => $this -> {'upper_theta_bounds'},
79 tablename => @{$this -> {'tablenames'}}[0],
80 tableidcolumn => @{$this -> {'tableidcolumns'}}[0],
81 model_id => $this -> {'model_id'},
82 problem_id => $this -> {'problem_id'},
83 output_id => $this -> {'output_id'} } );
85 if ( defined $this -> {'subproblems'} ) {
86 $this -> _set_labels;
88 delete $this -> {'lstfile'};
90 end new
92 # }}} new
94 # {{{ register_in_database
95 start register_in_database
96 if ( $PsN::config -> {'_'} -> {'use_database'} ) {
97 my ( $date_str, $time_str );
98 if ( $Config{osname} eq 'MSWin32' ) {
99 $date_str = `date /T`;
100 $time_str = ' '.`time /T`;
101 } else {
102 # Assuming UNIX
103 $date_str = `date`;
105 chomp($date_str);
106 chomp($time_str);
107 my $date_time = $date_str.$time_str;
108 my $dbh = DBI -> connect("DBI:mysql:host=".$PsN::config -> {'_'} -> {'database_server'}.
109 ";databse=".$PsN::config -> {'_'} -> {'project'},
110 $PsN::config -> {'_'} -> {'user'},
111 $PsN::config -> {'_'} -> {'password'},
112 {'RaiseError' => 1});
113 my $sth;
114 my @mod_str = ('','');
115 if ( defined $self -> {'model_id'} ) {
116 @mod_str = ('model_id, ',"$self->{'model_id'}, ");
118 $sth = $dbh -> prepare("INSERT INTO ".$PsN::config -> {'_'} -> {'project'}.
119 ".oproblem ".
120 "(output_id,".
121 $mod_str[0].
122 "nrecs,nobs,nind) ".
123 "VALUES ( '$output_id' ,".
124 $mod_str[1].
125 "'$self->{'nrecs'}' ".
126 ",'$self->{'nobs'}' ,'$self->{'nind'}' )");
127 $sth -> execute;
128 $self -> {'problem_id'} = $sth->{'mysql_insertid'};
129 $sth -> finish;
130 $dbh -> disconnect;
131 if ( defined $self -> {'problem_id'} ) {
132 foreach my $problem ( @{$self -> {'subproblems'}} ) {
133 $problem -> register_in_database( output_id => $output_id,
134 problem_id => $self -> {'problem_id'},
135 model_id => $model_id );
139 end register_in_database
140 # }}} register_in_database
142 # {{{ _read_arbitrary
144 start _read_arbitrary
146 my $start_pos = $self -> {'lstfile_pos'};
147 my $success = 0;
149 while ( $_ = @{$self -> {'lstfile'}}[$start_pos++] ) {
150 last if ( /^ PROBLEM.*SUBPROBLEM/ or /^ PROBLEM NO\.:\s+\d/ );
151 if ( /$regexp/ ) {
152 $self -> { $member } = $1;
153 $success = 1;
154 last;
157 if ( $success ) {
158 $self -> {'lstfile_pos'} = $start_pos;
159 } else {
160 debug -> warn( level => 1,
161 message => "rewinding to first position..." );
164 end _read_arbitrary
166 # }}} _read_arbitrary
168 # {{{ _read_block_structures
170 start _read_block_structures
172 my $start_pos = $self -> {'lstfile_pos'};
173 my $success = 0;
175 my $obarea = 0;
176 my $sbarea = 0;
178 my $oblock_set = -1;
179 my $sblock_set = -1;
180 while( $_ = @{$self -> {'lstfile'}}[ $start_pos++ ] ) {
181 last if ( /^ PROBLEM.*SUBPROBLEM/ or /^ PROBLEM NO\.:\s+\d/ );
183 if(/0OMEGA HAS BLOCK FORM:/) {
184 $self -> {'omega_block_structure_type'} = 'BLOCK';
185 $obarea = 1;
186 next;
188 if(/0SIGMA HAS BLOCK FORM:/) {
189 $self -> {'sigma_block_structure_type'} = 'BLOCK';
190 $sbarea = 1;
191 $obarea = 0;
192 next;
194 if ( /^0OMEGA HAS SIMPLE DIAGONAL FORM/ ) {
195 $self -> {'omega_block_structure_type'} = 'DIAGONAL';
197 if ( /^0SIGMA HAS SIMPLE DIAGONAL FORM/ ) {
198 $self -> {'sigma_block_structure_type'} = 'DIAGONAL';
199 $success = 1;
200 last;
202 if ( /^0INITIAL ESTIMATE/ ) {
203 $success = 1 if ( $sbarea or $obarea );
204 last;
206 if ( $obarea ) {
207 my @row = split;
208 # All rows with the last but one element set to 0 indicate the start of a new block
209 # $#row == 0 indicates the first row of the matrix.
210 if ( $#row == 0 or $row[$#row-1] == 0 ) {
211 # If the same number as previous set
212 if ( $oblock_set == $row[$#row] ) {
213 $self -> {'omega_block_sets'}{$oblock_set}{'size'}++;
214 } else {
215 $oblock_set = $row[$#row];
216 $self -> {'omega_block_sets'}{$oblock_set}{'size'} = 1;
218 # Always set dimension to 1 when starting a new block
219 $self -> {'omega_block_sets'}{$oblock_set}{'dimension'} = 1;
220 } else {
221 $self -> {'omega_block_sets'}{$oblock_set}{'dimension'}++;
223 push( @{$self -> {'omega_block_structure'}}, \@row );
225 if ( $sbarea ) {
226 my @row = split;
227 # All rows with the last but one element set to 0 indicate the start of a new block
228 if ( $#row == 0 or $row[$#row-1] == 0 ) {
229 # If the same number as previous set
230 if ( $sblock_set == $row[$#row] ) {
231 $self -> {'sigma_block_sets'}{$sblock_set}{'size'}++;
232 } else {
233 $sblock_set = $row[$#row];
234 $self -> {'sigma_block_sets'}{$sblock_set}{'size'} = 1;
236 # Always set dimension to 1 when starting a new block
237 $self -> {'sigma_block_sets'}{$sblock_set}{'dimension'} = 1;
238 } else {
239 $self -> {'sigma_block_sets'}{$sblock_set}{'dimension'}++;
241 push( @{$self -> {'sigma_block_structure'}}, \@row );
245 unless ( $success ) {
246 debug -> warn( level => 2,
247 message => "rewinding to first position..." );
248 } else {
249 $self -> {'lstfile_pos'} = $start_pos;
252 end _read_block_structures
254 # }}} _read_block_structures
256 # {{{ _read_covstep
257 start _read_covstep
259 my $start_pos = $self -> {'lstfile_pos'};
260 my $success = 0;
262 while( $_ = @{$self -> {'lstfile'}}[ $start_pos++ ] ) {
263 last if ( /^ PROBLEM.*SUBPROBLEM/ or /^ PROBLEM NO\.:\s+\d/ );
265 if(/0COVARIANCE STEP OMITTED:\s*\b(.*)\b/) {
266 $self -> {'covstep'} = 0 if $1 eq 'YES';
267 $self -> {'covstep'} = 1 if $1 eq 'NO';
268 $success = 1;
269 last;
271 if ( /^0MINIMIZATION/ ) {
272 last;
276 unless ( $success ) {
277 debug -> warn( level => 2,
278 message => "rewinding to first position..." );
279 } else {
280 $self -> {'lstfile_pos'} = $start_pos;
283 end _read_covstep
284 # }}} _read_covstep
286 # {{{ _read_inits
288 start _read_inits
290 my $start_pos = $self -> {'lstfile_pos'};
291 my ( @thetas, @omegas, @sigmas, $fixed );
292 my $thetarea = 0;
293 my $omegarea = 0;
294 my $sigmarea = 0;
295 my $success = 0;
296 while( $_ = @{$self -> {'lstfile'}}[ $start_pos++ ] ) {
297 last if ( /^ PROBLEM.*SUBPROBLEM/ or /^ PROBLEM NO\.:\s+\d/ );
298 if ( /^0INITIAL ESTIMATE OF THETA:/ ) {
299 $success = 1;
300 $thetarea = 1;
302 if ( /^0INITIAL ESTIMATE OF OMEGA:/ ) {
304 if ( /^0INITIAL ESTIMATE OF SIGMA:/ ) {
305 $sigmarea = 1;
306 $omegarea = 0;
307 $thetarea = 0;
308 $fixed = 0;
310 last if ( /^0ESTIMATION STEP OMITTED:/ );
311 if ( $thetarea and /^\s*-?\d*\.\d*/ ) {
312 my @T = split(' ',$_);
313 push(@{$self -> {'initthetas'}},eval($T[1]));
314 push(@{$self -> {'lower_theta_bounds'}},eval($T[0]));
315 push(@{$self -> {'upper_theta_bounds'}},eval($T[2]));
316 if ( $T[0] == $T[1] and $T[0] == $T[2] ) {
317 push(@{$self -> {'fixedthetas'}},1);
318 push(@{$self -> {'estimatedthetas'}},0);
319 } else {
320 push(@{$self -> {'fixedthetas'}},0);
321 push(@{$self -> {'estimatedthetas'}},1);
324 if ( /^0INITIAL ESTIMATE OF OMEGA:/ ) {
325 $thetarea = 0;
326 if ( defined $self -> {'omega_block_sets'} and
327 scalar keys %{$self -> {'omega_block_sets'}} > 0 ) {
328 $start_pos++;
329 my %om_bl = %{$self -> {'omega_block_sets'}};
330 my @blocks = sort {$a <=> $b} keys %om_bl;
331 foreach my $block ( @blocks ) {
332 my @fix_row = split(' ', $self -> {'lstfile'}[ $start_pos++ ]);
333 my $fix = $fix_row[1] eq 'YES' ? 1 : 0;
334 for ( my $size = 1; $size <= $om_bl{$block}{'size'}; $size++ ) {
335 for ( my $row = $start_pos ; $row < $start_pos + $om_bl{$block}{'dimension'}; $row++ ) {
336 my @init_row = split(' ', $self -> {'lstfile'}[ $row ]);
337 foreach my $init ( @init_row ) {
338 push( @{$self -> {'fixedomegas'}}, $fix );
339 push( @{$self -> {'estimatedomegas'}}, (not $fix and $size == 1) ? 1 : 0 );
340 push(@{$self -> {'initomegas'}}, eval($init) );
341 push(@{$self -> {'lower_omega_bounds'}},0);
342 push(@{$self -> {'upper_omega_bounds'}},1000000);
346 $start_pos += $om_bl{$block}{'dimension'};
348 } else {
349 while( not $self -> {'lstfile'}[ $start_pos ] =~ /^0/ ) {
350 my @init_row = split(' ', $self -> {'lstfile'}[ $start_pos++ ]);
351 my $init = eval($init_row[$#init_row]);
352 push( @{$self -> {'fixedomegas'}}, 0 ) unless ( $init == 0 );
353 push( @{$self -> {'estimatedomegas'}}, 1 );
354 push(@{$self -> {'initomegas'}}, eval($init) ) unless ( $init == 0 );
355 push(@{$self -> {'lower_omega_bounds'}},0);
356 push(@{$self -> {'upper_omega_bounds'}},1000000);
360 if ( ($omegarea or $sigmarea) and /^\s+\d+\s+\w+/ ) {
361 my @T = split(' ',$_);
362 $fixed = $T[1] eq 'YES' ? 1 : 0;
364 # if ( $omegarea and /^\s*-?\d*\.\d*/ ) {
365 # my @T = split(' ',$_);
366 # for my $i (0..(@T-1)) {
367 # $T[$i] = eval($T[$i]);
368 # push(@{$self -> {'initomegas'}},$T[$i]);# if ( $T[$i] != 0 );
369 # push(@{$self -> {'fixedomegas'}},$fixed);# if ( $T[$i] != 0 );
370 # push(@{$self -> {'lower_omega_bounds'}},0);
371 # push(@{$self -> {'upper_omega_bounds'}},1000000);
374 if ( $sigmarea and /^\s*-?\d*\.\d*/ ) {
375 my @T = split(' ',$_);
376 for my $i (0..(@T-1)) {
377 $T[$i] = eval($T[$i]);
378 push(@{$self -> {'initsigmas'}},$T[$i]);# if ( $T[$i] != 0 );
379 push(@{$self -> {'fixedsigmas'}},$fixed);# if ( $T[$i] != 0 );
380 push(@{$self -> {'estimatedsigmas'}},not $fixed);# if ( $T[$i] != 0 );
381 push(@{$self -> {'lower_sigma_bounds'}},0);
382 push(@{$self -> {'upper_sigma_bounds'}},1000000);
385 if ( /^0MINIMIZATION/ ) {
386 last;
390 unless ( $success ) {
391 debug -> warn( level => 2,
392 message => "rewinding to first position..." );
393 } else {
394 # Keep this code
395 # if ( $PsN::config -> {'_'} -> {'use_database'} and
396 # $self -> {'register_in_database'} ) {
397 # my $dbh = DBI -> connect("DBI:mysql:host=".$PsN::config -> {'_'} -> {'database_server'}.
398 # ";databse=".$PsN::config -> {'_'} -> {'project'},
399 # $PsN::config -> {'_'} -> {'user'},
400 # $PsN::config -> {'_'} -> {'password'},
401 # {'RaiseError' => 1});
402 # my $sth;
403 # my @mod_str = ('','');
404 # if ( defined $self -> {'model_id'} ) {
405 # @mod_str = ('model_id,',"$self->{'model_id'},");
407 # foreach my $param ( 'theta', 'omega', 'sigma' ) {
408 # foreach my $par_str ( @{$self -> {'init'.$param.'s'}} ) {
409 # $sth = $dbh -> prepare("INSERT INTO ".$PsN::config -> {'_'} -> {'project'}.
410 # ".estimate ".
411 # "(subproblem_id,problem_id,output_id,".
412 # $mod_str[0].
413 # "type,value,init) ".
414 # "VALUES ( 1 ,".
415 # "'$self->{'problem_id'}' ,".
416 # "'$self->{'output_id'}' ,".
417 # $mod_str[1].
418 # "'$param','$par_str','1')");
419 # $sth -> execute;
420 # push( @{$self -> {'estimate_ids'}}, $sth->{'mysql_insertid'} );
423 # $sth -> finish;
425 # $dbh -> disconnect;
428 $self -> {'lstfile_pos'} = $start_pos;
431 end _read_inits
433 # }}} _read_inits
435 # {{{ access_any
437 start access_any
439 unless( $#subproblems > 0 ){
440 debug -> warn( level => 2,
441 message => "subproblems undefined, using all." );
442 if( defined $self -> {'subproblems'} ) {
443 @subproblems = (1 .. scalar @{$self -> {'subproblems'}});
444 } else {
445 debug -> warn( level => 1,
446 message => "No subproblems defined in this problem." );
447 @subproblems = ();
452 my @own_subproblems = defined $self -> {'subproblems'} ? @{$self -> {'subproblems'}} : ();
453 foreach my $i ( @subproblems ) {
454 if ( defined $own_subproblems[$i-1] ) {
455 debug -> warn( level => 2,
456 message => "subproblems: $i" );
457 debug -> warn( level => 2,
458 message => "Attribute: ".$own_subproblems[$i-1] -> $attribute );
459 my $meth_ret = $own_subproblems[$i-1] -> $attribute;
460 #if ( ref ($own_subproblems[$i-1] -> $attribute) ) {
461 # Test if the returned value is an array (with hashes we can't allow selection based on parameter numbers, since a hash is not ordered)
462 if ( ref ( $meth_ret ) eq 'ARRAY' ) {
463 #my @subprob_attr = @{$own_subproblems[$i-1] -> $attribute};
464 my @subprob_attr = @{$meth_ret};
465 if ( scalar @parameter_numbers > 0 ) {
466 my @tmp_arr = ();
467 foreach my $num ( @parameter_numbers ) {
468 if ( $num > 0 and $num <= scalar @subprob_attr ) {
469 push( @tmp_arr, $subprob_attr[$num-1] );
470 } else {
471 debug -> die( message => "( $attribute ): no such parameter number $num!".
472 "(".scalar @subprob_attr." exists)" );
475 @subprob_attr = @tmp_arr;
477 push( @return_value, \@subprob_attr );
478 } else {
479 # Only push if defined
480 push( @return_value, $meth_ret ) if defined $meth_ret;
482 } else {
483 debug -> die( message => "No such subproblem ".($i-1) );
486 # Check the return_value to see if we have empty arrays
487 if ( $#return_value == 0 and ref ($return_value[0]) eq 'ARRAY' and scalar @{$return_value[0]} < 1 ) {
488 @return_value = ();
491 end access_any
493 # }}} access_any
495 # {{{ _set_labels
496 start _set_labels
498 if ( defined $self -> {'subproblems' } ) {
499 foreach my $type ( ('theta','omega','sigma') ) {
500 my $first_sub = @{$self -> {'subproblems'}}[0];
501 my $accessor = $type eq 'theta' ? $type.'s' : 'raw_'.$type.'s';
502 my @param = @{$first_sub -> $accessor};
503 if ( scalar @param > 0 ) {
504 my ( @names, @indexes );
505 my ($j,$ndiags);
506 foreach $j (1..scalar @param) {
507 if ( $type eq 'theta' ) {
508 push( @names, "TH$j" );
509 } else {
510 if ( $first_sub -> _isdiagonal('index' => $j) ) {
511 push @names, uc(substr($type,0,2)).++$ndiags;
512 if ( $type eq 'omega' ) {
513 push ( @{$self -> {'omega_indexes'}}, [$ndiags, $ndiags] );
514 } else {
515 push ( @{$self -> {'sigma_indexes'}}, [$ndiags, $ndiags] );
517 next;
518 } elsif ($param[$j-1] !=0) {
519 @indexes = $first_sub -> _rowcolind( index => $j);
520 push @names,uc(substr($type,0,2)).$indexes[0].'_'.$indexes[1];
521 if ( $type eq 'omega' ) {
522 push ( @{$self -> {'omega_indexes'}}, [$indexes[0], $indexes[1]] );
523 } else {
524 push ( @{$self -> {'sigma_indexes'}}, [$indexes[0], $indexes[1]] );
529 $self ->{$type.'names'} = \@names;
534 end _set_labels
535 # }}} _set_labels