From 308b3406fd5fa3a23e5e67f4546d9e40f53e9eb5 Mon Sep 17 00:00:00 2001 From: pontus_pih Date: Tue, 27 May 2008 09:47:46 +0000 Subject: [PATCH] final touches for 2.2.5 --- README.txt | 56 ++++++++++++++++++++----- bin/sse | 2 +- lib/tool/modelfit_subs.pm | 101 ++++++++++++++++++++++++++++++++++++++++++---- lib/tool/sse_subs.pm | 14 ++++--- 4 files changed, 147 insertions(+), 26 deletions(-) diff --git a/README.txt b/README.txt index 2fc4063..99bd43b 100644 --- a/README.txt +++ b/README.txt @@ -14,23 +14,60 @@ Changes from 2.2.4 to 2.2.5 Features - - mcs script renamed to sse - - sse calculates lots of nice results. + - Monte Carslo Simulations(mcs) tool has been renamed to Stochastic + Simulation and Estimation(sse), and has a bunch of new + features. More information can be found in the SSE user guide + distributed with PsN. - - msfi/msfo. + - Numerical Predictive Check(npc) and Visual Predictive Check(vpc) + has been added to the toolbox. + + - Much improved results handling. Intermediate results are printed + each time a NONMEM run has finished. A positive side effect is much + reduced memory usage over all and in particular when running with + many threads + + - Sumo tool is more verbose when parsing errors occur. + + - General improvement in handling of parsing errors. + + - Functionality added to parsing of data that enables memory + optimization in npc/vpc. + + - max_runtime option added for unix type systems. This feature will + be expanded to more systems in the future. + + - By default PsN will restart crashed runs. This feature can now be + disabled with "--no-handle_crashes" + + - By default execute will copy the tables and lst file to the + directory where the original model file is placed. By specifying + "--extra_output=file1,file2" PsN will also copy any file listed. To + avoid files being overwritten the option + "--prepend_model_file_name" can be used to add the name of the + model file (without extension) to the front of the output files. + + - Added support for G95. A more generic way of configuring compilers + is on the way. + + - Experimental msfi/msfo functionality. If the model allready has the MSFO= option or the $MSFI record set and one of the files exists, then that file will be used as $MSFI file in the first run. If both files exists, then the MSFO file - will be used as MSFI file. The file will be renamed psn_msfo-0. + will be used as MSFI file. The file will be renamed psn_msfo-0. - The MSFO option will be set to psn_msfo. + The MSFO option will be set to psn_msfo. If the NONMEM run crashes + PsN will use the new msfo files as $MSFI file. This functionality + is considered experimental and must be enabled with "--handle_msfo". + Notice that any msfo files existing when doing restart will be + ignored. Bugs - - Multiline ETABAR and PVAL parsed correctly. + - Multiline ETABAR and PVAL in output parsed correctly. - Modelfit did not copy extra subroutine files in some cases. @@ -45,12 +82,9 @@ Bugs - cdd xv can no be disabled with -no-xv or -noxv - - mc tool no uses considerably less memory. It is a lot slower - because it reads a lot of unecessary stuff from disk, it should be - reviewed. - - Memory leak in modelfit due to intermediate "raw_results.csv" in - NM_runX directory fixed by removing the feature, should be removed. + NM_runX directory fixed by reimplementing the way results handling + is done. - A initial estimate of form: $OMEGA BLOCK SAME would not be recognized by PsN model file parser. This is now handled diff --git a/bin/sse b/bin/sse index aad1b8c..dcdd437 100644 --- a/bin/sse +++ b/bin/sse @@ -90,7 +90,7 @@ $help_text{-estimate_simulation} = <<'EOF'; By default, the simulation model is also used for estimation with the simulated datasets. The resulting OFV values are used as reference when evaluating the estimation results of alternative models. By setting - -no_estimate_simulation the estimation of the simulation model is turned + -no-estimate_simulation the estimation of the simulation model is turned off, and the first alternative model is used as reference instead. See also -ref_ofv. EOF diff --git a/lib/tool/modelfit_subs.pm b/lib/tool/modelfit_subs.pm index ca142ee..7fa8c13 100644 --- a/lib/tool/modelfit_subs.pm +++ b/lib/tool/modelfit_subs.pm @@ -340,6 +340,7 @@ start new if( $this -> {'run_on_lsf'} or $this -> {'run_on_ud'} or $this -> {'run_on_umbrella'} or + $this -> {'run_on_zink'} or $this -> {'run_on_sge'} ){ $this -> {'run_local'} = 0; } @@ -779,8 +780,6 @@ start copy_model_and_input my $msfi_names = $candidate_model -> msfi_names; my $msfi_in; - print "Model already has msfi: ". Dumper $msfi_names . "and msfo: " . Dumper $msfo_names; - if( defined $msfo_names ){ $msfi_in = $msfo_names -> [0][0]; } elsif ( defined $msfi_names ){ @@ -991,8 +990,6 @@ start set_msfo_to_msfi if( $candidate_model -> outputs -> [0] -> msfo_has_terminated() ){ - print "Msfo terminated\n"; - $msfi = $msfo . '-' . ($queue_info -> {'crashes'}-1); $candidate_model->remove_records( type => 'estimation' ); @@ -1003,7 +1000,6 @@ start set_msfo_to_msfi } - print "$msfo -> $msfi\n"; if( -e $msfo ){ mv( $msfo, $msfi ); } @@ -1350,6 +1346,77 @@ end lsf_monitor # }}} +# {{{ zink_submit +start zink_submit +{ + require File::Temp;# qw/tempfile tempdir/; + require Sys::Hostname; + require LockFile::Simple;# qw/lock unlock trylock/; # Non-standard module + + ################################################################################################### + ###### JobSpecification code and variables ############## + ################################################################################################### + + ## Specifies the top level directory of the Zink directory structure. Should be specified via psn.conf + my $ZinkDir = $PsN::config -> {'_'} -> {'zink_dir'}; + + ## Specify the queing directory/Job Specification drop zone. I suggest this is hardcoded and not specifiable in psn.conf. + my $ZinkJobDir =$ZinkDir."/ZinkJobs"; + + ## $JobName: Name of job. Default could be model file name. + ## $JobPriority: Priority of job. Between 0-5 (0=low). Default should be 3. + ## $ExePath: Directory in which the run is to be executed. + ## $Command: String with command to be executed, e.g. "nmfe6 run1.mod run1.lst" + + my $host = hostname(); + my $fsubs = join( ',' , @{$model -> subroutine_files} ); + my $command = ($PsN::config -> {'_'} -> {'remote_perl'} ? ' ' . $PsN::config -> {'_'} -> {'remote_perl'} : ' perl ') . " -I" . + $PsN::lib_dir ."/../ " . + $PsN::lib_dir . "/nonmem.pm" . + " psn.mod psn.lst " . + $self -> {'nice'} . " ". + $nm_version . " " . + 1 . " " . # compilation + 1 . " " . # execution + $fsubs . " " . + $self -> {'nm_directory'}; + + my $path = getcwd(); + my $jobname = $queue_info -> {'model'} -> filename; + my $prio = $self -> {'zink_prio'}; + my ($FH,$jobId) = tempfile("$host-XXXXXXXXXXXXXXXXX",DIR => $ZinkJobDir,SUFFIX=>'.znk'); + lock $jobId; + print $FH "SUBMITHOST: $host\n"; + print $FH "JOBNAME: $jobname\n"; + print $FH "PRIORITY: 3\n"; + print $FH "EXEPATH: $path\n"; + print $FH "COMMAND: $command\n"; + close $FH; + unlock $jobId; + +} +end zink_submit +# }}} zink_submit + +# {{{ zink_monitor +start zink_monitor +{ + ## Specifies the top level directory of the Zink directory structure. Should be specified via psn.conf + my $ZinkDir = $PsN::config -> {'_'} -> {'zink_dir'}; + + ## Specify the queing directory/Job Specification drop zone. I suggest this is hardcoded and not specifiable in psn.conf. + my $ZinkDoneDir =$ZinkDir."/ZinkDone"; + + if( -e "$ZinkDoneDir$jobId" ){ + return $jobId; + } else { + return 0; + } + +} +end zink_monitor +# }}} zink_monitor + # {{{ ud_submit start ud_submit @@ -1603,7 +1670,8 @@ end ud_retrieve start sge_submit { my $fsubs = join( ',' , @{$model -> subroutine_files} ); - if( system( 'qsub -cwd -b y ' . + if( system( 'qsub -cwd -b y -N ' . + $queue_info -> {'model'} -> filename . ($self -> {'sge_resource'} ? '-l '.$self -> {'sge_resource'}.' ' : ' ') . ($self -> {'sge_queue'} ? '-q '.$self -> {'sge_queue'}.' ' : ' ') . ($PsN::config -> {'_'} -> {'remote_perl'} ? ' ' . $PsN::config -> {'_'} -> {'remote_perl'} : ' perl ') . " -I" . @@ -1703,7 +1771,7 @@ start run_nonmem # }}} } else { #Asume *nix - + # {{{ Unix execution my $perl_bin = ($PsN::config -> {'_'} -> {'perl'} ? $PsN::config -> {'_'} -> {'perl'} : ' perl '); @@ -1752,10 +1820,17 @@ start run_nonmem } elsif ( $self -> run_on_sge() ) { my $jobId = $self -> sge_submit( model => $candidate_model, + queue_info => $queue_info, nm_version => $nm_version ); #$queue_info->{'compile_only'} = 0; $queue_map->{$jobId} = $run_no; + } elsif ( $self -> run_on_zink() ) { + my $jobId = $self -> zink_submit( model => $candidate_model, + queue_info => $queue_info, + nm_version => $nm_version ); + + $queue_map->{$jobId} = $run_no; } # }}} @@ -1904,6 +1979,13 @@ start restart_needed if( not $output_file -> parsed_successfully() ){ if( $self -> {'handle_crashes'} and $queue_info_ref -> {'crashes'} < $self -> crash_restarts() ) { + + if( $self -> {'max_runtime'} > 0 + and + (time() > $queue_info_ref -> {'start_time'} + $self -> {'max_runtime'}) ){ + $output_file -> flush; + return 0; + } # If the output file could not be parsed successfully, this is # a sign of a crashed run. This is not a NONMEM error as such @@ -2093,7 +2175,6 @@ start restart_needed # This code must be adjusted for multiple problems!! my $degree = 0.1*${$tries}; if( $self -> {'handle_msfo'} ) { - print "Ost på räka\n"; $self -> reset_msfo( basic_model => $model, candidate_model => $candidate_model ); @@ -2605,6 +2686,10 @@ start run $pid = $self -> sge_monitor( jobId => $check_pid ); + } elsif( $self -> {'run_on_zink'} ) { + + $pid = $self -> zink_monitor( jobId => $check_pid ); + } elsif( $self -> {'run_on_lsf'} ) { $pid = $self -> lsf_monitor( jobId => $check_pid ); diff --git a/lib/tool/sse_subs.pm b/lib/tool/sse_subs.pm index c1406f3..f7c6d5a 100644 --- a/lib/tool/sse_subs.pm +++ b/lib/tool/sse_subs.pm @@ -45,12 +45,12 @@ foreach my $alt (@{$this -> {'alternative_models'}}){ } } -if ((scalar(@{$this -> {'alternative_models'}}) < 1) && (not $this->{'estimate_original'})){ +if ((scalar(@{$this -> {'alternative_models'}}) < 1) && (not $this->{'estimate_simulation'})){ 'debug' -> die(message => "Must estimate at least one model, ". "either an alternative or the simulation model."); } -if ($this->{'estimate_original'} && (defined $this->{'ref_ofv'})){ +if ($this->{'estimate_simulation'} && (defined $this->{'ref_ofv'})){ 'debug'-> die(message => "Not allowed to use a reference ofv-value ". "when estimating the simulation model."); } @@ -771,7 +771,7 @@ start prepare_results my $end_index = ( $self -> {'estimate_simulation'})? scalar(@alternatives): scalar(@alternatives) - 1; - MODEL: for my $model_index ( 0..$end_index ){ + for my $model_index ( 0..$end_index ){ ### Foreach such model, create a %results_section @@ -809,7 +809,7 @@ start prepare_results push( @{$self -> {'results'}[0]{'own'}},\%results_section ); - foreach my $measure ( 'ofv','theta','omega','sigma' ){ + MEASURE: foreach my $measure ( 'ofv','theta','omega','sigma' ){ # Create a header, we might use max_hash from modelfit for this # later on. @@ -832,10 +832,12 @@ start prepare_results } if( $start == 0 ){ for( 0..(10+scalar(@z_list)) ){ - push(@{$values[$_]},'Model not terminated for all samples'); + # At least two reasons for this. Either the "measure" does + # not exist or the model has not terminated. + push(@{$values[$_]},'NA'); } push( @{$labels[1]},$measure ); - next MODEL; + next MEASURE; } } -- 2.11.4.GIT