From 6a2677758fcec0689fd34ac5363de82e67c146d3 Mon Sep 17 00:00:00 2001
From: pontus_pih <pontus_pih>
Date: Wed, 1 Mar 2006 16:11:08 +0000
Subject: [PATCH] Lots of fixes for 2.2, see README.txt for details

---
 lib/common_options.pm      | 348 ++++++++++++++++++++++----
 lib/nonmem_subs.pm         |  13 +-
 lib/output_subs.pm         |  15 +-
 lib/tool/bootstrap_subs.pm |  12 +-
 lib/tool/cdd_subs.pm       |   3 +
 lib/tool/llp_subs.pm       |   8 +-
 lib/tool/mc_subs.pm        |   1 -
 lib/tool/modelfit_subs.pm  | 589 ++++++++++++++++++++++++---------------------
 lib/tool/scm_subs.pm       |   3 +
 lib/tool_subs.pm           |  16 ++
 10 files changed, 672 insertions(+), 336 deletions(-)

diff --git a/lib/common_options.pm b/lib/common_options.pm
index 021ca04..7a9b132 100644
--- a/lib/common_options.pm
+++ b/lib/common_options.pm
@@ -3,22 +3,23 @@ package common_options;
 use FindBin qw($Bin);
 use lib "$Bin/../lib";
 use Getopt::Long;
+use Text::Wrap;
 
 ## Configure the command line parsing
 Getopt::Long::config("auto_abbrev");
 
 my @tool_options = ( "abort_on_fail",
-		     "adaptive",
+		     "adaptive!",
 		     "clean:i",
 		     "compress",
 		     "condition_number_limit:f",
 		     "correlation_limit:f",
-		     "cpu_time:i",
+#		     "cpu_time:i",
 		     "directory:s",
 		     "drop_dropped",
-		     "eigen_values",
-		     "grid_batch_size:i",
-		     "grid_poll_interval:i",
+#		     "eigen_values",
+#		     "grid_batch_size:i",
+#		     "grid_poll_interval:i",
 		     "handle_maxevals",
 		     "large_theta_cv_limit:f",
 		     "large_omega_cv_limit:f",
@@ -42,11 +43,11 @@ my @tool_options = ( "abort_on_fail",
 		     "nonparametric_marginals",
 		     "picky",
 		     "quick_summarize|quick_summary",
-		     "remove_temp_files",
+#		     "remove_temp_files",
 		     "rerun:i",
-		     "results_file:s",
+#		     "results_file:s",
 		     "retries:i",
-		     "run_on_nordugrid",
+#		     "run_on_nordugrid",
 		     "run_on_lsf",
 		     "seed:s",
 		     "shrinkage",
@@ -57,8 +58,6 @@ my @tool_options = ( "abort_on_fail",
 		     "tweak_inits:i",
 		     "verbose!",
 		     "wrap_data",
-		     "correlation_limit:f",
-		     "condition_number_limit:i",
 		     "near_bound_sign_digits:i",
 		     "near_zero_boundary_limit:f",
 		     "sign_digits_off_diagonals:i",
@@ -66,7 +65,7 @@ my @tool_options = ( "abort_on_fail",
 		     "large_omega_cv_limit:f",
 		     "large_sigma_cv_limit:f",
 		     "confidence_level:f",
-		     "precision:i"
+		     "precision:i",
 		     );
 
 
@@ -84,12 +83,12 @@ my @script_options = ( "debug:i",
 		       "html_help",
 		       "project:s",
 		       "silent",
-		       "submit_self",
+#		       "submit_self",
 		       "user:s",
 		       "warn_with_trace:i"
 		       );
 
-@get_opt_strings = (@tool_options, @model_options, @script_options);
+@get_opt_strings = (sort(@tool_options), sort(@model_options), sort(@script_options));
 
 sub options_to_parameters {
   my $opts = shift;
@@ -101,13 +100,11 @@ sub options_to_parameters {
     $opt =~ s/[!:|].*//g;
     $parameter_string .= "$opt => \$options{'$opt'},\n";
   }
-  
   $parameter_string .= ' )';
-
   return $parameter_string;
 }
 
-$parameters = options_to_parameters(\@tool_options);
+$parameters = options_to_parameters([@tool_options,'top_tool']);
 
 
 @extra_files;
@@ -144,22 +141,21 @@ sub set_globals {
 sub get_defaults {
   my $options = shift;
   my $tool    = shift;
-
-  foreach my $default_option ( keys %{$PsN::config -> {'default_options'}} ){
-
+  foreach my $default_option ( keys %{$PsN::config -> {'default_'.$tool.'_options'}} ){
     unless( exists $options -> {$default_option} ){
-      $options -> {$default_option} = $PsN::config -> {'default_options'} -> {$default_option};
+      $options -> {$default_option} = $PsN::config -> {'default_'.$tool.'_options'} -> {$default_option};
     }
-
+    
   }
 
-  foreach my $default_option ( keys %{$PsN::config -> {'default_'.$tool.'_options'}} ){
+
+  foreach my $default_option ( keys %{$PsN::config -> {'default_options'}} ){
     unless( exists $options -> {$default_option} ){
-      $options -> {$default_option} = $PsN::config -> {'default_'.$tool.'_options'} -> {$default_option};
+      $options -> {$default_option} = $PsN::config -> {'default_options'} -> {$default_option};
     }
-
+    
   }
-
+  $options -> {'top_tool'} = 1;
 }
 
 sub print_help {
@@ -174,7 +170,15 @@ sub print_help {
   my $option_help;
 
   $option_help .= "[ -h | -? ] [ --help ]\n\t" . ' ' x (1+length($command));
-  foreach my $help( sort(keys %{$required}), sort(keys %{$optional}), sort(@get_opt_strings) ){
+  
+  my @loop_array;
+  if( $command eq 'execute' ){
+    @loop_array = sort(@get_opt_strings);
+  } else {
+    @loop_array = (sort(keys %{$required}), sort(keys %{$optional}));
+  }
+
+  foreach my $help( @loop_array ) {
     next if( $help eq 'help' or $help eq 'h|?' );
     unless( $is_required{$help} ){
       $option_help .= "[ ";
@@ -303,13 +307,14 @@ EOF
     $help_hash{'-?'} = <<'EOF';
     <p class="style2">-h | -?</p>
 
-    With <span class="style2">-h</span> or <span class="style2">-?</span> execute.pl prints the list of options and exit.
+    With <span class="style2">-h</span> or <span class="style2">-?</span> execute.pl prints the list of available options 
+    and exit.
 EOF
 
     $help_hash{-help} = <<'EOF';
     <p class="style2">-help</p>
 
-    With <span class="style2">-help</span> execute will print this, longer, help message.
+    With <span class="style2">-help</span> execute will print a longer help message.
 EOF
 
     $help_hash{-nm_version} = <<'EOF';
@@ -360,6 +365,47 @@ EOF
     the first time.
 EOF
 
+    $help_hash{-drop_dropped} = <<'EOF';
+    <p class="style2">-drop_dropped</p>
+
+    If there are drop columns in your control file and <span class="style2">-drop_dropped</span> 
+    is used, PsN will remove those columns from the data set used
+    internally. It saves both diskspace and conserves memory
+    usage. Note that PsN does NOT alter your original data set, only
+    those used internally in PsN.
+EOF
+
+    $help_hash{-extra_data_files} = <<'EOF';
+    <p class="style2">-extra_data_files='extra_data1.dta, COLUMN1, COLUMN2'</p>
+
+    NONMEM only allows 20 column datasets, but PsN can add code to
+    control files that reads extra data columns from a separate
+    file. To use this feature you must create a new data file which
+    has the same ID row as the main data file. Then you specify a
+    comma separated list with <span class="style2">-extra_data_files</span>. The first element 
+    in the list is the filename and the rest of the list is the header
+    of the extra data file. You can have multiple extra files if neccesary.
+EOF
+
+    $help_hash{-extra_files} = <<'EOF';
+    <p class="style2">-extra_files='extra_file1.dta, extra_file2.dta'</p>
+
+    If you need extra files in the directory where NONMEM is run you
+    specify them in list to the <span class="style2">-extra_files</span> list. It could for 
+    example be fortran subroutines you need compiled with NONMEM.
+EOF
+
+    $help_hash{-handle_maxevals} = <<'EOF';
+    <p class="style2">-handle_maxevals='number'</p>
+
+    NONMEM only allows 9999 function evaluations. PsN can expand this
+    limit by adding an MSFO option to $ESTIMATION, later when NONMEM
+    hits the max number of function evaluations(9999) PsN will remove
+    intial estimates from the modelfile and add $MSFI and restart
+    NONMEM. PsN will do this until the number of evaluations specified
+    with <span class="style2">-handle_maxevals</span> is reached.
+EOF
+
     $help_hash{-seed} = <<'EOF';
     <p class="style2">-seed='string'</p>
 
@@ -368,6 +414,87 @@ EOF
     parameters. To make sure that the same result is produced if you
     redo the same run, you can set your own random seed with the <span class="style2">-seed</span>
     option.
+
+EOF
+
+    $help_hash{'-summarize|summary'} = <<'EOF';
+    <p><span class="style2">-summarize</span> or <span class="style2">-summary</span></p>
+
+    <span class="style2">summarize</span> or <span class="style2">-summary</span> will do a set of diagnostics test 
+    and print minimization message for each model run.
+EOF
+
+    $help_hash{-verbose} = <<'EOF';
+    <p class="style2">-verbose</p>
+
+    With <span class="style2">verbose</span> set to 1, PsN will print
+    more details about NONMEM runs. More precisely PsN will print the
+    minimization message for each successfull run and a R:X for each
+    retry PsN makes of a failed run, where X is the run number.
+EOF
+
+    $help_hash{-wrap_data} = <<'EOF';
+    <p class="style2">-wrap_data</p>
+
+    NONMEM only allows 20 column datasets, but it is possible to wrap
+    observation lines into multiple rows by adding a CONT column. With
+    <span class="style2">wrap_data</span> PsN does it automatically.
+EOF
+
+    $help_hash{-lsf_job_name} = <<'EOF';
+    <p class="style2">-lsf_job_name='string'</p>
+
+    <span class="style2">lsf_job_name</span> sets the name of the LSF job name of every NONMEM run, 
+    they all get the same name.
+EOF
+
+    $help_hash{-lsf_options} = <<'EOF';
+    <p class="style2">-lsf_options='string'</p>
+
+    LSF jobs are submitted using bsub and all LSF related options are
+    translated to corresponding bsub options. For maximum flexibility
+    we allow any string to be passed as options to bsub, so if a specific 
+    bsub feature not available through any ot the other -lsf_ options 
+    is needed, use <span class="style2">lsf_options</span> to pass any option to bsub.
+EOF
+
+    $help_hash{-lsf_project_name} = <<'EOF';
+    <p class="style2">-lsf_project_name='string'</p>
+
+    Use <span class="style2">lsf_project_name</span> to assign a
+    project name to your LSF runs.
+EOF
+
+    $help_hash{-lsf_resources} = <<'EOF';
+    <p class="style2">-lsf_resources='string'</p>
+
+    <span class="style2">lsf_resources</span> specifies which LSF resources is required when submiting
+    NONMEM runs.
+EOF
+
+    $help_hash{-lsf_ttl} = <<'EOF';
+    <p class="style2">-lsf_ttl='string'</p>
+
+    <span class="style2">lsf_ttl</span> sets the maximum time a NONMEM run should be allowed to run on 
+    the LSF grid.
+EOF
+
+    $help_hash{-lsf_queue} = <<'EOF';
+    <p class="style2">-lsf_queue='string'</p>
+
+    <span class="style2">lsf_queue</span> specifies which LSF queue PsN should submit NONMEM runs 
+    to and is used in conjuction with <span class="style2">-run_on_lsf</span>
+EOF
+
+    $help_hash{-min_retries} = <<'EOF';
+    <p class="style2">-min_retries='string'</p>
+
+    <span class="style2">min_retries</span> forces the PsN to try
+    several initial values for each estimate and selecting the best
+    one. The best model is chosen in the following maner: if <span class="style2">-picky</span>
+    is used the model must pass the picky test. Then the one with
+    highest number of significant digits and an ofv value no more than
+    five units above than the lowest ofv value among all models.  
 EOF
 
     $help_hash{-remove_temp_files} = <<'EOF';
@@ -382,19 +509,49 @@ EOF
     $help_hash{-clean} = <<'EOF';
     <p class="style2">-clean</p>
 
-    A more thorough version of <span class="style2">'-remove_temp_files'</span>. If the <span class="style2">-clean</span>
+    If the <span class="style2">-clean</span>
     option is set to 1, execute will remove the entire 'NM_runX'
-    directory after the NONMEM run is finished. The dfault value of
+    directory after the NONMEM run is finished. The default value of
     <span class="style2">-clean </span>is 0.
 EOF
 
+    $help_hash{-missing_data_token} = <<'EOF';
+    <p class="style2">-missing_data_token='string'</p>
+
+    <span class="style2">missing_data_token</span> sets the string
+    that PsN accepts as missing data, default is -99.
+EOF
+
+    $help_hash{-nm_directory} = <<'EOF';
+    <p class="style2">-nm_directory='string'</p>
+
+    The argument of <span class="style2">nm_directory</span> is
+    directory where NONMEM is installed. Normally its easiest to setup
+    a version in psn.conf and use <span class="style2">-nm_version</span> to access it.
+EOF
+
+    $help_hash{-no_remote_compile} = <<'EOF';
+    <p class="style2">-no_remote_compile</p>
+
+    When running on LSF it is no guaranteed that NONMEM is available
+    on the computing node, then <span class="style2">-no_remote_compile</span> allows you to compile 
+    NONMEM localy and only submit the NONMEM executable to the grid.
+EOF
+
+    $help_hash{-no_remote_execution} = <<'EOF';
+    <p class="style2">-no_remote_execution</p>
+
+    <span class="style2">no_remote_execution</span> prohibits execution on the LSF grid. Used together 
+    with <span class="style2">-no_remote_compile</span> it cancels out <span class="style2">-run_on_lsf</span>
+EOF
+
     $help_hash{-compress} = <<'EOF';
     <p class="style2">-compress</p>
 
     The execute utility will compress the contents of 'NM_runX' to the
     file 'nonmem_files.tgz' if the <span class="style2">-compress</span> option is used and if you
     have the archive and compress programs <strong>tar</strong> and <strong>gzip</strong> installed. If
-    you use the <span class="style2">-remove_temp_files</span> options, temporary files will be
+    you use the <span class="style2">-clean</span> options, run files will be
     removed before the compression. The <span class="style2">-compress</span> option obviously has
     no effect if you also use the <span class="style2">-clean</span> option.
 EOF
@@ -413,7 +570,6 @@ EOF
     parameters are still valid. For this option to have effect, the
     -retries option must be set to number larger than zero. The
     default setting uses tweak_inits.<-->
-
     <?php print '<p>  If NONMEM terminates nonsuccessfully, PsN can perturb the initial estimates  and run NONMEM again. The generation of new initial estimates <img src="images/init1.gif"> for the <em>i</em>:th retry are performed according to</p><p align="center"><img src="images/perturb1.gif" width="236" height="32"></p> <p>where <img src="images/init_orig1.gif" width="29" height="28"> are the initial estimates of the original run. The updating procedure makes sure that boundary conditions on the parameters are still valid. For this option to be valid, the <span class="style2">-retries</span> option must be set to a number larger than zero. The default setting uses tweak_inits. </p>'; ?>
 EOF
 
@@ -428,11 +584,11 @@ EOF
     $help_hash{-picky} = <<'EOF';
     <p class="style2">-picky</p>
 
-    The <span class="style2">-picky</span> option is only valid together with
-    <span class="style2">-tweak_inits</span>. Normally PsN only tries new initial estimates if
+    The <span class="style2">-picky</span> option is only valid together with <span class="style2">-tweak_inits</span>. 
+    Normally PsN only tries new initial estimates if 
     '<span class="style2">MINIMZATION SUCCESSFUL</span>' is not found in the NONMEM output
-    file. With the <span class="style2">-picky</span> option, PsN will regard any of the following
-    messages as a signal for rerunning:
+    file. With the <span class="style2">-picky</span> option, PsN will regard any of the
+    following messages as a signal for rerunning:
 <p class="style2">
     0ESTIMATE OF THETA IS NEAR THE BOUNDARY<br>
     0PARAMETER ESTIMATE IS NEAR ITS BOUNDARY<br>
@@ -440,14 +596,64 @@ EOF
     0S MATRIX ALGORITHMICALLY SINGULAR</p>
 EOF
 
+    $help_hash{'-quick_summarize|quick_summary'} = <<'EOF';
+    <p><span class="style2">-quick_summarize</span> or <span class="style2">-quick_summary</span></p>
+
+    If either of <span class="style2">quick_summarize</span> and <span class="style2">quick_summary</span> is used, PsN will print 
+    the ofv value and minimization message for each NONMEM run.
+EOF
+
+    $help_hash{-rerun} = <<'EOF';
+    <p class="style2">-rerun</p>
+
+    PsN can redo or resume a run using information in PsN run 
+    directory(see documentation for <span class="style2">-directory</span>). It is called 
+    a rerun. During a rerun PsN will consider to redo parts of 
+    the run. With the <span class="style2">-rerun</span> option you can control which parts 
+    will be redone. The default value of <span class="style2">-rerun</span> is 1. 
+    With rerun set to 1 PsN will rerun any model with a missing 
+    list file. Notice that every "retry" (see the documentation 
+    for <span class="style2">-retries</span> and <span class="style2">-min_retries</span>) will be considered for a rerun.
+    This means you can change the value of the <span class="style2">-retries</span> and 
+    <span class="style2">-min_retries</span> options if you like more or less retries. 
+    Setting <span class="style2">-rerun</span> to 0 means that PsN will not check for 
+    missing or incompleter "retry" list files. This is usefull
+    if you have one or more run modelfiles and you wish to have 
+    a PsN raw_results file or a PsN summary, you do a "execute" 
+    run with them as arguments and specify <span class="style2">-rerun=0</span>, PsN will not 
+    do any NONMEM run, but produce usefull output summary. 
+    You can also set <span class="style2">-rerun</span> to 2, and PsN will ignore any existing 
+    list files and rerun everything, creating raw_results and 
+    summaries from the new listfiles.
+EOF
+
+    $help_hash{-run_on_lsf} = <<'EOF';
+    <p class="style2">-run_on_lsf</p>
+
+    PsN connects with Platform Load Sharing Facility (LsF). With 
+    <span class="style2">-run_on_lsf</span>. PsN will submit to the queue defined in "psn.conf" 
+    unless specified with <span class="style2">-lsf_queue</span>.
+EOF
+
     $help_hash{-retries} = <<'EOF';
     <p class="style2">-retries='integer'</p>
 
     The <span class="style2">-retries</span> option tells the execute utility how many times it
-    shall try to rerun a NONMEM job if it gets an error message. In
+    shall try to rerun a NONMEM job if it fails according to given criterias.. In
     the current version of PsN (2.2), the <span class="style2">-retries</span> option is only
     valid together with <span class="style2">-tweak_inits</span>. The default value of the
-    <span class="style2">-retries</span> option is 0.
+    <span class="style2">-retries</span> option is 6.
+EOF
+
+    $help_hash{-significant_digits_rerun} = <<'EOF';
+    <p class="style2">-significant_digits_rerun='number'</p>
+
+    The <span class="style2">-picky</span> option is only valid together with <span class="style2">-tweak_inits</span>. 
+    Normally PsN only tries new initial estimates if 
+    '<span class="style2">MINIMZATION SUCCESSFUL</span>' is not found in the NONMEM output
+    file. With the <span class="style2">-significant_digits_rerun</span>, PsN will rerun if 
+    the resulting significant digits is lower than the value 
+    specified with this option.
 EOF
 
     $help_hash{-abort_on_fail} = <<'EOF';
@@ -455,7 +661,19 @@ EOF
 
     If the <span class="style2">-abort_on_fail</span> option is set and one of the NONMEM runs
     fails, execute will stop scheduling more runs and try to stop
-    those that are currently running.
+    those that are currently running. A run is considered failed if it
+    fails to produce a list file which PsN can read. This can occure
+    if a nonmem run crashes or gets killed.
+EOF
+
+    $help_hash{-adaptive} = <<'EOF';
+    <p class="style2">-adaptive</p>
+
+    <span class="style2">-adaptive</span> enables a highly experimental feature to dynamically 
+    assign the number of threads depending on the number of running
+    nonmem processes on the computer. It requires a server program
+    which is not distributed with PsN. If you are interrested in this
+    feature, contact the PsN developers.
 EOF
 
     $help_hash{-run_on_nordugrid} = <<'EOF';
@@ -488,7 +706,7 @@ EOF
     <p class="style2">-silent</p>
 
     The silent option turns off all output from PsN. Results and log
-    files are still written to disk, but nothing i printed to the
+    files are still written to disk, but nothing is printed to the
     screen.
 EOF
 
@@ -520,6 +738,16 @@ EOF
     from. This is definitly only for developers.
 EOF
 
+    $help_hash{-warn_with_trace} = <<'EOF';
+    <p class="style2">-warn_with_trace</p>
+
+    If the <span class="style2">-debug</span> level is bigger than zero PsN will print warning 
+    messages. If <span class="style2">-warn_with_trace</span> is set, PsN will print a stack 
+    trace from the point where the warning message was printed. 
+    This is definitly only for developers.
+
+EOF
+
     $help_hash{-sde} = <<'EOF';
     <p class="style2">-sde</p>
 
@@ -528,6 +756,7 @@ EOF
     will fail.
 EOF
 
+    $help_hash{'-h'} = $help_hash{'-?'};
 
 
   if( defined $help_text ){
@@ -550,9 +779,20 @@ EOF
 
       open(OPTIONS, '>', 'html/' . $command . '_options.php' );
       my $opt_help;
-      foreach my $option(sort(keys %help_hash)){
-	if( $option =~ /^-/ ){
-	  $opt_help .= $help_hash{$option}."\n\n";
+
+      if( $command eq 'execute' ){
+	@loop_array = @get_opt_strings;
+      } else {
+	@loop_array = (sort(keys %{$required_options}), sort(keys %{$optional_options}));
+      }
+      
+      foreach my $option( @loop_array ){
+	#foreach my $option(keys %help_hash){
+	$option =~ s/[^\w]*$|:.*//;
+	if( exists $help_hash{'-'.$option}){
+	  $opt_help .= $help_hash{'-'.$option}."\n\n";
+	} else {
+	  $opt_help .= "      <p class=\"style2\">-$option</p>     <p>No help available for '$option'</p>";
 	}
       }
       print OPTIONS $help_hash{Options} . $opt_help;
@@ -574,6 +814,7 @@ EOF
 	    $help .= "\nNo help available for '$option'\n\n";
 	  }
 	} 
+
 	$help =~ s/<\?.*\?>//g;
         $help =~ s/<[^>]*>//g;
 	print $help;
@@ -585,15 +826,28 @@ EOF
       $help .= common_options::print_help($command,$required_options, $optional_options);
 
       
-      if( $options{'help'} and not( $options{'?'} or $options{'h'} ) ){ 
+      if( $options{'help'} ){
 	
 	$help .= "\n\n".$help_hash{Description}."\n\n";
 	$help .= $help_hash{Examples}."\n\n";
 	$help .= $help_hash{Options}."\n\n";
 	
-	foreach my $option(sort(keys %help_hash)){
-	  if( $option =~ /^-/ ){
-	    $help .= $help_hash{$option}."\n\n";
+	my @loop_array;
+	
+	if( $command eq 'execute' ){
+	  @loop_array = @get_opt_strings;
+	} else {
+	  @loop_array = (sort(keys %{$required_options}), sort(keys %{$optional_options}));
+	}
+	
+	foreach my $option( @loop_array ){
+	  #print "special case: $option\n" if ( $option =~ /\W+$|:.*/ );
+	  $option =~ s/[^\w]*$|:.*//;
+	  #$option = '-'.$option unless( $option =~ /^-/ );
+	  if( exists $help_hash{'-'.$option}){
+	    $help .= $help_hash{'-'.$option}."\n\n";
+	  } else {
+	    $help .= "      -$option\n\n      No help available for '$option'\n\n\n";
 	  }
 	}
 	
diff --git a/lib/nonmem_subs.pm b/lib/nonmem_subs.pm
index 94cad64..b27967e 100644
--- a/lib/nonmem_subs.pm
+++ b/lib/nonmem_subs.pm
@@ -54,7 +54,7 @@ start new
 	  if ( $#nmkeys == 0 and
 	       defined $nmkeys[0] and
 	       defined $PsN::config -> { 'nm_versions' } -> {$nmkeys[0]} ) {
-	    print "Using ",$PsN::config -> { 'nm_versions' } -> {$nmkeys[0]}," instead\n";
+	    #print "Using ",$PsN::config -> { 'nm_versions' } -> {$nmkeys[0]}," instead\n";
 	    $nmdir = $PsN::config -> { 'nm_versions' } -> {$nmkeys[0]};
 	  }
 	}
@@ -109,11 +109,11 @@ start compile
 	my $err_version = ( defined $nmdir and $nmdir ne '' ) ? $nmdir : '[not configured]';
 	debug -> die( message => "Unable to find a supported version of NONMEM\n".
 		      "The NONMEM installation directory is $err_version for version [".
-		      $self -> {'version'}."] according to psn.conf" );
+		      $self -> {'version'}."] according to psn.conf." );
       } 
 
       # first clean up from old compile
-      unlink( 'FCON', 'FDATA', 'FREPORT','FSUBS', 'FSUBS.f','LINK.LNK','FSTREAM', 'PRDERR', 'nonmem.exe', 'nonmem' );
+      unlink( 'FLIB','FCON', 'FDATA', 'FREPORT','FSUBS', 'FSUBS.f','LINK.LNK','FSTREAM', 'PRDERR', 'nonmem.exe', 'nonmem', 'nonmem5', 'nonmem6', 'nonmem5_adaptive', 'nonmem6_adaptive' );
       
       my $nm;
       
@@ -146,10 +146,10 @@ start compile
 
       if(-e 'FSUBS'){ 
 	if( $self -> {'compiler'} eq 'g77' ){
-	  cp("FSUBS", "FSUBS.f");
+	  rename("FSUBS", "FSUBS.f");
 	  $fsub='FSUBS.f';
 	} else {
-	  cp("FSUBS", "FSUBS.for");
+	  rename("FSUBS", "FSUBS.for");
 	  $fsub='FSUBS.for';
 	}
       }
@@ -168,7 +168,8 @@ start compile
 	$tmp =~ s/\s+$//; 
 	push(@link, $tmp);
       }
-      
+      close( FH );
+
       my $compile_message;
       my $compile_command;
 
diff --git a/lib/output_subs.pm b/lib/output_subs.pm
index 2b19809..36ce640 100644
--- a/lib/output_subs.pm
+++ b/lib/output_subs.pm
@@ -101,6 +101,7 @@ end new
 # }}} new
 
 # {{{ register_in_database
+
 start register_in_database
   if ( $PsN::config -> {'_'} -> {'use_database'} ) { 
     my $md5sum;
@@ -184,6 +185,7 @@ start register_in_database
     }
   }
 end register_in_database
+
 # }}} register_in_database
 
 # {{{ full_name
@@ -754,8 +756,17 @@ start _read_problems
 	}
 	$self -> {'lstfile'} = undef;
 	unless( $success ){
-	  debug -> die( message => 'The listfile "' . $self -> full_name . '" seems malformatted.' );
+	  if( $self -> {'abort_on_fail'} ){
+	    debug -> die( message => 'The listfile "' . $self -> full_name . '" seems malformatted.' );
+	  } else {
+	    debug -> warn( level => 1,
+			   message => 'The listfile "' . $self -> full_name . '" seems malformatted.' );
+	    return 0;
+	  }	   
+	} else {
+	  $self -> {'parsed_successfully'} = 1;
 	}
+	$self -> {'parsed'} = 1;
       }
 end _read_problems
 
@@ -785,7 +796,7 @@ start access_any
 	}
 	unless( $#problems > 0 ){
 	  debug -> warn(level => 2,
-			message => "Output -> access_any: problems undefined, using all" );
+			message => "Problems undefined, using all" );
 	  @problems = (1 .. scalar @{$self -> {'problems'}});
 	}
 	my @own_problems = @{$self -> {'problems'}};
diff --git a/lib/tool/bootstrap_subs.pm b/lib/tool/bootstrap_subs.pm
index 511f9ae..3bcb551 100644
--- a/lib/tool/bootstrap_subs.pm
+++ b/lib/tool/bootstrap_subs.pm
@@ -217,6 +217,7 @@ start general_setup
 		   logfile	         => undef,
 		   raw_results           => undef,
 		   prepared_models       => undef,
+		   top_tool              => 0,
 		   %subargs );
 	  
 	  ui -> print( category => 'bootstrap',
@@ -554,6 +555,7 @@ start general_setup
 		   logfile		 => $self -> {'logfile'}[$model_number-1],
 		   raw_results           => undef,
 		   prepared_models       => undef,
+		   top_tool              => 0,
 		   %subargs ) );
 
 # 	      ( clean            => $self -> {'clean'},
@@ -1952,14 +1954,14 @@ end print_results
 
 start create_matlab_scripts
     {
-      if( defined $PsN::config -> {'_'} -> {'matlab_dir'} ){
-	unless( -e $PsN::config -> {'_'} -> {'matlab_dir'} . '/histograms.m' and
-		-e $PsN::config -> {'_'} -> {'matlab_dir'} . '/bca.m' ){
+      if( defined $PsN::lib_dir ){
+	unless( -e $PsN::lib_dir . '/histograms.m' and
+		-e $PsN::lib_dir . '/bca.m' ){
 	  'debug' -> die( message => 'Bootstrap matlab template scripts are not installed, no matlab scripts will be generated.' );
 	  return;
 	}
 
-	open( PROF, $PsN::config -> {'_'} -> {'matlab_dir'} . '/histograms.m' );
+	open( PROF, $PsN::lib_dir . '/histograms.m' );
 	my @file = <PROF>;
 	close( PROF );
 	my $found_code;
@@ -2103,7 +2105,7 @@ start create_matlab_scripts
 	
 	splice( @file, $code_area_start, ($code_area_end - $code_area_start), @auto_code );	
 	open( OUTFILE, ">", $self -> {'directory'} . "/histograms.m" );
-	print OUTFILE "addpath " . $PsN::config -> {'_'} -> {'matlab_dir'} . ";\n";
+	print OUTFILE "addpath " . $PsN::lib_dir . ";\n";
 	print OUTFILE @file ;
 	close OUTFILE;
 
diff --git a/lib/tool/cdd_subs.pm b/lib/tool/cdd_subs.pm
index eab4d47..e2fb3da 100644
--- a/lib/tool/cdd_subs.pm
+++ b/lib/tool/cdd_subs.pm
@@ -388,6 +388,7 @@ start general_setup
 		   logfile	         => undef,
 		   raw_results           => undef,
 		   prepared_models       => undef,
+		   top_tool              => 0,
 		   %subargs );
 
 #        	$Data::Dumper::Maxdepth=1;
@@ -802,6 +803,7 @@ start general_setup
 		   logfile	         => undef,
 		   raw_results           => undef,
 		   prepared_models       => undef,
+		   top_tool              => 0,
 		   %subargs ) );
 
 
@@ -1038,6 +1040,7 @@ start modelfit_analyze
 		   logfile	    => undef,
 		   raw_results      => undef,
 		   prepared_models  => undef,
+		   top_tool         => 0,
 		   retries          => 1 );
 	  $Data::Dumper::Maxdepth = 2;
 #	  die Dumper $mod_eval;
diff --git a/lib/tool/llp_subs.pm b/lib/tool/llp_subs.pm
index 4eab631..1331234 100644
--- a/lib/tool/llp_subs.pm
+++ b/lib/tool/llp_subs.pm
@@ -1867,13 +1867,13 @@ end print_results
 # {{{ create_matlab_scripts
 start create_matlab_scripts
     {
-      if( defined $PsN::config -> {'_'} -> {'matlab_dir'} ){
-	unless( -e $PsN::config -> {'_'} -> {'matlab_dir'} . '/profiles.m') {
+      if( defined $PsN::lib_dir ){
+	unless( -e $PsN::lib_dir . '/profiles.m') {
 	  'debug' -> die( message => 'LLP matlab template scripts are not installed, no matlab scripts will be generated.' );
 	  return;
 	}
 
-	open( PROF, $PsN::config -> {'_'} -> {'matlab_dir'} . '/profiles.m' );
+	open( PROF, $PsN::lib_dir . '/profiles.m' );
 	my @file = <PROF>;
 	close( PROF );	
       	my $found_code;
@@ -1916,7 +1916,7 @@ start create_matlab_scripts
 	
 	splice( @file, $code_area_start, ($code_area_end - $code_area_start), @auto_code );	
 	open( OUTFILE, ">", $self -> {'directory'} . "/profiles.m" );
-	print OUTFILE "addpath " . $PsN::config -> {'_'} -> {'matlab_dir'} . ";\n";
+	print OUTFILE "addpath " . $PsN::lib_dir . ";\n";
 	print OUTFILE @file ;
 	close OUTFILE;
 	
diff --git a/lib/tool/mc_subs.pm b/lib/tool/mc_subs.pm
index 7a6b5a1..3506cc6 100644
--- a/lib/tool/mc_subs.pm
+++ b/lib/tool/mc_subs.pm
@@ -302,7 +302,6 @@ push( @{$self -> {'tools'}},
 	compress       => $self -> {'compress'},
 	threads        => $subm_threads,
 	retries        => $self -> {'retries'},
-	remove_temp_files  => $self -> {'remove_temp_files'},
 	base_directory => $self -> {'directory'},
 	directory      => $self -> {'directory'}.'/'.$subdir.'_dir'.$model_number,
 	subtools       => $#subtools >= 0 ? \@subtools : undef,
diff --git a/lib/tool/modelfit_subs.pm b/lib/tool/modelfit_subs.pm
index e8ab4f4..1d1b417 100644
--- a/lib/tool/modelfit_subs.pm
+++ b/lib/tool/modelfit_subs.pm
@@ -24,6 +24,7 @@ start include statements
 	use OSspecific;
 	use ui;
 	use moshog_client;
+	use Time::HiRes;
       }
 end include statements
 
@@ -384,16 +385,23 @@ start prepare_raw_results
 		  }
 		}
 	      } elsif( $category eq 'eigen') {
-		foreach my $prob_eigens ( @{$self -> {'models'} -> [$i] -> outputs -> [0] -> eigens} ){
-		  foreach my $subprob_eigens( @{$prob_eigens} ){
-		    if( defined $subprob_eigens ){
-		      $numpar = $numpar < scalar @{$subprob_eigens} ? scalar @{$subprob_eigens} : $numpar ;
+		if( $self -> {'models'} -> [$i] -> outputs -> [0] -> parsed and
+		    $self -> {'models'} -> [$i] -> outputs -> [0] -> parsed_successfully ){
+		  foreach my $prob_eigens ( @{$self -> {'models'} -> [$i] -> outputs -> [0] -> eigens} ){
+		    foreach my $subprob_eigens( @{$prob_eigens} ){
+		      if( defined $subprob_eigens ){
+			$numpar = $numpar < scalar @{$subprob_eigens} ? scalar @{$subprob_eigens} : $numpar ;
+		      }
 		    }
 		  }
 		}
 	      } else {
-		my $labels = $self -> {'models'} -> [$i] -> {'outputs'} -> [0] -> labels( parameter_type => $category );
-
+		my $labels;
+		if( $self -> {'models'} -> [$i] -> outputs -> [0] -> parsed and
+		    $self -> {'models'} -> [$i] -> outputs -> [0] -> parsed_successfully ){
+		  
+		  $labels = $self -> {'models'} -> [$i] -> {'outputs'} -> [0] -> labels( parameter_type => $category );
+		}
 		# we can't use labels directly since different models may have different
 		# labels (still within the same modelfit)
 		if( defined $labels ) {
@@ -423,93 +431,97 @@ start prepare_raw_results
 	my $pushed_rows = 0; # All rows pushed on result array in previous models
 	my $raw_line_structure = ext::Config::Tiny -> new( );
  	for ( my $i = 0; $i < scalar @{$self -> {'models'}}; $i++ ) { # models level
-	  my @probs = @{$self -> {'models'} -> [$i] -> {'outputs'} -> [0] -> problem_structure};
-	  my $np = scalar @probs; # #probs
-	  my $model_row = 0;
-	  # ------------  Push model, problem and sub-problem numbers  --------------
-
-	  for( my $j = 0; $j < $np; $j++ ) {
-	    my $ns = $probs[$j]; # #subprobs
-	    for( my $k = 0; $k < $ns; $k++ ) {
-	      my $row = $pushed_rows+$model_row++;
-	      push( @{$self -> {'raw_results'} -> [$row]}, (($i+1),($j+1),($k+1)) );
-	    }
+	  unless( $self -> {'models'} -> [$i] -> outputs -> [0] -> parsed ){
+	    $self -> {'models'} -> [$i] -> outputs -> [0] -> _read_problems;
 	  }
+	  if( $self -> {'models'} -> [$i] -> outputs -> [0] -> parsed_successfully ){
+	    my @probs = @{$self -> {'models'} -> [$i] -> {'outputs'} -> [0] -> problem_structure};
+	    my $np = scalar @probs; # #probs
+	    my $model_row = 0;
+	    # ------------  Push model, problem and sub-problem numbers  --------------
+
+	    for( my $j = 0; $j < $np; $j++ ) {
+	      my $ns = $probs[$j]; # #subprobs
+	      for( my $k = 0; $k < $ns; $k++ ) {
+		my $row = $pushed_rows+$model_row++;
+		push( @{$self -> {'raw_results'} -> [$row]}, (($i+1),($j+1),($k+1)) );
+	      }
+	    }
 
-	  # ---------------------  Loop all result categories  ----------------------
-	  my $position = 0;
-	  foreach my $category ( @{$self -> {'raw_results_header'}},'npomega' ){
-	    next if( $category eq 'model' or $category eq 'problem' or $category eq 'subproblem' );
-	    my $model_row = 0; # Need to mask previous definition of model_row
-	    my ( $accessor, $res );
-
-	    # {{{ Get the values for the category
-
-	    if ( $category eq 'theta' or $category eq 'omega' or $category eq 'sigma' or
-		 $category eq 'setheta' or $category eq 'seomega' or $category eq 'sesigma' or
-		 $category eq 'npomega' or $category eq 'eigen' ) {
-	      $accessor = $category.'s';
-	      $res = $self -> {'models'} -> [$i] -> {'outputs'} -> [0] -> $accessor;
-
-	      if( defined $res and ref $res eq 'ARRAY' and $category ne 'npomega' ){
-		my $prob_num = 0;
-		foreach my $prob ( @{$res} ){
-		  if( defined $prob and ref $prob eq 'ARRAY' and $prob -> [0] and ref $prob -> [0] eq 'ARRAY' ){
-		    $raw_line_structure -> { $i+$prob_num }{ $category } = scalar @{$self -> {'raw_results'} -> [$i+$prob_num]}+$position 
-		                                                           . ",". scalar @{$prob -> [0]};
-		    #$position += $#{$prob -> [0]};
+	    # ---------------------  Loop all result categories  ----------------------
+	    my $position = 0;
+	    foreach my $category ( @{$self -> {'raw_results_header'}},'npomega' ){
+	      next if( $category eq 'model' or $category eq 'problem' or $category eq 'subproblem' );
+	      my $model_row = 0; # Need to mask previous definition of model_row
+	      my ( $accessor, $res );
+
+	      # {{{ Get the values for the category
+
+	      if ( $category eq 'theta' or $category eq 'omega' or $category eq 'sigma' or
+		   $category eq 'setheta' or $category eq 'seomega' or $category eq 'sesigma' or
+		   $category eq 'npomega' or $category eq 'eigen' ) {
+		$accessor = $category.'s';
+		$res = $self -> {'models'} -> [$i] -> {'outputs'} -> [0] -> $accessor;
+
+		if( defined $res and ref $res eq 'ARRAY' and $category ne 'npomega' ){
+		  my $prob_num = 0;
+		  foreach my $prob ( @{$res} ){
+		    if( defined $prob and ref $prob eq 'ARRAY' and $prob -> [0] and ref $prob -> [0] eq 'ARRAY' ){
+		      $raw_line_structure -> { $i+$prob_num }{ $category } = scalar @{$self -> {'raw_results'} -> [$i+$prob_num]}+$position 
+			  . ",". scalar @{$prob -> [0]};
+		      #$position += $#{$prob -> [0]};
+		    }
+		    $prob_num++;
 		  }
-		  $prob_num++;
 		}
-	      }
 
-	    } elsif ( $category eq 'shrinkage_etas' ) {
-	      # Shrinkage does not work for subproblems right now.
-	      $res = $self -> {'models'} -> [$i] -> eta_shrinkage;
-	    } elsif ( $category eq 'shrinkage_wres' ) {
-	      # Shrinkage does not work for subproblems right now.
-	      # get ofv just to get the prob-subp structure
-	      $res = $self -> {'models'} -> [$i] -> wres_shrinkage;
-	    } else {
-	      $accessor = $category;
-	      $res = $self -> {'models'} -> [$i] -> {'outputs'} -> [0] -> $accessor;
-	    }
+	      } elsif ( $category eq 'shrinkage_etas' ) {
+		# Shrinkage does not work for subproblems right now.
+		$res = $self -> {'models'} -> [$i] -> eta_shrinkage;
+	      } elsif ( $category eq 'shrinkage_wres' ) {
+		# Shrinkage does not work for subproblems right now.
+		# get ofv just to get the prob-subp structure
+		$res = $self -> {'models'} -> [$i] -> wres_shrinkage;
+	      } else {
+		$accessor = $category;
+		$res = $self -> {'models'} -> [$i] -> {'outputs'} -> [0] -> $accessor;
+	      }
 
-	    # }}} Get the values for the category
-	    my $results_file;
-	    if( $category eq 'npomega' ){
-	      $results_file = 'raw_nonp_results';
-	    } else {
-	      $results_file = 'raw_results';
-	    }
+	      # }}} Get the values for the category
+	      my $results_file;
+	      if( $category eq 'npomega' ){
+		$results_file = 'raw_nonp_results';
+	      } else {
+		$results_file = 'raw_results';
+	      }
 
-	    if( defined $res ) {
-	      for( my $j = 0; $j < $np; $j++ ) {
-		my $ns = $probs[$j]; # #subprobs
-		if( defined $res -> [$j] ) {
-		  for( my $k = 0; $k < $ns; $k++ ) {
-		    my $row = $pushed_rows+$model_row++;
-		    if( ref $res -> [$j] eq 'ARRAY' ){
-		      if( defined $res -> [$j][$k] ) {
-			if ( ref $res -> [$j][$k] eq 'ARRAY' ) {
-			  push( @{$self -> {$results_file} -> [$row]}, @{$res -> [$j][$k]} );
-			  push( @{$self -> {$results_file} -> [$row]},
-				(undef) x ($max_hash{$category}- scalar @{$res -> [$j][$k]})  );
+	      if( defined $res ) {
+		for( my $j = 0; $j < $np; $j++ ) {
+		  my $ns = $probs[$j]; # #subprobs
+		  if( defined $res -> [$j] ) {
+		    for( my $k = 0; $k < $ns; $k++ ) {
+		      my $row = $pushed_rows+$model_row++;
+		      if( ref $res -> [$j] eq 'ARRAY' ){
+			if( defined $res -> [$j][$k] ) {
+			  if ( ref $res -> [$j][$k] eq 'ARRAY' ) {
+			    push( @{$self -> {$results_file} -> [$row]}, @{$res -> [$j][$k]} );
+			    push( @{$self -> {$results_file} -> [$row]},
+				  (undef) x ($max_hash{$category}- scalar @{$res -> [$j][$k]})  );
+			  } else {
+			    push( @{$self -> {$results_file} -> [$row]}, $res -> [$j][$k] );
+			  }
 			} else {
-			  push( @{$self -> {$results_file} -> [$row]}, $res -> [$j][$k] );
+			  push( @{$self -> {$results_file} -> [$row]},
+				(undef) x $max_hash{$category}  );
 			}
 		      } else {
 			push( @{$self -> {$results_file} -> [$row]},
-			      (undef) x $max_hash{$category}  );
+			      $res -> [$j] );
 		      }
-		    } else {
-		      push( @{$self -> {$results_file} -> [$row]},
-			    $res -> [$j] );
 		    }
-		  }
-		} else {
-		  
-		  # {{{ Push undefs for missing subprobs
+		  } else {
+		    
+		    # {{{ Push undefs for missing subprobs
 
 		  for( my $k = 0; $k < $ns; $k++ ) {
 		    my $row = $pushed_rows+$model_row++;
@@ -518,13 +530,13 @@ start prepare_raw_results
 		  }
 
 		  # }}} Push undefs for missing subprobs
-		  
-		}
+		    
+		  }
 
-	      }
-	    } else {
-	      
-	      # {{{ Push undefs for missing probs/subprobs
+		}
+	      } else {
+		
+		# {{{ Push undefs for missing probs/subprobs
 
 	      for( my $j = 0; $j < $np; $j++ ) {
 		my $ns = $probs[$j]; # #subprobs
@@ -536,15 +548,17 @@ start prepare_raw_results
 	      }
 
 	      # }}} Push undefs for missing probs/subprobs
-	      
+		
+	      }
 	    }
+	    $self -> {'models'} -> [$i] -> {'outputs'} -> [0] -> flush;
+	    map( $pushed_rows += $_, @probs );
 	  }
-	  $self -> {'models'} -> [$i] -> {'outputs'} -> [0] -> flush;
-	  map( $pushed_rows += $_, @probs );
 	}
 	$raw_line_structure -> write( 'raw_file_structure' );
+
 	# }}} values
-	
+
 	&{$self -> {'_raw_results_callback'}}( $self, \%max_hash )
 	    if ( defined $self -> {'_raw_results_callback'} );
 
@@ -591,8 +605,14 @@ end prepare_raw_results
 start print_raw_results
       {
 	if ( defined $self -> {'raw_results'} ) {
+	  my $raw_file;
+	  if( ref $self -> {'raw_results_file'} eq 'ARRAY' ){
+	    $raw_file = $self -> {'raw_results_file'} -> [0];
+	  } else {
+	    $raw_file = $self -> {'raw_results_file'};
+	  }
 	  my ($dir,$file) = OSspecific::absolute_path( $self -> {'directory'},
-						       $self -> {'raw_results_file'} );
+						       $raw_file );
 	  my $append = $self -> {'raw_results_append'} ? '>>' : '>';
 	  open( RRES, $append.$dir.$file );
 	  print RRES join(',',@{$self -> {'raw_results_header'}} ),"\n"
@@ -783,24 +803,44 @@ start copy_model_and_output
     }
   }
 
-  unlink 'nonmem', 'nonmem6', 'nonmem5','nonmem.exe', 'nonmem6_adaptive', 'nonmem5_adaptive';
-  if( $self -> {'clean'} >= 1){
-    unlink 'nonmem', 'nonmem'.$self -> {'nm_version'}, 'nonmem.exe','FDATA', 'FREPORT', 'FSUBS', 'FSUBS.f', 'FSUBS.for', 'LINK.LNK', 'FSTREAM';
-    foreach my $data ( @{$model -> datas} ){
-      unlink $data -> filename;
+  # Keep files if debugging
+
+  if( 'debug' -> level == 0) {
+    unlink 'nonmem', 'nonmem6', 'nonmem5', 
+           'nonmem.exe', 'nonmem5_adaptive','nonmem6_adaptive', 'nonmem_adaptive',
+           'FDATA';
+    unlink( @{$model -> datafiles}, @{$model -> extra_data_files} );
+  }
+
+  if( $self -> {'clean'} >= 1 and 'debug' -> level == 0 ){
+    unlink 'nonmem', 'nonmem'.$self -> {'nm_version'}, 
+           'nonmem.exe','FDATA', 'FREPORT', 'FSUBS', 'FSUBS.f', 
+           'FSUBS.for', 'LINK.LNK', 'FSTREAM', 'FCON.orig', 'FLIB', 'FCON','PRDERR';
+    
+    if( defined $model -> extra_files ){
+      foreach my $x_file( @{$model -> extra_files} ){
+	my ( $dir, $filename ) = OSspecific::absolute_path( $model -> directory,
+							    $x_file );
+	unlink( $filename );
+      }
     }
+
     for ( my $i = 1; $i <= $self -> {'retries'}; $i++ ) {
+
+      if( $self -> {'clean'} >= 2 ){
+	foreach my $table ( @{$model -> table_names -> [0]} ){
+	  unlink( $table, "$table-$i" );
+	}
+	unlink( "psn-$i.lst" );
+	unlink( 'compilation_output.txt.'.$i );
+	unlink( 'compilation_output.txt' );
+      }      
+
       unlink( 'FCON.'.$i );
-      unlink( "psn-$i.lst" );
       unlink( "psn-$i.mod" );
-      unlink( 'compilation_output.txt.'.$i );
     }
   }
 
-  if ( $self -> {'clean'} >= 2 ) {
-    unlink 'FCON', 'PRDERR';
-  }
-
   if ( $self -> {'clean'} >= 3 ) {
     # Do nothing. "run_nonmem" will remove entire work directory
     # before returning.
@@ -1150,6 +1190,8 @@ start run_nonmem
       my $modelfile_tainted = 1;
       my $nonmem;
 
+      # -------------- Notes about local vs remote compilation/execution -----------------
+
       # Here we check wheter compilation and/or exection will be done
       # localy or remote. (Remote compilation is only supported on LSF
       # so far). So, unless we are doing everything remotely, we will
@@ -1213,7 +1255,7 @@ start run_nonmem
       # Otherwise $tries can be incremented twice for one run. The
       # oposite is not true however, for instance a reset of maxevals
       # is not a retry but sets $marked_for_rerun to 1.
-
+      
       while( $marked_for_rerun and $tries < $retries ){
 	# {{{ Run nonmem and get minimization messages
 
@@ -1226,7 +1268,7 @@ start run_nonmem
 	      # {{{ local compile requested
 
 	      unless( $nonmem -> compile() ){
-		debug -> die( message => "NONMEM compilation failed:\n" . $nonmem -> error_message );
+		debug -> die( message => "NONMEM compilation of ". $model -> full_name . "failed:\n" . $nonmem -> error_message );
 	      }
 	      
 	      unless ( -e 'FCON.orig' ) {
@@ -1309,6 +1351,7 @@ start run_nonmem
 	  
 	  # We need the trail of files to select the most appropriate at the end
 	  # (see copy_model_and_output)
+	  cp( 'compilation_output.txt', 'compilation_output.txt.'.($tries+1) );
 	  cp( 'FCON', 'FCON.'.($tries+1) );
 	  cp( 'psn.lst', 'psn-'.($tries+1).'.lst' );
 	  cp( 'psn.mod', 'psn-'.($tries+1).'.mod' );
@@ -1327,7 +1370,7 @@ start run_nonmem
 	  # and we copy it to psn.lst to make it the current version.
 	  cp( 'psn-'.($tries+1).'.lst', 'psn.lst' );
 	  
-	}	  
+	}
 
 	if ( $PsN::config -> {'_'} -> {'use_database'} ) {
 	  # Save the temporary run in the database
@@ -1344,7 +1387,6 @@ start run_nonmem
 	}
 #	  }
 #	}
-	cp( 'compilation_output.txt', 'compilation_output.txt.'.($tries+1) );
 
 	$marked_for_rerun = 0;
 
@@ -1353,8 +1395,17 @@ start run_nonmem
 
 	if ( -e 'psn.lst' ) {
 	  $output_file = output -> new ( filename => 'psn.lst',
-					 model_id => $self -> {'model_id'} );
+					 model_id => $self -> {'model_id'},
+					 abort_on_fail => $self -> {'abort_on_fail'} );
+	  unless( defined $output_file -> problems ){
+	    return (-1,$final_model,undef);
+	  }
 	  $minimization_successful       = $output_file -> minimization_successful;
+
+	  unless( defined $minimization_successful ) {
+	    debug -> die( message => "No minimization status found in " . $output_file ->filename );
+	  }
+
 	  $significant_digits            = $output_file -> significant_digits;
 	  $evals[$tries]              += $output_file -> feval -> [0][0];
 
@@ -1369,10 +1420,6 @@ start run_nonmem
 	  $run_results[$tries] -> {'pass_picky'} = 0;
 
 	  # }}}
-	  
-	  unless( defined $minimization_successful ) {
-	    debug -> die( message => "No minimization status found in " . $output_file ->filename );
-	  }
 
 	  $minimization_message = $output_file -> minimization_message;
 	  my $ofv = $output_file -> ofv;
@@ -2325,35 +2372,35 @@ start print_finish_message
       my $log_text = $run+1 . ',' . $self -> {'models'} -> [$run] -> filename . ',';
       if( $self -> {'verbose'} or $self -> {'quick_summarize'} ){
 	foreach my $param ( 'ofv', 'covariance_step_successful', 'minimization_message' ) {
-	  my $ests = $final_model -> outputs -> [0] -> $param;
 	  if( $param eq 'minimization_message' ){
 	    $ui_text .= "\n    ---------- Minimization Message ----------\n";
 	  }
-	  
-	  # Loop the problems
-	  for ( my $j = 0; $j < scalar @{$ests}; $j++ ) {
-	    if ( ref( $ests -> [$j][0] ) ne 'ARRAY' ) {
-	      $ests -> [$j][0] =~ s/^\s*//;
-	      $ests -> [$j][0] =~ s/\s*$//;
-	      $log_text .= $ests -> [$j][0] .',';
-	      #chomp($ests -> [$j][0]);
-	      $ui_text .= sprintf("%10s",$ests -> [$j][0]);
-	    } else {
-	      
-	      # Loop the parameter numbers (skip sub problem level)
-	      for ( my $num = 0; $num < scalar @{$ests -> [$j][0]}; $num++ ) {
-		#$ests -> [$j][0][$num] =~ s/^\s*//;
-		#$ests -> [$j][0][$num] =~ s/\s*$/\n/;
-		$log_text .= $ests -> [$j][0][$num] .',';
-		#chomp($ests -> [$j][0][$num]);
-		if( $param eq 'minimization_message' ){
-		  $ui_text .= "    ";
+	  if( defined $final_model ){
+	    my $ests = $final_model -> outputs -> [0] -> $param;
+	    # Loop the problems
+	    for ( my $j = 0; $j < scalar @{$ests}; $j++ ) {
+	      if ( ref( $ests -> [$j][0] ) ne 'ARRAY' ) {
+		$ests -> [$j][0] =~ s/^\s*//;
+		$ests -> [$j][0] =~ s/\s*$//;
+		$log_text .= $ests -> [$j][0] .',';
+		#chomp($ests -> [$j][0]);
+		$ui_text .= sprintf("%10s",$ests -> [$j][0]);
+	      } else {
+		
+		# Loop the parameter numbers (skip sub problem level)
+		for ( my $num = 0; $num < scalar @{$ests -> [$j][0]}; $num++ ) {
+		  #$ests -> [$j][0][$num] =~ s/^\s*//;
+		  #$ests -> [$j][0][$num] =~ s/\s*$/\n/;
+		  $log_text .= $ests -> [$j][0][$num] .',';
+		  #chomp($ests -> [$j][0][$num]);
+		  if( $param eq 'minimization_message' ){
+		    $ui_text .= "    ";
+		  }
+		  $ui_text .= sprintf("%12s",$ests -> [$j][0][$num]);
 		}
-		$ui_text .= sprintf("%12s",$ests -> [$j][0][$num]);
 	      }
 	    }
 	  }
-	  
 	  if( $param eq 'minimization_message' ){
 	    $ui_text .= "    ------------------------------------------\n\n";
 	  }
@@ -2393,88 +2440,82 @@ start run
       # modelfile. And moves the outputfile to the directory
       # where the script was started from.
 
-      my $threads;
+      my $cwd = getcwd();
+      chdir( $self -> {'directory'} );
+      
       my @models;
       if ( defined $self -> {'models'} ) {
 	@models = @{ $self -> {'models'} };
       } else {
 	debug -> die( message => "Have no models!" );
       }
-      my $cwd = getcwd();
-      chdir( $self -> {'directory'} );
-
-      $threads = $self -> {'threads'};
+      
+      my $threads = $self -> {'threads'};
       $threads = $#models + 1 if ( $threads > $#models + 1); 
-
+      
       # Currently parallel execution is not supported on windows
       # platforms. Because of regular expression issues with
       # Parallel::Forkmanager.
       $threads = 1 if( $Config{osname} eq 'MSWin32' );
-
+      
       ui -> print( category => 'all',
 		   message  => 'Starting ' . scalar(@models) . ' NONMEM executions. '. $threads .' in parallel.' ) 
 	  unless $self -> {'parent_threads'} > 1;
       ui -> print( category => 'all',
 		   message  => "Run number\tModel name\tOFV\tCovariance step successful." )  if $self -> {'verbose'};      
-
+      
       # Print model-NM_run translation file
       open( MNT, ">model_NMrun_translation.txt");
       for ( my $run = 0; $run <= $#models; $run ++ ) {
 	print MNT sprintf("%-40s",$models[$run]->filename),"NM_run",($run+1),"\n";
       }
       close( MNT );
-
+      
       # {{{ Setup of parallel forkmanager
-
+      
       my $moshog_client;
-
+      
       my $moshog_client;
       if( $self -> {'adaptive'} and $self -> {'threads'} > 1 ) {
-      
+	
 	# Make a moshog request. Answer will be read on run_on_finish
-      
+	
 	$moshog_client = moshog_client -> new(start_threads => $self -> {'threads'});
 	# print "Making initial request for: ", scalar @models - $self -> {'threads'} , " threads\n";
 	$moshog_client -> request( request => scalar @models - $self -> {'threads'} );
 	
 	# print "Initial gran, got new threads: ", $self -> {'threads'} + $moshog_client -> granted(), "\n";	
-
+	
 	$threads = $self -> {'threads'} + $moshog_client -> granted();
-
+	
       }
-
+      
       my $pm = ext::Parallel::ForkManager -> new ($threads) if ( $threads > 1 );
-
+      
       # These are variables used in forkmanager callbacks.
       #
       # $run is the loop variable of the nonmem loop. Can be used to
       # keep track of progress. (Which we do in the "adaptive" code).
-
+      
       my $run = 0;
       my @children;
       my $keepforking = 1;
       my $aborting = 0;
       my $fork_finished = 0;
-
+      
       if ( $threads > 1 ) {
-
+	
 	# This following call defines a function that will be
 	# called when all children have exited. It detects
 	# errors from children and kills remaining children.
 	$pm -> run_on_finish( 
-			      sub { my ( $pid, $exit_code, $ident, $exit_signal ) = @_;
-				    my $tries = $exit_code;
-
+			      sub { my ( $pid, $tries, $ident, $exit_signal ) = @_;
+				    
 #				    if( $exit_signal ){
 #				      debug -> die( message => "A NONMEM run failed" );
 #				    }
 
-				    if( $tries < 0 ) { 
-
-# !!!!!!!!!!!!!!!!!!!!!
-# tries starts on 0 so this will likely never occur
-# Talk to Pontus
-
+				    if( $tries < 0 ) { 					
 				      if( $self -> {'abort_on_fail'} and not $aborting){
 					$aborting = 1;
 					debug -> die( message => "Some nonmem runs failed, aborting" );
@@ -2500,12 +2541,8 @@ start run
 				      # $self -> {'evals'} -> [$ident] = \@evals;
 				      
 				      my $modulus = (($#models+1) <= 10) ? 1 : (($#models+1) / 10)+1;
-
-				      if ( $ident % $modulus == 0
-					   or
-					   $ident == 0
-					   or
-					   $ident == $#models ) {
+				      
+				      if ( $ident % $modulus == 0 or $ident == 0 or $ident == $#models ) {
 					if( $fork_finished ){
 					  ui -> print( category => 'all',
 						       message  => 'F:'.($ident+1).' .. ',
@@ -2514,73 +2551,60 @@ start run
 					      unless ( $self -> {'parent_threads'} > 1 or $self -> {'verbose'} );
 					}
 				      }
-
-#				      if( $self -> {'adaptive'} ){
-#					my $request = scalar @models - $run - $self -> {'threads'};
-#					unless( $request + $self -> {'threads'} eq $threads or $request <= 0 ){
-#					  print "Exit Requesting\n"; sleep( rand() % 3 );
-#					  $moshog_client -> request( request => $request );
-#					  $threads = $self -> {'threads'} + $moshog_client -> granted();
-#					  print "Child finished, got new threads: $threads.";
-
-#					}
-
-#					if( $request <= 0 ){
-#					  $threads = $self -> {'threads'};
-#					}
-
-#					print  "We had ", $pm -> set_max_procs($threads) , " threads \n"; 
-#				      }
-
 				    }
 				  } );
-	$pm -> run_on_wait( 
-			    sub {
-			      
-			      if( $self -> {'adaptive'} and $self -> {'threads'} > 1 ){
-				my $request = scalar @models - $run - $self -> {'threads'};
-				unless( $request + $self -> {'threads'} eq $threads or $request <= 0 ){
-				  $moshog_client -> request( request => $request );
-				  $threads = $self -> {'threads'} + $moshog_client -> granted();
-				  				}
-				
-				if( $request <= 0 ){
-				  $threads = $self -> {'threads'};
-				}
-				
-				my $old_threads = $pm -> set_max_procs($threads);
 
-				if( $self -> {'verbose'} and $old_threads != $threads ){
-				  ui -> print( category => 'all',
-					       message => "Thread: $threads ($old_threads).",
-					       newline => 0 );
+	# run_on_wait seam to create a penalty when children finishes,
+	# so if we are not adaptive, we skip it.
+
+	if( $self -> {'adaptive'} ){
+	  $pm -> run_on_wait( 
+			      sub {
+				
+				if( $self -> {'adaptive'} and $self -> {'threads'} > 1 ){
+				  my $request = scalar @models - $run - $self -> {'threads'};
+				  unless( $request + $self -> {'threads'} eq $threads or $request <= 0 ){
+				    $moshog_client -> request( request => $request );
+				    $threads = $self -> {'threads'} + $moshog_client -> granted();
+				  }
+				  
+				  if( $request <= 0 ){
+				    $threads = $self -> {'threads'};
+				  }
+				  
+				  my $old_threads = $pm -> set_max_procs($threads);
+				  
+				  if( $self -> {'verbose'} and $old_threads != $threads ){
+				    ui -> print( category => 'all',
+						 message => "Thread: $threads ($old_threads).",
+						 newline => 0 );
+				  }
 				}
 			      }
-			    }
-			    , 5 );
+			      , 5 );
+	}
 
       }
 
       # }}}
-
+      
       # {{{ Local execution
 
       my $actual_runs = -1; # Keep track of how many runs we start (only important for the
-                            # sleep statement below )
+      # sleep statement below )
 
       # $run is defined above so that we can access it from
       # forkmanager callbacks.
       for ( $run = 0; $run <= $#models; $run ++ ) {
 	$actual_runs++;
-	if ( $threads > 6 and $actual_runs != 0 and not (($actual_runs)%6) ) {
-	  sleep(1);
-	}
+#	if ( $threads > 6 and $actual_runs != 0 and not (($actual_runs)%6) ) {
+#	  sleep(1);
+#	}
 	my $done = 0;
 	my ( @seed, $tries, $final_model, @evals, $evals_ref );
 	if ( $keepforking ) {
-
 	  if ( -e $self -> {'models'} -> [$run] -> outputs -> [0] -> full_name and
-	       not -e "./NM_run" . ($run+1) ){
+	       not -e '/.NM_run' . ($run+1) and not $self -> {'rerun'} > 0 ){
 	    mkdir( "./NM_run" . ($run+1) );
 	    open( DONE, ">./NM_run". ($run+1) ."/done" );
 	    print DONE "This is faked\n 0 attepmts\nevals: 0\nseed: 1 1\n" ;
@@ -2591,7 +2615,8 @@ start run
 	    cp( $self -> {'models'} -> [$run] -> outputs -> [0] -> full_name, './NM_run' . ($run+1) .'/psn.lst' );
 	  }
 
-	  if ( -e "./NM_run" . ($run+1) . "/done" and not $self -> {'rerun'} > 0 ) {
+	  if ( -e $self -> {'models'} -> [$run] -> outputs -> [0] -> full_name and 
+	       -e "./NM_run" . ($run+1) . "/done" and not $self -> {'rerun'} > 0 ) {
 
 	    # Should check for tablefiles.
 
@@ -2610,16 +2635,9 @@ start run
 
 	    my $modulus = (($#models+1) <= 10) ? 1 : (($#models+1) / 10)+1;
 
-	    if ( $run % $modulus == 0
-		 or
-		 $run == 0
-		 or
-		 $run == $#models ) {
-	      
-	      ui -> print( category => 'all',
-			   message  => 'D:'.( $run + 1 ).' .. ',
-			   wrap => 0,
-			   newline => 0)
+	    if ( $run % $modulus == 0 or $run == 0 or $run == $#models ) {
+	      ui -> print( category => 'all', wrap => 0, newline => 0,
+			   message  => 'D:'.( $run + 1 ).' .. ' )
 		  unless( $self -> {'parent_threads'} > 1 or $self -> {'verbose'} );
 	    }
 
@@ -2631,7 +2649,27 @@ start run
 	    $self -> print_finish_message( final_model => $final_model,
 					   run => $run );
 	  }
-	  $children[$run] = $pm -> start ( $run ) if ( $threads > 1  and not $done);
+
+	  if( $threads > 1 and not $done ){
+	    if( $run > 1 ){
+	      my $start_sleep = Time::HiRes::time();
+	      my $min_sleep = defined $PsN::config -> {'_'} -> {'min_fork_delay'} ? $PsN::config -> {'_'} -> {'min_fork_delay'} : 1000_000;
+	      my $max_sleep = defined $PsN::config -> {'_'} -> {'max_fork_delay'} ? $PsN::config -> {'_'} -> {'max_fork_delay'} : 60;
+	      if( $min_sleep > $max_sleep * 1000_000 ){
+		$max_sleep = $min_sleep;
+	      }
+
+	      # Dont wait for psn.lst if clean >= 3 it might have been
+	      # removed.
+
+	      while( not( -e 'NM_run'.($run-1).'/psn.lst' ) and not $self -> {'clean'} >= 3
+		     and 
+		     (Time::HiRes::time() - $start_sleep) < $max_sleep ){
+		Time::HiRes::usleep($min_sleep);
+	      }
+	    }
+	    $children[$run] = $pm -> start ( $run );
+	  }
 
 	  # Make sure that each process gets a unique random sequence:
 	  random_set_seed_from_phrase(random_uniform_integer(1,0,10000*$run));
@@ -2648,51 +2686,33 @@ start run
 
 	    my $modulus = (($#models+1) <= 10) ? 1 : (($#models+1) / 10);
 
-	    if ( $run % $modulus == 0
-		 or
-		 $run == 0
-		 or
-		 $run == $#models ) {
+	    if ( $run % $modulus == 0 or $run == 0 or $run == $#models ) {
 
 	      # The unless checks if tool owning the modelfit is
 	      # running more modelfits, in wich case we should be
 	      # silent to avoid confusion. The $done check is made do
 	      # diffrentiate between allready run processes.
-
-	      ui -> print( category => 'all',
-			   message  => 'S:'.($run+1).' .. ',
-			   wrap => 0,
-			   newline => 0)
+	      
+	      ui -> print( category => 'all', wrap => 0, newline => 0,
+			   message  => 'S:'.($run+1).' .. ' )
 		  unless ( $self -> {'parent_threads'} > 1 or $done or $self -> {'verbose'} );
 	    }
 
 	    # Here we skip to the next model if we are running in
-	    # parallel.
-	    
+	    # parallel.	    
 	    next if $threads > 1;
 	  }
+
 	} else {
 	  last;
 	}
 	
 	if ( not $done ) {
-	  my $cutoff_thetas;
-	  # Thetas that may be fixed to zero if the go below cutoff
-	  if ((ref( $self -> {'cutoff_thetas'} ) eq 'ARRAY')) {
-	    if ((ref($self -> {'cutoff_thetas'}->[$run]) eq 'ARRAY')) {
-	      $cutoff_thetas = $self->{'cutoff_thetas'}->[$run];
-	    } else {
-	      $cutoff_thetas = $self->{'cutoff_thetas'};
-	    }
-	  } else {
-	    if (defined ($self->{'cutoff_thetas'})) {
-	      die("cutoff_thetas: ". $self->{'cutoff_thetas'}. " is not of the correct type in modelfit->run\n");
-	    }
-	  }
 
-	  my %options_hash = ( 'handle_rounding_errors' => undef, 	  # <- Handle rounding errors a bit more intelligently
-			       'handle_hessian_npd' => undef,	          # <- Handle hessian not postiv definite a bit more intelligently             
-			       'handle_maxevals' => undef,                # <- Restart after maxeval
+	  my %options_hash = ( 'handle_rounding_errors' => undef, # <- Handle rounding errors a bit more intelligently
+			       'handle_hessian_npd' => undef,	  # <- Handle hessian not postiv definite a bit more intelligently             
+			       'handle_maxevals' => undef,        # <- Restart after maxeval
+			       'cutoff_thetas' => 'ARRAY',
 			       'tweak_inits' => undef,
 			       'retries' => undef,
 			       'picky' => undef,
@@ -2700,8 +2720,23 @@ start run
 			       'nm_version' => undef );
 	  
 	  foreach my $option ( keys %options_hash ) {
-	    $options_hash{$option} = (ref( $self -> {$option} ) eq 'ARRAY') ?
-		$self -> {$option} -> [$run] : $self -> {$option};
+
+	    # This loops allows run specific parameters to be
+	    # specified. We check that the the parameter given is an
+	    # array we take out one element of that array and pass it
+	    # as a run specific parameter. If the option is specified
+	    # as being an "ARRAY" type in the has above, but there are
+	    # no subarrays, we send the entire array as an parameter.
+
+	    if( ref( $self -> {$option} ) eq 'ARRAY' ) {
+	      if( ref( $self -> {$option} -> [$run] ) ne 'ARRAY' and $options_hash{$option} eq 'ARRAY' ) {
+		$options_hash{$option} = $self -> {$option};
+	      } else {
+		$options_hash{$option} = $self -> {$option} -> [$run];
+	      }
+	    } else {
+	      $options_hash{$option} = $self -> {$option};
+	    }
 	  }
 
 	  # This will stop nasty prints from model, output and data
@@ -2716,7 +2751,6 @@ start run
 
 	  ( $tries, $final_model, $evals_ref ) =
 	      $self -> run_nonmem ( %options_hash,
-				    cutoff_thetas		=> $cutoff_thetas,
 				    model			=> $models[$run],
 				    run_no			=> $run );
 
@@ -2724,19 +2758,26 @@ start run
 
 	  ui -> category( $old_category );
 
-	  open( DONE, ">./NM_run". ($run+1) ."/done" );
-	  @seed = random_get_seed;
-	  @evals = @{$evals_ref} if ( defined $evals_ref );
-	  print DONE "This file indicates that PsN has run nonmem",
-	             "and tried its best to get it through using\n",
-	             ($tries+1), " number of attempts\n";
-	  print DONE "evals: @evals\n";
-	  print DONE "seed: @seed\n";
-	  close( DONE );
+	  if( $self -> {'clean'} >= 3 ){
+	    unlink( <$work_dir/*> );
+	    unless( rmdir( $work_dir ) ){debug -> warn( message => "Unable to remove an NM_runX directory: $! ." )};
+	    sleep(2);
+	  } else {
+
+	    unless( $tries > 0 ){
+	      open( DONE, ">./NM_run". ($run+1) ."/done" );
+	      @seed = random_get_seed;
+	      @evals = @{$evals_ref} if ( defined $evals_ref );
+	      print DONE "This file indicates that PsN has run nonmem",
+	      "and tried its best to get it through using\n",
+	      ($tries+1), " number of attempts\n";
+	      print DONE "evals: @evals\n";
+	      print DONE "seed: @seed\n";
+	      close( DONE );
+	    }
+
+	  }
 	  
-	}
-	
-	if ( not $done ) {
 	  $self -> print_finish_message( final_model => $final_model,
 					 run => $run );
 	}
@@ -2794,7 +2835,7 @@ start run
 
       $self -> print_raw_results();
 
-       # {{{ Result section
+      # {{{ Result section
 
 #       my $models_harvest = $self -> harvest_output( accessors => ['filename'],
 # 						    search_models => 1,
@@ -2913,9 +2954,15 @@ start run
 #       @results     = @{$self -> {'results'}};
 
 #       $self -> print_results;
-       # }}}
+      # }}}
 
       chdir( $cwd );
+      if( not $self -> {'top_tool'} and $self -> {'clean'} >= 3 ){
+	my $dir = $self -> {'directory'};
+	unlink( <$dir/*> );
+	rmdir( $dir );
+      }
+
     }
 end run
 
diff --git a/lib/tool/scm_subs.pm b/lib/tool/scm_subs.pm
index c102064..f458ce8 100644
--- a/lib/tool/scm_subs.pm
+++ b/lib/tool/scm_subs.pm
@@ -1080,6 +1080,7 @@ start modelfit_setup
 	      directory      => $self -> {'directory'}.'/orig_modelfit_dir'.$model_number.'/',
 	      models         => [$model],
 	      subtools       => undef,
+	      top_tool       => 0,
 	      parent_tool_id   => $self -> {'tool_id'},
 	      threads        => $mfit_threads,
 	      parent_threads => $own_threads );
@@ -1121,6 +1122,7 @@ start modelfit_setup
  		directory      => $self -> {'directory'}.'/modelfit_dir'.$model_number,
 		parent_threads => $own_threads,
 		parent_tool_id   => $self -> {'tool_id'},
+		top_tool       => 0,
 		%subargs ) );
       }
 end modelfit_setup
@@ -1432,6 +1434,7 @@ start modelfit_analyze
 		   base_criteria_values => [$new_base_crit_val_ref],
 		   parent_tool_id       => $self -> {'tool_id'},
 		   parent_threads       => $own_threads , 
+		   top_tool             => 0,
 		   config_file          => undef,
 		   resulting_model      => undef,
 		   tools                => undef,
diff --git a/lib/tool_subs.pm b/lib/tool_subs.pm
index 3f8fd92..df1f3f7 100644
--- a/lib/tool_subs.pm
+++ b/lib/tool_subs.pm
@@ -839,6 +839,22 @@ start run
 	$self -> post_fork_analyze;
 
 	chdir($return_dir);	
+	if( $self -> {'clean'} >= 3 ){
+	  
+	  my $top_dir = $self -> {'directory'};
+	  foreach my $dir ( <$top_dir/m*> ){
+	    if( $dir =~ /m[012345689]+/ ){
+	      unlink( <$dir/*> );
+	      rmdir( $dir );
+	    }
+	  }
+
+	  if( not $self -> {'top_tool'} ){
+	    my $dir = $self -> {'directory'};
+	    unlink( <$dir/*> );
+	    unless( rmdir( $dir ) );
+	  }
+	}
 #	@results     = @{$self -> {'results'}};
 #	@prepared_models = @{$self -> {'prepared_models'}};
       }
-- 
2.11.4.GIT