From dc8835882ab83925e68b5e5a1ba1dabb03cd84b1 Mon Sep 17 00:00:00 2001 From: pontus_pih Date: Thu, 20 Sep 2007 11:12:46 +0000 Subject: [PATCH] This is a massive update that merges all changes from PsN_2_2_0_patches_serial. It will be tagged 2.2.4-stable --- Makefile | 18 +- R-scripts/bootstrap.R | 26 +- R-scripts/cdd.R | 5 +- R-scripts/llp.R | 8 +- README.txt | 115 ++ bin/bootstrap | 84 +- bin/cdd | 54 +- bin/check_termination | 7 +- bin/execute | 10 +- bin/llp | 28 +- bin/mc_cdd | 10 +- bin/mcs | 10 +- bin/scm | 39 +- bin/setup.pl | 4 +- diagrams/cdd.dia | 8 +- diagrams/data.dia | 43 +- diagrams/debug.dia | 56 +- diagrams/model.dia | 2327 +++++++++++++++++++++++---------- diagrams/modelfit.dia | 287 ++-- diagrams/output.dia | 88 +- diagrams/tool.dia | 138 +- html/bootstrap_docs.php | 604 ++++----- html/buglist.php | 632 +++++---- html/cdd_docs.php | 545 ++++---- html/download.php | 197 ++- html/index.php | 630 +++++---- html/llp_docs.php | 114 +- html/scm_docs.php | 1086 +++++++-------- lib/common_options.pm | 42 +- lib/data_subs.pm | 8 +- lib/debug_subs.pm | 26 +- lib/model/.cvsignore | 6 +- lib/model/cwres_module_subs.pm | 151 ++- lib/model/problem/error_subs.pm | 45 + lib/model/problem/record_subs.pm | 8 +- lib/model/problem_subs.pm | 122 +- lib/model_subs.pm | 78 +- lib/nonmem_subs.pm | 41 +- lib/output/problem/subproblem_subs.pm | 9 +- lib/output/problem_subs.pm | 63 +- lib/output_subs.pm | 20 +- lib/psn.conf | 28 +- lib/tool/.cvsignore | 1 + lib/tool/bootstrap_subs.pm | 7 + lib/tool/cdd_subs.pm | 43 +- lib/tool/llp_subs.pm | 8 + lib/tool/mc_subs.pm | 36 +- lib/tool/modelfit_subs.pm | 738 ++++++----- lib/tool/scm/config_file_subs.pm | 11 +- lib/tool/scm_subs.pm | 6 +- lib/tool/xv_step_subs.pm | 15 +- lib/tool_subs.pm | 14 +- test/lib/scripts_test.pm | 12 +- 53 files changed, 4891 insertions(+), 3820 deletions(-) diff --git a/Makefile b/Makefile index 28574f6..7149eb1 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ HTML_STUBS=_synopsis.php _description.php _options.php _examples.php DOCUMENTS=$(foreach pre,$(BIN),$(foreach suff,$(HTML_STUBS),$(addprefix html/$(pre),$(suff)))) -DIA2CODE=/home/pontus/code/dia2code/dia2code/dia2code +DIA2CODE=dia2code FILLSCRIPT=perl ./bin/fill_diacode.pl DIRPM=libgen/ DIRSUBS=lib/ @@ -61,9 +61,11 @@ LIBFILES=debug.pm \ model/problem/theta.pm \ model/problem/tol.pm \ model/cwres_module.pm \ - model/problem.pm \ + model/mirror_plot_module.pm \ + model/iofv_module.pm \ model/shrinkage_module.pm \ model/nonparametric_module.pm \ + model/problem.pm \ model.pm \ tool.pm \ tool/modelfit.pm \ @@ -117,6 +119,8 @@ RELFILES=$(addprefix PsN-Source/lib/,$(LIBFILES)) \ bin/unwrap_data \ bin/data_stats \ bin/gam42toconf \ + bin/se_of_eta \ + bin/update_inits \ setup.pl \ README.txt ) @@ -200,19 +204,19 @@ libgen/fcon.pm : diagrams/fcon.dia documents: $(DOCUMENTS) $(addprefix html/bootstrap,$(HTML_STUBS)) : bin/bootstrap lib/common_options.pm - $< --help --html + perl $< --help --html $(addprefix html/cdd,$(HTML_STUBS)) : bin/cdd lib/common_options.pm - $< --help --html + perl $< --help --html $(addprefix html/execute,$(HTML_STUBS)) : bin/execute lib/common_options.pm - $< --help --html + perl $< --help --html $(addprefix html/llp,$(HTML_STUBS)) : bin/llp lib/common_options.pm - $< --help --html + perl $< --help --html $(addprefix html/scm,$(HTML_STUBS)) : bin/scm lib/common_options.pm - $< --help --html + perl $< --help --html release: libgen rel_dir $(RELFILES) documents diff --git a/R-scripts/bootstrap.R b/R-scripts/bootstrap.R index cf7d955..a65b725 100644 --- a/R-scripts/bootstrap.R +++ b/R-scripts/bootstrap.R @@ -2,7 +2,7 @@ ## Justin Wilkins ## October 2005 ## Lars Lindbom -## August 2006 +## August 2006, April 2007 ## set basic options ## autogenerated code begins @@ -20,10 +20,13 @@ showmedian <- FALSE # show line for median show95CI <- TRUE # show line for 95 % confidence interval (percentile) showquart <- FALSE # show line for quartiles +excl.id <- c() # exclude samples that have this individual + ## autogenerated code ends -## read file +## read files bootstrap.data <- read.csv("raw_results1.csv", header=T) +incl.ids <- read.csv("included_individuals1.csv", header=F) ## replace underscores for (i in 1:length(names(bootstrap.data))) { @@ -47,12 +50,17 @@ for (i in names(bootstrap.data)) { n <- length(colnames(bootstrap.data)) - index nparams <- length(colnames(bootstrap.data)) -cols <- c() - ## separate out original model fit p1 <- subset(bootstrap.data, bootstrap.data$model != 0) o1 <- subset(bootstrap.data, bootstrap.data$model == 0) +incl.flag <- rep(0,length(rownames(p1))) +for( i in excl.id ) { + incl.flag <- incl.flag + rowSums( incl.ids == i ) +} + +p1 <- p1[(incl.flag==0),] + #names(p1)[2] <- "minimization.successful" #names(p1)[3] <- "covariance.step.successful" #names(p1)[4] <- "covariance.step.warnings" @@ -82,9 +90,8 @@ for (i in index:nparams) { dp <- density(p1[[i]], na.rm=T) parmlabel <- names(p1)[i] - postscript(file=paste("bootstrap.", parmlabel, ".ps", sep=""), paper="a4", - title=paste("Bootstrap results - ", parmlabel, sep=""), - horizontal=T) + pdf(file=paste("bootstrap.", parmlabel, ".pdf", sep=""), paper="special", + title=paste("Bootstrap results - ", parmlabel, sep=""),width=10,height=7 ) qu <- quantile(p1[[i]], c(0.025, 0.975), na.rm=T) @@ -160,9 +167,8 @@ for (i in index:nparams) { if (total == 0) { bspage <- bspage + 1 - postscript(file=paste("bootstrap.page", bspage, ".ps", sep=""), paper="a4", - title="Bootstrap results", - horizontal=T) + pdf(file=paste("bootstrap.page", bspage, ".pdf", sep=""), paper="special", + title="Bootstrap results",width=10,height=7) par(mfrow = c(3,3)) } total <- total + 1 diff --git a/R-scripts/cdd.R b/R-scripts/cdd.R index df277fb..b52b8a7 100644 --- a/R-scripts/cdd.R +++ b/R-scripts/cdd.R @@ -109,9 +109,8 @@ if( markeropt == 3 ) { ## Construct plot -postscript(file="cdd.ps", paper="a4", - title="Case-deletion diagnostics", - horizontal=T) +pdf(file="cdd.pdf", paper="special", + title="Case-deletion diagnostics",width=10,height=7) plot (cdd.pt$cov.ratios, cdd.pt$cook.scores, type="p", diff --git a/R-scripts/llp.R b/R-scripts/llp.R index f0df823..56be20e 100644 --- a/R-scripts/llp.R +++ b/R-scripts/llp.R @@ -200,9 +200,11 @@ for (i in 1:(n*2)) { llp.parabola <- as.data.frame(mdat) - postscript(file=paste("llp.", parmlabel, ".ps", sep=""), paper="a4", - title=paste("Log-likelihood profile - ", parmlabel, sep=""), - horizontal=T) +# postscript(file=paste("llp.", parmlabel, ".ps", sep=""), paper="a4", +# title=paste("Log-likelihood profile - ", parmlabel, sep=""), +# horizontal=T) + pdf(file=paste("llp.", parmlabel, ".pdf", sep=""), paper="special", + title=paste("Log-likelihood profile - ", parmlabel, sep=""),width=10,height=7) plot (llp.parabola$Parameter, llp.parabola$dOFV, type="l", diff --git a/README.txt b/README.txt index 4a57c3b..a086610 100644 --- a/README.txt +++ b/README.txt @@ -10,6 +10,76 @@ All available from CPAN ( www.cpan.org ) Changelog ---------------------------------------------- +Changes from 2.2.3 to 2.2.4 + +Features + + - "--compute_cwres" option renamed to simply "--cwres" + + - Simulated data suitable for creating mirror plots can be generated + by using a new options called --mirror_plots. The option takes a + value which defines the number of simulations that will be + geneated. + + - A new option, --iofv, enables the computation of individual + objective function values. These values are printed to a table file + called iotab*, where * is the same number as is defined for any + sdtab or patab tables. These names correspond to the normal Xpose + table file format. If no sdtab or patab table is found in the + NONMEM control stream, the iotab file is given number 1. + + - msfo resumes. Basic support for automatically rerunning a model + that has the MSFO=msffile option on $ESTIMATION. If the option is + set, the "msffile" exists and the option --msfo_resume is givven, + PsN will remove initial estimates, and add the $MSFI record. This + is very untested and if you have OMEGA blocks, the model will run + but PsN will be unable to read the output file properly. + + - "no_remote_compile" and "no_remote_execution" options has been + removed, as they were quite seldomly used and added a lot of + complexity to parallization code. + + - Added --prepend_model_file_name option which adds the model file + name to output files to prevent them from being overwritten. + + - A new option called "crash_restarts" which controls the number of + times PsN will restart a crashed run. (Simply put, run is + classified as crashed if the output file looks like it is cut off) + + - PsN will now automatically run R-scripts generated for the + bootstrap, cdd and llp if R is installed and configured. + + - When the CONT data item is used - either defined by the user or + automatically through the -wrap_data option - the tables get a + weird format where the real table content is printed on CONT=0 rows + and nonsense data printed on CONT=1 rows. A new option called + -unwrap_table_files reformats the table files and removes the CONT + column as well as all CONT=1 rows. + + - update_inits script that will put final estimates from a NONMEM + output file and put into a given model file. This is quite + experimental. PsN will reformat the model file quite lot and + comments in the file might get lost. Your model file will be copied + with the addition of a ".org" extension, so you wont loose your + model. + +Bug Fixes + + - SGE queue and resource options were ignored. Fixed thanks to Jeroen + Elassaiss and Stefan Verhoeven. + + - LSF Monitor fixed to work with the new serialized structure of PsN, + again thanks to Jeroen and Stefan. + + - CWRES table file number now defaults to 1 if there is no patab or + sdtab to take the number from. + + - Fixed run bugs under Windows 2000. + + - Better support for G77 under windows. + + - Parser for output files improved + Changes from 2.2.2 to 2.2.3 Features @@ -19,8 +89,33 @@ Features perturbed initial estimates. This is now implemented. One raw_results file is now created in each NM_run* catalog. + - The R-scripts for graphical visualization of bootstrap and cdds + have been updated. The format for the graphs is now pdf instead of + postscript. The bootstrap script can now be adjusted to exclude + bootstrap samples that contain copies of a given set of + individuals. + + - When the wrap_data option is used, the secondary columns, i.e. the + data columns that are kept on rows with CONT=1, are now available + in $ERROR. + Bug Fixes + - The calculation of CWRES was previously done on the _last_ run of a + set of retries, not on the _best_ run. This has now been corrected. + + - The restarting of crashed runs was broken. This is now fixed. + + - The asynchronuous job scheduling broke the handling of seed numbers + for random sequences (e.g. perturbation of initial estimates). This + is now fixed. The present version is however not compatible with + older versions and old runs should therefore not be resumed using + this version. + + - wrap_data should now work as it should. The functionality was + broken as a consequence of the move to the asynchronuous job + handling. + - Labels for omegas were not handled correctly for SAME blocks. Among other things, this had the effect that the updating of initial estimates from the original model to the method created models @@ -60,6 +155,26 @@ Bug Fixes comment) as keywords for a block (FIX, SAME, etc). This is now corrected. + - If you "fix" more than one omega or sigma in a block PsN would put + a FIX after each value, which NONMEM doesn't allow. Now PsN will + only put one FIX inside a block. However, if a any part of the + block has been fixed, its likely so that PsN can not(!) unfix + it.(so, one bug fixed, another introduced) + + - If you an initial estimate was added with "_init_attr", the option + "add_if_absent" was true and the last initial estimate was "SAME" + PsN would try to add the estimate after "SAME" which is wrong, now + PsN adds a new record. + + - Fixed a parsing error of the lst file. If simulation without a + SIGMA was performed, the parser failed to read OMEGA initial + estiamtes. + + - The internal method "indexes" would not properly handle "SAME" in + initial estimates. The result was that "update_inits" would skip + initial values, and get them out of order. Fixed. + + Changes from 2.2.2-beta4 to 2.2.2 Features diff --git a/bin/bootstrap b/bin/bootstrap index 3baf371..2c5af8d 100755 --- a/bin/bootstrap +++ b/bin/bootstrap @@ -25,12 +25,12 @@ Getopt::Long::config("auto_abbrev"); my %options; my %required_options = (); -my %optional_options = ("bca"=>'', - "mplots"=>'', - "rplots"=>'', - "samples:i"=>'', +my %optional_options = ("samples:i"=>'', "sample_size:s"=>'', "stratify_on:s"=>'', + "bca"=>'', + "mplots"=>'', + "rplots"=>'', "skip_minimization_terminated"=>'', "skip_covariance_step_terminated"=>'', "skip_with_covstep_warnings"=>'', @@ -62,7 +62,7 @@ my %help_text; $help_text{Pre_help_message} = <<'EOF';

bootstrap

- Perl script for non-parametric boostrap of NONMEM runs. + Perl script for non-parametric bootstrap of NONMEM runs.

Usage:

EOF @@ -81,7 +81,7 @@ EOF $help_text{Examples} = <<'EOF';

Example:

-

bootstrap -samples=200 run89.mod

+

bootstrap -samples=200 run89.mod

This will run a non-parametric bootstrap of 200 samples and give you good estimates of the standard errors of the parameter @@ -89,7 +89,7 @@ EOF intervals too, but they will generally not be of high quality. -

bootstrap -samples=2000 -bca run89.mod

+

bootstrap -samples=2000 -bca run89.mod

This will run a non-parametric bootstrap using the BCa technique (See An introduction to the bootstrap, Efron, 1993). The BCa is @@ -97,7 +97,7 @@ EOF intervals. -

bootstrap -samples=2000 -bca -stratify_on=5 run89.mod

+

bootstrap -samples=2000 -bca -stratify_on=5 run89.mod

This is the same BCa approach as above but with stratification on the factors of column five. @@ -108,21 +108,21 @@ EOF The options are given here in their long form. Any option may be abbreviated to any nonconflicting prefix. The -threads option - may be abbreviated to -t (or even -thr) but -debug may not be - abbreviated to -d because it conflicts with -debug_packages and - -debug_subroutines. + may be abbreviated to -t (or even -thr) but -debug may not be + abbreviated to -d because it conflicts with -debug_packages and + -debug_subroutines. The following options are valid: EOF $help_text{-samples} = <<'EOF'; -

-samples

+

-samples

The number of bootstrap samples. EOF $help_text{-sample_size} = <<'EOF'; -

-sample_size

+

-sample_size

The number of subjects in each bootstrap data set. The default value is set to the number of individuals in the original data @@ -152,9 +152,9 @@ EOF EOF $help_text{-bca} = <<'EOF'; -

-bca

+

-bca

- Using the -bca option, the bootstrap + Using the -bca option, the bootstrap utility will calculate the confidence intervals through the BCa method. The default approach however, is not to use the BCa (see Efron B, An introduction to the Bootstrap, 1993). The BCa is @@ -163,7 +163,7 @@ EOF EOF $help_text{-stratify_on} = <<'EOF'; -

-stratify_on=integer|string

+

-stratify_on=integer|string

It may be necessary to use stratification in the resampling procedure. For example, if the original data consists of two @@ -173,7 +173,7 @@ EOF procedure to resample within the two groups, producing bootstrap data sets that all contain 10 rich + 90 sparse data patients but with different compositions. The default is not to use - stratification. Set -stratify_on to + stratification. Set -stratify_on to the column that defines the two groups. If a string is used with stratify_on the header in the datafile is used to map the string to a column number. @@ -183,28 +183,28 @@ EOF EOF $help_text{-skip_covariance_step_terminated} = <<'EOF'; -

-skip_covariance_step_terminated

+

-skip_covariance_step_terminated

With this option enabled, the bootstrap will skip all samples where the NONMEM run terminated the covariance step. EOF $help_text{-skip_with_covstep_warnings} = <<'EOF'; -

-skip_with_covstep_warnings

+

-skip_with_covstep_warnings

With this option enabled, the bootstrap will skip all samples where the NONMEM run had warnings from the covariance step. EOF $help_text{-skip_minimization_terminated} = <<'EOF'; -

-skip_minimization_terminated

+

-skip_minimization_terminated

With this option enabled, the bootstrap will skip all samples where the NONMEM run terminated the minimization step. EOF $help_text{-skip_estimate_near_boundary} = <<'EOF'; -

-skip_estimate_near_boundary

+

-skip_estimate_near_boundary

With this option enabled, the bootstrap will skip all samples where the NONMEM run signal that some estimates are near its @@ -212,57 +212,57 @@ EOF EOF $help_text{-estimate_near_boundary_limit} = <<'EOF'; -

-estimate_near_boundary_limit='number'

+

-estimate_near_boundary_limit='number'

- If the -summarize or -summary options are set, the bootstrap + If the -summarize or -summary options are set, the bootstrap will do a set of diagnostics checks. Among other things it checks the ratio between runs with estimates near their boundaries and those without. If the ratio is to high ( by default 20% ) a warning is printed to the screen. You can change - the ratio with -estimate_near_boundary_limit. + the ratio with -estimate_near_boundary_limit. EOF $help_text{-covariance_step_successful_limit} = <<'EOF'; -

-covariance_step_successful_limit='number'

+

-covariance_step_successful_limit='number'

- If the -summarize or -summary options are set, the bootstrap + If the -summarize or -summary options are set, the bootstrap will do a set of diagnostics checks. Among other things it checks the ratio between runs with covariance step succesful and those without. If the ratio is to high ( by default 80% ) a warning is printed to the screen. You can change the ratio with - -covariance_step_successful_limit. + -covariance_step_successful_limit. EOF $help_text{-covariance_step_warnings_limit} = <<'EOF'; -

-covariance_step_warnings_limit='number'

+

-covariance_step_warnings_limit='number'

- If the -summarize or -summary options are set, the bootstrap + If the -summarize or -summary options are set, the bootstrap will do a set of diagnostics checks. Among other things it checks the ratio between runs with warnings in the covariance step and those without. If the ratio is to high (by default 20%) a warning is printed to the screen. You can change the ratio - with -covariance_step_warnings_limit. + with -covariance_step_warnings_limit. EOF $help_text{-minimization_successful_limit} = <<'EOF'; -

-minimization_successful_limit='number'

+

-minimization_successful_limit='number'

- If the -summarize or -summary options are set, the bootstrap + If the -summarize or -summary options are set, the bootstrap will do a set of diagnostics checks. Among other things it checks the ratio between runs with successful minimizations and those without. If the ratio is to high (by default 80%) a warning is printed to the screen. You can change the ratio with - -minimization_successful_limit. + -minimization_successful_limit. EOF $help_text{-mplots} = <<'EOF'; -

-mplots

+

-mplots

Generate matlab scripts for making various plots of the result. EOF $help_text{-rplots} = <<'EOF'; -

-rplots

+

-rplots

Generate R scripts for making various plots of the result. EOF @@ -272,10 +272,6 @@ EOF common_options::online_help('bootstrap',\%options, \%help_text, \%required_options, \%optional_options); -debug -> level( $options{'debug'} ); -debug -> package( $options{'debug_package'} ); -debug -> subroutine( $options{'debug_subroutine'} ); -debug -> warn_with_trace( $options{'warn_with_trace'} ); ## Check that we do have a model file if ( scalar(@ARGV) < 1 ) { @@ -284,13 +280,18 @@ if ( scalar(@ARGV) < 1 ) { } if( scalar(@ARGV) > 1 ){ - print "Bootstrap can only handle one modelfile. Use 'boostrap -h' for help.\n"; + print "Bootstrap can only handle one modelfile. Use 'bootstrap -h' for help.\n"; exit; } 'ui' -> category( 'bootstrap' ); 'ui' -> silent(1) if $options{'silent'}; +debug -> level( $options{'debug'} ); +debug -> package( $options{'debug_package'} ); +debug -> subroutine( $options{'debug_subroutine'} ); +debug -> warn_with_trace( $options{'warn_with_trace'} ); + my $eval_string = common_options::model_parameters(\%options); my $model = model -> new ( eval( $eval_string ), @@ -318,6 +319,7 @@ if( $@ ) { my $bs = tool::bootstrap -> new ( eval( $common_options::parameters ), top_tool => 1, + prepend_model_file_name => 1, models => [ $model ], samples => $options{'samples'}, subjects => \%subj_hash, @@ -364,4 +366,4 @@ if( $options{'rplots'} ){ } ui -> print( category => 'bootstrap', - message => "Boostrap done."); + message => "Bootstrap done."); diff --git a/bin/cdd b/bin/cdd index 736e71d..c70dd56 100755 --- a/bin/cdd +++ b/bin/cdd @@ -24,7 +24,7 @@ my %options; my %required_options = ( 'case_column:s' => 'column_name|column_number'); my %optional_options = ( "bins:i" => '', - "xv!" => '-xv|-noxv', + "xv" => '', "rplots" => '', "selection_method:s" => '\'random\'|\'consecutive\'', "outside_n_sd_check:f" => '' ); @@ -41,17 +41,12 @@ common_options::get_defaults( \%options, 'cdd' ); my %help_text; $help_text{Pre_help_message} = <<'EOF'; - cdd - - Perl script for Case-Deletion Diagnostics of NONMEM runs. - - Usage: EOF $help_text{Description} = <<'EOF'; - Description: +

Description

- The Case Deletion Diagnostics tool is run from the command line + The Case Deletion Diagnostics tool is run using the command from the command line with a few mandatory arguments. CDD is run as a diagnostic after a model is regarded finished or at least mature enough to run validation tool on. You need to specify the NONMEM modelfile @@ -61,7 +56,7 @@ $help_text{Description} = <<'EOF'; EOF $help_text{Examples} = <<'EOF'; - Example: +

Examples

cdd -model=run89.mod -case_column=10 @@ -75,8 +70,8 @@ $help_text{Examples} = <<'EOF'; 2 individuals from center 2 and so on. EOF -$help_text{Options} = <<'EOF'; - Options: +$help_text{Options} = <<'EOF'; +

Options

The options are given here in their long form. Any option may be abbreviated to any nonconflicting prefix. The -threads option @@ -88,19 +83,19 @@ $help_text{Options} = <<'EOF'; EOF $help_text{-h} = <<'EOF'; - -h | -? +

-h | -?

With -h or -? cdd will print a list of options and exit. EOF $help_text{-help} = <<'EOF'; - -help +

-help

With -help cdd will print this, longer, help message. EOF $help_text{-bins} = <<'EOF'; - -bins=$number +

-bins=$number

Sets the number of databins, or cdd datasets, to use. If the number of unique values, or factors, in the based_on column is @@ -112,7 +107,7 @@ $help_text{-bins} = <<'EOF'; EOF $help_text{-selection_method} = <<'EOF'; - -selection_method='random' or 'consecutive' +

-selection_method='random' or 'consecutive'

Specifies whether the factors selected for exclusion should be drawn randomly or consecutively from the datafile. @@ -120,26 +115,34 @@ $help_text{-selection_method} = <<'EOF'; EOF $help_text{-case_column} = <<'EOF'; - -case_column=column_name|column_number +

-case_column=column_name|column_number

+ + This flag is mandatory. Through this flag you set the column on + which the case-deletion is done. You can either use the name of + the column as specified in the $INPUT record in the model file + or you can use the column number. + EOF - $help_text{-rplots} = <<'EOF'; -

-rplots

+$help_text{-rplots} = <<'EOF'; +

-rplots

Generate R scripts for making various plots of the result. EOF +$help_text{-xv} = <<'EOF'; +

-xv|-noxv

+ + Turns the cross-validation feature on (-xv) or off (-noxv). The + default behavior is to run the cross-validation step. +EOF + $help_text{Post_help_message} = <<'EOF'; Also see 'execute -help' for a description of common options. EOF common_options::online_help( 'cdd', \%options, \%help_text, \%required_options, \%optional_options); -debug -> level( $options{'debug'} ); -debug -> package( $options{'debug_package'} ); -debug -> subroutine( $options{'debug_subroutine'} ); -debug -> warn_with_trace( $options{'warn_with_trace'} ); - ## Check that we do have a model file if ( scalar(@ARGV) < 1 ) { print "A model file must be specified. Use 'cdd -h' for help.\n"; @@ -159,6 +162,11 @@ unless ( defined $options{'case_column'} ){ ui -> category( 'cdd' ); ui -> silent(1) if( $options{'silent'} ); +debug -> level( $options{'debug'} ); +debug -> package( $options{'debug_package'} ); +debug -> subroutine( $options{'debug_subroutine'} ); +debug -> warn_with_trace( $options{'warn_with_trace'} ); + my $eval_string = common_options::model_parameters(\%options); my $model = model -> new ( eval( $eval_string ), diff --git a/bin/check_termination b/bin/check_termination index 7e87f03..093b3d2 100755 --- a/bin/check_termination +++ b/bin/check_termination @@ -20,9 +20,9 @@ $Data::Dumper::Maxdepth=$ARGV[1]; # Model evaluation. Part one. Version 0.1 -# This script checks a nonmem output file. The parts +# This script checks a nonmem output file for flaws. The parts # concerning parts of the covariance step obviously needs the $COV -# record. In addition, the condition number test needs the PRINT=E +# record. In addition, the conditioning number test needs the PRINT=E # option of the $COV record. The limits for the relative tests may be # changed below: @@ -188,8 +188,7 @@ for ( my $i = 0; $i < scalar @{$output -> problems}; $i++ ) { defined $eigens -> [$i][$j] and scalar @{$eigens -> [$i][$j]} > 0 ) { if ( $output -> condition_number -> [$i][$j] < $condition_number_limit ) { - acknowledge( 'Condition number ('. - sprintf("%6.1f",$output -> condition_number -> [$i][$j]).')', 'OK', $log ); + acknowledge( 'Condition number ', 'OK', $log ); } else { acknowledge( 'Large condition number ('. sprintf("%6.0f",$output -> condition_number -> [$i][$j]).')', 'WARNING', $log ); diff --git a/bin/execute b/bin/execute index 328ec05..6a37a27 100755 --- a/bin/execute +++ b/bin/execute @@ -37,11 +37,6 @@ common_options::set_globals( \%options ); common_options::get_defaults( \%options, 'execute' ); common_options::online_help('execute', \%options, undef,{},{}); -debug -> level( $options{'debug'} ); -debug -> package( $options{'debug_package'} ); -debug -> subroutine( $options{'debug_subroutine'} ); -debug -> warn_with_trace( $options{'warn_with_trace'} ); - my @outputfiles; my $fake; if( $options{'outputfile'} ){ @@ -61,6 +56,11 @@ ui -> category( 'modelfit' ); random_set_seed_from_phrase( $options{'seed'} ) if ( defined $options{'seed'} ); +debug -> level( $options{'debug'} ); +debug -> package( $options{'debug_package'} ); +debug -> subroutine( $options{'debug_subroutine'} ); +debug -> warn_with_trace( $options{'warn_with_trace'} ); + my $models_array; my $eval_string = common_options::model_parameters(\%options); diff --git a/bin/llp b/bin/llp index 16ff0ee..9b2583b 100755 --- a/bin/llp +++ b/bin/llp @@ -80,14 +80,14 @@ EOF $help_text{Examples} = <<'EOF';

Example:

-

llp -model=run89.mod -thetas='1,2'

+

llp run89.mod -thetas='1,2'

This will make the llp tool try to estimate the confidence intervals for thetas one and two of the model in run89.mod. It will base the first guesses on the standard error estimates from run89.lst. -

llp -model=run89.mod -thetas='1,2' -rse_thetas='20,30'

+

llp run89.mod -thetas='1,2' -rse_thetas='20,30'

In this example, we explicitly specify the relative standard errors which is necessary if we do not have an output file with @@ -118,15 +118,8 @@ $help_text{-help} = <<'EOF'; With -help llp will print this, longer, help message. EOF -$help_text{-model} = <<'EOF'; -

-model='filename'

- - The name of the model file. May be specified with or without the - "'". -EOF - $help_text{-outputfile} = <<'EOF'; -

-outputfile='filename'

+

-outputfile=filename

The name of the NONMEM output file. The default value is the name of the model file with a '.mod' substituted with @@ -165,7 +158,10 @@ EOF

-significant_digits=integer

Specifies the number of significant digits that is required for - the test of the increase in objective function value. + the test of the increase in objective function value. The + default is three, which means that the method will stop once the + difference in objective function value is between 3.835 and + 3.845 if -ofv_increase is set to 3.84 (default). EOF $help_text{-normq} = <<'EOF'; @@ -235,11 +231,6 @@ EOF common_options::online_help( 'llp', \%options, \%help_text, \%required_options, \%optional_options); -debug -> level( $options{'debug'} ); -debug -> package( $options{'debug_package'} ); -debug -> subroutine( $options{'debug_subroutine'} ); -debug -> warn_with_trace( $options{'warn_with_trace'} ); - ## Check that we do have a model file if ( scalar(@ARGV) < 1 ) { print "At least on model file must be specified. Use 'llp -h' for help.\n"; @@ -259,6 +250,11 @@ unless( $options{'thetas'} or $options{'omegas'} or $options{'sigmas'}){ ui -> category( 'llp' ); ui -> silent(1) if( $options{'silent'} ); +debug -> level( $options{'debug'} ); +debug -> package( $options{'debug_package'} ); +debug -> subroutine( $options{'debug_subroutine'} ); +debug -> warn_with_trace( $options{'warn_with_trace'} ); + my @thetas = split( ',',$options{'thetas'} ); my @omegas = split( ',',$options{'omegas'} ); my @sigmas = split( ',',$options{'sigmas'} ); diff --git a/bin/mc_cdd b/bin/mc_cdd index c66ccf3..6e08033 100644 --- a/bin/mc_cdd +++ b/bin/mc_cdd @@ -125,11 +125,6 @@ EOF common_options::online_help( 'mc_cdd', \%options, \%help_text, \%required_options, \%optional_options); -debug -> level( $options{'debug'} ); -debug -> package( $options{'debug_package'} ); -debug -> subroutine( $options{'debug_subroutine'} ); -debug -> warn_with_trace( $options{'warn_with_trace'} ); - ## Check that we do have a model file if ( scalar(@ARGV) < 1 ) { print "A model file must be specified. Use 'mc_cdd -h' for help.\n"; @@ -159,6 +154,11 @@ unless ( defined $options{'samples'} ){ ui -> category( 'mc_cdd' ); ui -> silent(1) if( $options{'silent'} ); +debug -> level( $options{'debug'} ); +debug -> package( $options{'debug_package'} ); +debug -> subroutine( $options{'debug_subroutine'} ); +debug -> warn_with_trace( $options{'warn_with_trace'} ); + my $eval_string = common_options::model_parameters(\%options); my $model = model -> new ( eval( $eval_string ), diff --git a/bin/mcs b/bin/mcs index 26fd77c..6e318e9 100644 --- a/bin/mcs +++ b/bin/mcs @@ -125,11 +125,6 @@ EOF common_options::online_help( 'mc', \%options, \%help_text, \%required_options, \%optional_options); -debug -> level( $options{'debug'} ); -debug -> package( $options{'debug_package'} ); -debug -> subroutine( $options{'debug_subroutine'} ); -debug -> warn_with_trace( $options{'warn_with_trace'} ); - ## Check that we do have a model file if ( scalar(@ARGV) < 1 ) { print "A model file must be specified. Use 'mc -h' for help.\n"; @@ -154,6 +149,11 @@ unless ( defined $options{'samples'} ){ ui -> category( 'mc' ); ui -> silent(1) if( $options{'silent'} ); +debug -> level( $options{'debug'} ); +debug -> package( $options{'debug_package'} ); +debug -> subroutine( $options{'debug_subroutine'} ); +debug -> warn_with_trace( $options{'warn_with_trace'} ); + my $eval_string = common_options::model_parameters(\%options); my $model = model -> new ( eval( $eval_string ), diff --git a/bin/scm b/bin/scm index 38e76f6..6fd03c1 100755 --- a/bin/scm +++ b/bin/scm @@ -55,6 +55,11 @@ $help_text{Pre_help_message} = <<'EOF'; EOF $help_text{Description} = <<'EOF'; +

Description:

The Stepwise Covariate Model (SCM) building tool of PsN implements @@ -69,19 +74,17 @@ EOF followed by Backward Elimination, which proceeds as the Forward Selection but reversely, using stricter criteria for model improvement. -

- The Stepwise Covariate Model building procedure is run by the scm - utility. The options to the scm utility are of three diffrent - types, first are the common options which works like the common - options for all tools. Then are the SCM specific command line - options. And last are options which have a more complex structures - and are considered to cumbersome to specify on the command - line. These options must therefore be specified in a separate - configuration file. It is possible to provide values for all - options, including the common options, in the configuration file - but it should be noted that the command line overrides the - configuration file, if an option is specified at both - locations. + +

+ + The Stepwise Covariate Model building procedure is run by the PsN + tool scm. The options to scm can (and should) be rather complex to + describe all features of a covariate model building procedure. To + make it easier for the user, a configuration file should be + written for each scm run. The options can be specified in this + file instead of the command line. + @@ -200,11 +203,6 @@ EOF common_options::online_help( 'scm', \%options, \%help_text, \%required_options, \%optional_options); -debug -> level( $options{'debug'} ); -debug -> package( $options{'debug_package'} ); -debug -> subroutine( $options{'debug_subroutine'} ); -debug -> warn_with_trace( $options{'warn_with_trace'} ); - if ( $options{'config_file'} eq '' ){ print "Please specify a config file \n"; exit; @@ -213,6 +211,11 @@ if ( $options{'config_file'} eq '' ){ 'ui' -> category( 'scm' ); 'ui' -> silent(1) if( $options{'silent'} ); +debug -> level( $options{'debug'} ); +debug -> package( $options{'debug_package'} ); +debug -> subroutine( $options{'debug_subroutine'} ); +debug -> warn_with_trace( $options{'warn_with_trace'} ); + my $config_file; if( -e $options{'config_file'} ){ my $file = file -> new( name => $options{'config_file'}, path => '.' ); diff --git a/bin/setup.pl b/bin/setup.pl index b94d5c6..16bbd9b 100644 --- a/bin/setup.pl +++ b/bin/setup.pl @@ -3,14 +3,14 @@ use Config; use CPAN; use File::Spec; -my $version = '2.2.2'; +my $version = '2.2.4'; my $name_safe_version = $version; $name_safe_version =~ s/\./_/g; my @utilities = ('bootstrap', 'cdd', 'execute', 'llp', 'scm', 'sumo', 'mcs', 'mc_cdd', 'data_stats', 'create_extra_data_model', 'single_valued_columns', 'gam42toconf', 'create_cont_model', 'create_cont_data', 'unwrap_data', 'create_subsets', - 'check_termination' ); + 'check_termination','se_of_eta','update_inits' ); my @win_modules = ('Math::Random'); my @nix_modules = ('Math::Random','Storable'); diff --git a/diagrams/cdd.dia b/diagrams/cdd.dia index 7e52709..e108c17 100644 --- a/diagrams/cdd.dia +++ b/diagrams/cdd.dia @@ -346,7 +346,7 @@ - + @@ -1840,7 +1840,7 @@ - #register_in_database# + #register_cdd_in_database# ## @@ -2270,13 +2270,13 @@ - + - + diff --git a/diagrams/data.dia b/diagrams/data.dia index b8d7674..05c9a8f 100644 --- a/diagrams/data.dia +++ b/diagrams/data.dia @@ -85,9 +85,6 @@ - - - ## @@ -100,13 +97,13 @@ - + - + @@ -144,12 +141,6 @@ - - - - - - @@ -166,7 +157,7 @@ - + @@ -175,9 +166,9 @@ - + - + @@ -4370,13 +4361,13 @@ - + - + @@ -4414,12 +4405,6 @@ - - - - - - @@ -4436,7 +4421,7 @@ - + @@ -4445,9 +4430,9 @@ - + - + @@ -5268,15 +5253,15 @@ - + - + - + - + diff --git a/diagrams/debug.dia b/diagrams/debug.dia index 79d0031..e80af8b 100644 --- a/diagrams/debug.dia +++ b/diagrams/debug.dia @@ -68,16 +68,16 @@ - + - + - + #debug# @@ -112,12 +112,6 @@ - - - - - - @@ -131,21 +125,21 @@ - + - + - + - + - + - + @@ -172,7 +166,7 @@ #scalar integer# - #1# + #0# ## @@ -727,36 +721,6 @@ - - - #psn_in_inc# - - - ## - - - ## - - - - - - ## - - - - - - - - - - - - - - - diff --git a/diagrams/model.dia b/diagrams/model.dia index 928e324..bdd6445 100644 --- a/diagrams/model.dia +++ b/diagrams/model.dia @@ -10,7 +10,7 @@ - #A3# + #A4# @@ -25,7 +25,7 @@ - + @@ -65,19 +65,19 @@ - + - + - + - + - + @@ -94,19 +94,19 @@ - + - + - + - + - + @@ -123,19 +123,19 @@ - + - + - + - + @@ -152,13 +152,13 @@ - + - + - + @@ -247,7 +247,7 @@ - #compute_cwres# + #cwres# #scalar boolean# @@ -270,6 +270,29 @@ + #mirror_plots# + + + #scalar integer# + + + #0# + + + ## + + + + + + + + + + + + + #directory# @@ -2070,6 +2093,23 @@ + + + #fuzzy_match# + + + #scalar boolean# + + + #0# + + + ## + + + + + @@ -2265,6 +2305,23 @@ + + + #with_correlations# + + + #scalar boolean# + + + #0# + + + ## + + + + + @@ -2313,6 +2370,23 @@ + + + #with_correlations# + + + #scalar boolean# + + + #0# + + + ## + + + + + @@ -2419,13 +2493,13 @@ - + - + - + @@ -2482,7 +2556,7 @@ - + @@ -2491,7 +2565,7 @@ - + @@ -2717,19 +2791,23 @@ - + - + - - - + + + + + + + @@ -2772,22 +2850,22 @@ - + - + - + - - - - - + + + + + @@ -2841,21 +2919,25 @@ - + - + - - - - + + + + + + + + @@ -2898,21 +2980,21 @@ - + - + - + - - - - + + + + @@ -2938,16 +3020,16 @@ - + - + - - - - + + + + @@ -2973,13 +3055,13 @@ - + - + - + @@ -3036,7 +3118,7 @@ - + @@ -3045,7 +3127,7 @@ - + @@ -3147,13 +3229,13 @@ - + - + - + @@ -3210,7 +3292,7 @@ - + @@ -3219,7 +3301,7 @@ - + @@ -3248,13 +3330,13 @@ - + - + - + @@ -3311,7 +3393,7 @@ - + @@ -3320,7 +3402,7 @@ - + @@ -3380,13 +3462,13 @@ - + - + - + @@ -3443,7 +3525,7 @@ - + @@ -3452,7 +3534,7 @@ - + @@ -3481,16 +3563,16 @@ - + - + - - - - + + + + @@ -3516,16 +3598,16 @@ - + - + - - - - + + + + @@ -3551,16 +3633,16 @@ - + - + - - - - + + + + @@ -3586,16 +3668,16 @@ - + - + - - - - + + + + @@ -3621,16 +3703,16 @@ - + - + - + - + @@ -3678,22 +3760,22 @@ - + - + - + - + - + - + @@ -4239,16 +4321,16 @@ - + - + - - - - + + + + @@ -4274,16 +4356,16 @@ - + - + - + - + @@ -4331,22 +4413,22 @@ - + - + - + - + - + - + @@ -4526,16 +4608,16 @@ - + - + - - - - + + + + @@ -4561,19 +4643,19 @@ - + - + - + - + #model# @@ -4679,7 +4761,76 @@ - #compute_cwres# + #cwres# + + + #scalar boolean# + + + #0# + + + ## + + + + + + + + + + + + + + #iofv# + + + #scalar boolean# + + + #0# + + + ## + + + + + + + + + + + + + + #mirror_plots# + + + #scalar integer# + + + #0# + + + ## + + + + + + + + + + + + + + #mirror_from_lst# #scalar boolean# @@ -5317,7 +5468,7 @@ - + @@ -6763,6 +6914,23 @@ + + + #with_correlations# + + + #scalar boolean# + + + #0# + + + ## + + + + + @@ -6876,6 +7044,23 @@ + + + #with_correlations# + + + #scalar boolean# + + + #0# + + + ## + + + + + @@ -8425,6 +8610,23 @@ + + + #fuzzy_match# + + + #scalar boolean# + + + #0# + + + ## + + + + + @@ -10682,6 +10884,23 @@ + + + #fuzzy_match# + + + #scalar boolean# + + + #0# + + + ## + + + + + @@ -10897,6 +11116,23 @@ + + + #fuzzy_match# + + + #scalar boolean# + + + #0# + + + ## + + + + + @@ -11358,13 +11594,13 @@ - + - + - + @@ -11421,7 +11657,7 @@ - + @@ -11430,7 +11666,7 @@ - + @@ -11459,19 +11695,23 @@ - + - + - - - + + + + + + + @@ -11513,19 +11753,19 @@ - - + + - + - + - + @@ -11582,7 +11822,7 @@ - + @@ -11591,7 +11831,7 @@ - + @@ -11620,19 +11860,23 @@ - + - + - - - + + + + + + + @@ -11674,19 +11918,19 @@ - - + + - + - + - + @@ -11743,7 +11987,7 @@ - + @@ -11752,7 +11996,7 @@ - + @@ -11781,16 +12025,16 @@ - + - + - - - - + + + + @@ -11816,16 +12060,16 @@ - + - + - - - - + + + + @@ -11851,17 +12095,17 @@ - + - + - - - - - + + + + + @@ -11915,16 +12159,16 @@ - + - + - - - - + + + + @@ -11950,17 +12194,17 @@ - + - + - - - - - + + + + + @@ -12014,16 +12258,16 @@ - + - + - - - - + + + + @@ -12049,17 +12293,17 @@ - + - + - - - - - + + + + + @@ -12113,16 +12357,16 @@ - + - + - - - - + + + + @@ -12148,17 +12392,17 @@ - + - + - - - - - + + + + + @@ -12212,13 +12456,13 @@ - + - + - + @@ -12275,7 +12519,7 @@ - + @@ -12284,7 +12528,7 @@ - + @@ -12313,16 +12557,16 @@ - + - + - - - - + + + + @@ -12348,17 +12592,17 @@ - + - + - - - - - + + + + + @@ -12412,16 +12656,16 @@ - + - + - - - - + + + + @@ -12447,17 +12691,17 @@ - + - + - - - - - + + + + + @@ -12511,13 +12755,13 @@ - + - + - + @@ -12574,7 +12818,7 @@ - + @@ -12583,7 +12827,7 @@ - + @@ -12603,8 +12847,81 @@ - - + + + + #secondary_columns# + + + #array of arrays# + + + ## + + + ## + + + + + + + + + + + + + + + + #_format_record# + + + ## + + + ## + + + + + + ## + + + + + + + + + + + + + + + + + #formatted# + + + #array of strings# + + + ## + + + ## + + + + + + + + @@ -12612,16 +12929,16 @@ - + - + - - - - + + + + @@ -12647,17 +12964,17 @@ - + - + - - - - - + + + + + @@ -12711,13 +13028,13 @@ - + - + - + @@ -12774,7 +13091,7 @@ - + @@ -12783,7 +13100,7 @@ - + @@ -12812,16 +13129,16 @@ - + - + - - - - + + + + @@ -12847,16 +13164,16 @@ - + - + - - - - + + + + @@ -12882,17 +13199,17 @@ - + - + - - - - - + + + + + @@ -12946,17 +13263,17 @@ - + - + - - - - - + + + + + @@ -13010,17 +13327,17 @@ - + - + - - - - - + + + + + @@ -13074,16 +13391,16 @@ - + - + - - - - + + + + @@ -13109,17 +13426,17 @@ - + - + - - - - - + + + + + @@ -13173,16 +13490,16 @@ - + - + - - - - + + + + @@ -13208,17 +13525,17 @@ - + - + - - - - - + + + + + @@ -13272,16 +13589,16 @@ - + - + - - - - + + + + @@ -13307,17 +13624,17 @@ - + - + - - - - - + + + + + @@ -13371,13 +13688,13 @@ - + - + - + @@ -13434,7 +13751,7 @@ - + @@ -13443,7 +13760,7 @@ - + @@ -13472,16 +13789,16 @@ - + - + - - - - + + + + @@ -13507,17 +13824,17 @@ - + - + - - - - - + + + + + @@ -13571,16 +13888,16 @@ - + - + - - - - + + + + @@ -13606,17 +13923,17 @@ - + - + - - - - - + + + + + @@ -13670,16 +13987,16 @@ - + - + - - - - + + + + @@ -13705,17 +14022,17 @@ - + - + - - - - - + + + + + @@ -13769,13 +14086,13 @@ - + - + - + @@ -13832,7 +14149,7 @@ - + @@ -13841,7 +14158,7 @@ - + @@ -13919,16 +14236,16 @@ - + - + - - - - + + + + @@ -13954,13 +14271,13 @@ - + - + - + @@ -14017,7 +14334,7 @@ - + @@ -14026,7 +14343,7 @@ - + @@ -14055,13 +14372,13 @@ - + - + - + @@ -14118,7 +14435,7 @@ - + @@ -14127,7 +14444,7 @@ - + @@ -14156,13 +14473,13 @@ - + - + - + @@ -14219,7 +14536,7 @@ - + @@ -14228,7 +14545,7 @@ - + @@ -14257,13 +14574,13 @@ - + - + - + @@ -14320,7 +14637,7 @@ - + @@ -14329,7 +14646,7 @@ - + @@ -14358,13 +14675,13 @@ - + - + - + @@ -14421,7 +14738,7 @@ - + @@ -14430,7 +14747,7 @@ - + @@ -14459,16 +14776,16 @@ - + - + - - - - + + + + @@ -14494,17 +14811,17 @@ - + - + - - - - - + + + + + @@ -14558,17 +14875,17 @@ - + - + - - - - - + + + + + @@ -14622,13 +14939,13 @@ - + - + - + @@ -14685,7 +15002,7 @@ - + @@ -14694,7 +15011,7 @@ - + @@ -14723,16 +15040,16 @@ - + - + - - - - + + + + @@ -14758,17 +15075,17 @@ - + - + - - - - - + + + + + @@ -14822,13 +15139,13 @@ - + - + - + @@ -14885,7 +15202,7 @@ - + @@ -14894,7 +15211,7 @@ - + @@ -14947,16 +15264,16 @@ - + - + - - - - + + + + @@ -14982,13 +15299,13 @@ - + - + - + @@ -15045,7 +15362,7 @@ - + @@ -15054,7 +15371,7 @@ - + @@ -15083,16 +15400,16 @@ - + - + - - - - + + + + @@ -15118,13 +15435,13 @@ - + - + - + @@ -15181,7 +15498,7 @@ - + @@ -15190,7 +15507,7 @@ - + @@ -15419,13 +15736,13 @@ - + - + - + @@ -15482,7 +15799,7 @@ - + @@ -15491,7 +15808,7 @@ - + @@ -15520,13 +15837,13 @@ - + - + - + @@ -15583,7 +15900,7 @@ - + @@ -15592,7 +15909,7 @@ - + @@ -15621,13 +15938,13 @@ - + - + - + @@ -15684,7 +16001,7 @@ - + @@ -15693,7 +16010,7 @@ - + @@ -15722,13 +16039,13 @@ - + - + - + @@ -15785,7 +16102,7 @@ - + @@ -15794,7 +16111,7 @@ - + @@ -15870,13 +16187,13 @@ - + - + - + @@ -15933,7 +16250,7 @@ - + @@ -15942,7 +16259,7 @@ - + @@ -15971,13 +16288,13 @@ - + - + - + @@ -16034,7 +16351,7 @@ - + @@ -16043,7 +16360,7 @@ - + @@ -16072,13 +16389,13 @@ - + - + - + @@ -16135,7 +16452,7 @@ - + @@ -16144,7 +16461,7 @@ - + @@ -16173,19 +16490,19 @@ - + - + - + - + - + #record# @@ -16215,7 +16532,7 @@ - + @@ -16236,7 +16553,7 @@ - + @@ -16245,7 +16562,7 @@ - + @@ -16560,6 +16877,23 @@ + + + #fuzzy_match# + + + #scalar boolean# + + + #0# + + + ## + + + + + @@ -16570,17 +16904,17 @@ - + - + - - - - - + + + + + @@ -16634,17 +16968,17 @@ - + - + - - - - - + + + + + @@ -16698,17 +17032,17 @@ - + - + - - - - - + + + + + @@ -16762,17 +17096,17 @@ - + - + - - - - - + + + + + @@ -16826,17 +17160,17 @@ - + - + - - - - - + + + + + @@ -16890,13 +17224,13 @@ - + - + - + @@ -16953,7 +17287,7 @@ - + @@ -16962,7 +17296,7 @@ - + @@ -16991,15 +17325,15 @@ - + - + - - - + + + @@ -17051,16 +17385,16 @@ - + - + - - - - + + + + @@ -17086,13 +17420,13 @@ - + - + - + @@ -17149,7 +17483,7 @@ - + @@ -17158,7 +17492,7 @@ - + @@ -17236,16 +17570,16 @@ - + - + - - - - + + + + @@ -17271,15 +17605,15 @@ - + - + - - - + + + @@ -17326,21 +17660,21 @@ - + - + - + - + - + @@ -17388,22 +17722,22 @@ - + - + - + - + - + - + @@ -17942,16 +18276,16 @@ - + - + - + - + @@ -17999,22 +18333,22 @@ - + - + - + - + - + - + @@ -18044,17 +18378,17 @@ - + - + - - - - - + + + + + @@ -18108,16 +18442,16 @@ - + - + - - - - + + + + @@ -18143,13 +18477,13 @@ - + - + - + @@ -18185,7 +18519,7 @@ - + @@ -18782,16 +19116,16 @@ - + - + - - - - + + + + @@ -18838,25 +19172,25 @@ - - + + - + - + - + - + - + #nonparametric_module# @@ -18886,7 +19220,7 @@ - + @@ -19709,16 +20043,16 @@ - + - + - - - - + + + + @@ -19765,25 +20099,25 @@ - - + + - + - + - + - + - + #cwres_module# @@ -19895,7 +20229,7 @@ #array of strings# - #['cwtab.50','cwtab.51','cwtab.52','cwtab.53','cwtab.54','cwtab.55','cwtab.56','cwtab.57','cwtab.58']# + #['cwtab.est','cwtab']# ## @@ -19935,6 +20269,29 @@ + #mirror_plots# + + + #scalar integer# + + + ## + + + ## + + + + + + + + + + + + + #problem# @@ -20019,16 +20376,620 @@ - + + + + + + + + + + + + + + + + + + + + + ## + + + + + + + + ## + + + ## + + + + + + + + + + + ## + + + ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #mirror_plot_module# + + + ## + + + ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #enabled# + + + #scalar boolean# + + + #0# + + + ## + + + + + + + + + + + + + + #cwres# + + + #scalar boolean# + + + #0# + + + ## + + + + + + + + + + + + + + #mirror_from_lst# + + + #scalar boolean# + + + #0# + + + ## + + + + + + + + + + + + + + #nr_of_mirrors# + + + #scalar integer# + + + ## + + + ## + + + + + + + + + + + + + + #base_model# + + + #mandatory object model# + + + ## + + + ## + + + + + + + + + + + + + + + + #post_process# + + + ## + + + ## + + + + + + ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ## + + + + + + + + ## + + + ## + + + + + + + + + + + ## + + + ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #iofv_module# + + + ## + + + ## + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + #enabled# + + + #scalar boolean# + + + #0# + + + ## + + + + + + + + + + + + + + #base_model# + + + #mandatory object model# + + + #0# + + + ## + + + + + + + + + + + + + + #nm_version# + + + #scalar string# + + + ## + + + ## + + + + + + + + + + + + + + + + #post_process# + + + ## + + + ## + + + + + + ## + + + + + + + + + + + + + + + + + + #post_run_process# + + + ## + + + ## + + + + + + ## + + + + + + + + + + + + + + + + + + + + + + + + - + - - - - + + + + @@ -20075,8 +21036,8 @@ - - + + diff --git a/diagrams/modelfit.dia b/diagrams/modelfit.dia index 925ea92..1d6a07f 100644 --- a/diagrams/modelfit.dia +++ b/diagrams/modelfit.dia @@ -233,7 +233,7 @@ - + @@ -242,7 +242,7 @@ - + #modelfit# @@ -1111,6 +1111,71 @@ + #compute_iofv# + + + ## + + + ## + + + + + + ## + + + + + + + + + + + + + + + + + #run_no# + + + #scalar integer# + + + ## + + + ## + + + + + + + + #queue_info# + + + #mandatory hash of various# + + + ## + + + ## + + + + + + + + + #compute_cwres# @@ -1481,6 +1546,23 @@ + #jobId# + + + #scalar integer# + + + #-1# + + + ## + + + + + + + #model# @@ -2270,7 +2352,7 @@ - #_print_done_file# + #create_sub_dir# ## @@ -2299,44 +2381,10 @@ - #run# - - - #scalar integer# - - - ## - - - ## - - - - - - - - #evals_ref# - - - #ref of array# - - - ## - - - ## - - - - - - - - #tries# + #subDir# - #scalar integer# + #mandatory scalar string# ## @@ -2345,12 +2393,12 @@ ## - + - #work_dir# + #tmp_dir# #scalar string# @@ -2362,14 +2410,14 @@ ## - + - #_read_done_file# + #_get_run_options# ## @@ -2378,7 +2426,7 @@ ## - + ## @@ -2398,7 +2446,7 @@ - #run# + #run_id# #scalar integer# @@ -2413,43 +2461,12 @@ - - - - - #create_sub_dir# - - - ## - - - ## - - - - - - ## - - - - - - - - - - - - - - - #subDir# + #options_hash# - #mandatory scalar string# + #hash of various# ## @@ -2458,12 +2475,12 @@ ## - + - #tmp_dir# + #work_dir# #scalar string# @@ -2475,14 +2492,14 @@ ## - + - #_get_run_options# + #ud_submit# ## @@ -2511,10 +2528,10 @@ - #run_id# + #model# - #scalar integer# + #mandatory object model# ## @@ -2528,10 +2545,10 @@ - #options_hash# + #jobId# - #hash of various# + #scalar integer# ## @@ -2543,12 +2560,43 @@ + + + + + #ud_monitor# + + + ## + + + ## + + + + + + ## + + + + + + + + + + + + + + - #work_dir# + #jobId# - #scalar string# + #scalar integer# ## @@ -2564,7 +2612,7 @@ - #ud_submit# + #ud_retrieve# ## @@ -2593,10 +2641,10 @@ - #model# + #jobId# - #mandatory object model# + #scalar integer# ## @@ -2610,7 +2658,7 @@ - #jobId# + #run_no# #scalar integer# @@ -2622,14 +2670,14 @@ ## - + - #ud_monitor_retrieve# + #ud_retrieve2# ## @@ -2712,7 +2760,7 @@ #scalar integer# - ## + #-1# ## @@ -2805,6 +2853,55 @@ + + + + #lsf_monitor# + + + ## + + + ## + + + + + + ## + + + + + + + + + + + + + + + + + #jobId# + + + #scalar integer# + + + ## + + + ## + + + + + + + diff --git a/diagrams/output.dia b/diagrams/output.dia index 4768373..7cc0ead 100644 --- a/diagrams/output.dia +++ b/diagrams/output.dia @@ -85,9 +85,6 @@ - - - ## @@ -117,9 +114,6 @@ - - - ## @@ -132,16 +126,16 @@ - + - + - + #output# @@ -176,12 +170,6 @@ - - - - - - @@ -209,7 +197,7 @@ - + @@ -8647,16 +8635,16 @@ - + - + - + #problem# @@ -8691,12 +8679,6 @@ - - - - - - @@ -8724,7 +8706,7 @@ - + @@ -10633,19 +10615,19 @@ - + - + - - + + @@ -10676,9 +10658,6 @@ - - - @@ -10693,28 +10672,25 @@ - - - - + - + - + - + - + @@ -10752,12 +10728,6 @@ - - - - - - @@ -10785,7 +10755,7 @@ - + @@ -14108,18 +14078,18 @@ - + - + - + - - - - + + + + @@ -14149,9 +14119,6 @@ - - - @@ -14166,14 +14133,11 @@ - - - - + diff --git a/diagrams/tool.dia b/diagrams/tool.dia index e49c27d..cee65c3 100644 --- a/diagrams/tool.dia +++ b/diagrams/tool.dia @@ -68,7 +68,7 @@ - + @@ -77,7 +77,7 @@ - + #tool# @@ -758,6 +758,29 @@ + #prepend_model_file_name# + + + #scalar boolean# + + + #0# + + + ## + + + + + + + + + + + + + #raw_results# @@ -1607,6 +1630,31 @@ + + + + #unwrap_table_files# + + + #scalar boolean# + + + #0# + + + ## + + + + + + + + + + + + #grid_poll_interval# @@ -1816,6 +1864,29 @@ + #msfo_resume# + + + #scalar boolean# + + + #0# + + + ## + + + + + + + + + + + + + #raw_nonp_results# @@ -2000,7 +2071,7 @@ - #restart_crashes# + #crash_restarts# #scalar integer# @@ -2067,6 +2138,55 @@ + + + + #sge_resource# + + + #scalar string# + + + ## + + + ## + + + + + + + + + + + + + + + #sge_queue# + + + #scalar string# + + + ## + + + ## + + + + + + + + + + + + @@ -3247,14 +3367,14 @@ - + - - + + @@ -3311,14 +3431,14 @@ - + - - + + diff --git a/html/bootstrap_docs.php b/html/bootstrap_docs.php index 725f232..33a1954 100755 --- a/html/bootstrap_docs.php +++ b/html/bootstrap_docs.php @@ -1,381 +1,223 @@ - - - - PsN :: Documentation :: Bootstrap - - - - - - - - - -
-
the Bootstrap
-
-
-
- - -

Output

-

- The Boostrap creates a log file and a result file. Both are by default placed in the bootstrap_dirX directory - and are called boostraplog.csv and bootstrap-results.csv. -

- -
-
- - - - - - - - - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Home
Documentation
Download
Buglist
Mailing List
-
- - - + + + + PsN :: Documentation :: Bootstrap + + + + + + + + + +
+
the Bootstrap
+
+
+
+ + +

Output

+

+ The Bootstrap creates a log file and a result file. Both are by default placed in the bootstrap_dirX directory + and are called boostraplog.csv and bootstrap_results.csv. +

+ +
+
+ + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Home
Documentation
Download
Buglist
Mailing List
+
+ + + diff --git a/html/buglist.php b/html/buglist.php index e2b36f4..b0fdbf5 100755 --- a/html/buglist.php +++ b/html/buglist.php @@ -1,320 +1,312 @@ - - - - PsN :: Bug list - - - - - - - - - -
-
Buglist
-
-
-
-

Bugs

-

-As PsN is a fairly complex piece of software, bugs do emerge. Here is -what you can do if you encounter one. -

-

Submitting -a bug

-

-If PsN crashes, you get a weird error message, have erroneous output -or something that is just plain odd? If you think its a bug, submit it -and it will be investigated. Before you do, make sure it has not -been submitted already. First check the list below, then search the mailing -list archives, and check the SourceForge -bug tracker for PsN. If you don't have time for this, submit the -bug anyway. -

-

-The best way to submit a bug is to send a message to -the PsN-general mailing list, or to one of the authors. Try to -describe, in reasonable detail, what the bug is and what you did when -i appeared. Also your PsN version and name of operating system and -anything that might be helpful. From there we will try to help you -out. Maybe you will also get help from other users on the mailing -list. Do not send model files and data files unless we ask you to. -

-

Bug list

-

-Bugs should generally be fixed as soon as possible. Some bugs, however, -would require rewriting larger parts of the code or risk breaking -behavior in other code bits. Those bugs make the list on this -page. When possible a workaround is given, otherwise be patient, -eventually these bugs will be fixed.

-
-
NorduGrid connection.
-
The NorduGrid connection -is currently broken. This has quite a high priority, and will be dealt -with it soon.
- -
MAXEVAL -option
- -
There is some ambiguity in NONMEM model files. You can specify -the maximum function evaluations as MAXEVAL or in a short -form MAX. In certain cases this confuses PsN. So to be on -the safe side, you should use the short form in the model file. If -that doesn't work, try the long version. If you still have problems, -contact us.
- -
Data file paths
- -
If you specify a path to the data file in your model file and -the path is different from where the model file is, then PsN won't -find the data file. This is because PsN is always looking in the same -directory as where the model file is. -
- -
Microsoft Windows
- -
Differences in the Unix style operating systems and Windows -makes it hard to maintain a consistent behavior between the two, -although we do our best to keep PsN as portable as possible. The most notable difference is that Parallel::Forkmanager, -which is responsible for parallelizing execution, is unable to create -processes under Windows. As a consequence, nothing will be done in -parallel under Windows.
-
-
-

 

-
- - - - - - - - - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Home
Documentation
Download
Buglist
Mailing List
-
- - - + + + + PsN :: Bug list + + + + + + + + + +
+
Buglist
+
+
+
+

Bugs

+

+As PsN is a fairly complex piece of software, bugs do emerge. Here is +what you can do if you encounter one. +

+

Submitting +a bug

+

+If PsN crashes, if you get a weird error message, if you get an erroneous output +or if you experience something that is just plain odd (but still PsN-related... :-) ), what should you do? If you think that it really is a bug, submit it +and it will be investigated. Before you do, make sure it has not +been submitted already. First check the list below, then search the mailing +list archives, and check the SourceForge +bug tracker for PsN. If you don't have time for this, submit the +bug anyway. +

+

+The best way to submit a bug is to send a message to +the PsN-general mailing list, or to one of the authors. Try to +describe, in reasonable detail, what the bug is and what you did when +i appeared. Also, add your PsN version, the name of your operating system (e.g. Windows XP or Mac OS X) and +anything that might be helpful. From there we will try to help you +out. Maybe you will also get help from other users on the mailing +list. Do not send model files and data files unless we ask you to. +

+

Bug list

+

+Our aim is to get bugs fixed as soon as possible. To fix some bugs, however, +would require rewriting larger parts of the code or risk breaking +behavior in other code bits. Those bugs make the list on this +page. When possible, a workaround is given, otherwise be patient, +eventually these bugs will be fixed.

+
+
NorduGrid connection.
+
The NorduGrid connection +is currently broken and will probably stay so for as long as we do not see an increased interest in this particular distributed computing system.
+ +
MAXEVAL +option
+ +
There is some ambiguity in NONMEM model files. You can specify +the maximum function evaluations as MAXEVAL or in a short +form MAX. In certain cases this confuses PsN. So to be on +the safe side, you should use the short form in the model file. If +that doesn't work, try the long version. If you still have problems, +contact us.
+ +
+
Data file paths
+
+ +
If you specify a path to the data file in your model file and +the path is different from where the model file is, then PsN won't +find the data file. This is because PsN is always looking in the same +directory as where the model file is.
+ +
 
+
+
+

 

+
+ + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Home
Documentation
Download
Buglist
Mailing List
+
+ + + diff --git a/html/cdd_docs.php b/html/cdd_docs.php index 83f81e9..d76df84 100755 --- a/html/cdd_docs.php +++ b/html/cdd_docs.php @@ -1,326 +1,219 @@ - - - - PsN :: Documentation :: cdd - - - - - - - - - -
-
Case-deletion Diagnostics (cdd)
-
-
-
-
-

Synopsis

-$ cdd [ -h | -? ] [ --help ]
---case_column='column_name|column_number'
-[ --bins=integer ]
-[ --selection_method=''random'|'consecutive'' ]
-[ --abort_on_fail ]
-[ --clean ]
-[ --compress ]
-[ --cpu_time=integer ]
-[ --debug=0 ]
-[ --debug_package='string' ]
-[ --debug_subroutine='string' ]
-[ --directory='string' ]
-[ --extra_data_file='string' ]
-[ --extra_files='string' ]
-[ --grid_batch_size=integer ]
-[ --missing_data_token='string' ]
-[ --nm_directory='string' ]
-[ --nm_version='string' ]
-[ --outputfile='string' ]
-[ --picky ]
-[ --remove_temp_files ]
-[ --results_file='string' ]
-[ --retries=integer ]
-[ --run_on_nordugrid ]
-[ --seed='string' ]
-[ --threads=integer ]
-[ --tweak_inits ] -

Description

-

The Case-deletion Diagnostics tool is run from the command line -with a few mandatory arguments. It is run as a diagnostic after -a model is regarded finished or at least mature enough for validation. You need to specify the NONMEM model file -with a model that has successful termination. You also have to -specify the number or the name of the column in the data file, on which to -select for deletion. This is done with the case_column option.

-

Options

-

-The options are given here in their long form. Any option may be -abbreviated to any nonconflicting prefix. The -threads option -may be abbreviated to -t (or even -thr) but -debug may not be -abbreviated to -d because it conflicts with -debug_packages and-debug_subroutines.

-

The following options are valid:

-

-h | -?

-

With -h or -? cdd will print a list of options and then exit.

-

-help

-

With -help cdd will print a longer help message.

-

-bins=number

-

Sets the number of data bins or cdd data sets that should be used. If the - number of unique values or factors in the based_on column is - higher than the number of bins, one or more factors will be - deleted in each of the cdd data set. Specifying bins as higher than - the number of factors will have no effect. In that case the bin number will be - set to the number of factors. - The default value is equal to the number of unique values in the based_on column.

-

-selection_method='random' or 'consecutive'

-

Specifies whether the factors selected for exclusion should be - drawn randomly or consecutively from the data file. - Default value = 'consecutive'

-

-case_column=column_name|column_number

-

See the execute documentation for a description of common options.

-

Output

-

The Case-deletion Diagnostics tool creates a log file and a result file. Both are by default placed in the cdd_dirX directory and are called cdd.csv and cdd-results.csv.

-

Example

-

- cdd modelfile.mod -case_column=10 -

-

- This will perform a Case-deletion Diagnostic based on the factors in column number ten. If, - for example, that column holds the names of the seven centers - included in the study, this command will create seven copies of - the data set, each with individuals from one specific - center deleted. Say that the centers are numbered 1 to 7. Then - all individuals from center 1 will be excluded from data set 1, the individuals from center 2 will be excluded from data set 2, and so on. -

-
-
- - - - - - - - - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Home
Documentation
Download
Buglist
Mailing List
-
- - - + + + + PsN :: Documentation :: cdd + + + + + + + + + +
+
Case-deletion Diagnostics (cdd)
+
+
+
+
+ + + +
+
+ + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Home
Documentation
Download
Buglist
Mailing List
+
+ + + diff --git a/html/download.php b/html/download.php index 721fc82..d8eaf76 100755 --- a/html/download.php +++ b/html/download.php @@ -121,7 +121,7 @@ MM_reloadPage(true);

PsN is distributed as a gziped tar ball and zipped file. Both should work -on both Windows and UNIX.

+on Windows and UNIX (Including Linux an OS X).

@@ -154,14 +154,14 @@ on both Windows and UNIX.

- - - + + + - + - + - - - + + + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -192,14 +226,14 @@ on both Windows and UNIX.

- - - + + + - + - + @@ -210,7 +244,7 @@ on both Windows and UNIX.

+ of 2.3.0 might appear here then.

PsN-2.2.1.tar.gz

Mostly bug fixes. See the README.txt for details.

2006-11-07

PsN-2.2.4.tar.gz

Please see the change log or the README.txt file for details.

2007-09-10

PsN-2.2.1.zip

PsN-2.2.4.zip

 

2006-11-07

2007-09-10
@@ -170,16 +170,50 @@ on both Windows and UNIX.

PsN-2.2.0.tar.gz

The new 2.2.0 release

2006-08-09

PsN-2.2.3.tar.gz

Please see the change log or the README.txt file for details.

2007-04-30

PsN-2.2.0.zip

PsN-2.2.3.zip

 

2006-08-09

2007-04-30

PsN-2.2.2.tar.gz

Includes a rather large re-write of the jobs submission. Memory foot print should be much lower. See the README.txt for details.

2007-03-05

PsN-2.2.2.zip

 

2007-03-05

+

PsN-2.2.1.tar.gz

Version 2.2.1

2006-11-07

PsN-2.2.1.zip

 

2006-11-07

PsN-2.2.0.tar.gz

The new 2.2.0 release

2006-08-09

PsN-2.2.0.zip

 

2006-08-09

 

We have begun to release beta tests of the stable releases. In these we will include all bug fixes and feature additions that we think are suitable to add quickly to the stable branch.

PsN-2.2.2-beta1.tar.gz

First beta of the 2.2.2 release.

2006-11-08

PsN-2.2.2-beta4.tar.gz

4:th beta of the 2.2.2 release. Please use the stable release 2.2.3. It is newer than this beta release.

2007-01-29

PsN-2.2.2-beta1.zip

PsN-2.2.2-beta4.zip

 

2006-11-08

2007-01-29

 

There is currently no development version available, but we are planning to implement some interresting features and a release - of 2.3.0 might apear here then.
@@ -240,21 +274,65 @@ be broken.

Compatibility

-

PsN is tested with Perl 5.8.4 and 5.8.6 to 5.8.8 on -Linux. On Windows its tested with ActiveState -ActivePerl 5.6.8.811. To compile NONMEM GNU G77 version 3.3.5 and Compaq's PsN is tested on various platfiorms. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
 Operating system version PerlNONMEMFortran compiler
Microsoft WindowsXPActiveState ActivePerl 5.6.8.811 and 5.8.8.819 5 and 6 Digital Visual + Fortran version 6.5
 2000ActiveState ActivePerl 5.6.8.811 and 5.8.8.819 5 and 6Digital Visual -Fortran version 5.0 have been tested. -

PsN development is done under Linux, therefore the function under -Windows is less tested. + Fortran version 6.5

MacintoshOS X 10.4.7 and above 5.8.6 and 5.8.85 and 6Intel Fortran 9.1
LinuxRed Hat 9, Gentoo  5 and 6GNU g77 2.96 and 3.3
     
+

+

-One aim of PsN is to make it platform independent. So please do try +Our aim is to make PsN platform independent. Please try PsN on other platforms and with other Perl versions and compilers and report your success or failure. -

Installation instructions

Installation is easy and consists of five simple @@ -272,8 +350,8 @@ script

    Run the installation script from within PsN-Source. If you are running windows and have ActiveState - ActivePerl installed you should be able to double click on - setup.pl. Otherwise open a command line window, go to + ActivePerl installed you should be able to double-click on + setup.pl. Otherwise open a command line window (Windows Start->Run, type 'cmd'), go to the PsN-Source directory and type:

    perl setup.pl

    @@ -283,54 +361,49 @@ script

    1. Answer the questions on screen. The default is probably the - best for most users. If you cannot install PsN were the install + best for most users. If you cannot install PsN where the install script suggests and you wish to use PsN in your own perl scripts, you must make sure that the directory where you installed the PsN core and toolkit is in Perl's include - path. For convenience You should also check that the directory - where the utilities are installed is in your search path. -

      - + path. For convenience you should also check that the directory + where the utilities are installed is in your search path.

    2. -

      Edit the configuration file - (PsN-installdir/PsN/psn.conf) to make PsN aware your NONMEM - installation:

      For a more detailed documentation of the psn.conf look here.

      +

      Edit the configuration file psn.conf to make PsN aware your NONMEM + installation. If you accept the default options in the PsN setup script, this file is usually found in C:\Perl\site\lib\PsN_version_number\ (Windows) or /usr/local/lib/perl5/site_perl/perl_version/PsN_version/ or /usr/lib/perl5/site_perl/perl_version/PsN_version/ (Linux and Macintosh).

      For a more detailed documentation of the psn.conf look here.

      -

      [nm_versions] -
      - 5=/export/home/nmv1.1 -
      - 5_big=/export/home/nmv1.1_big
      +

      [nm_versions]
      + default=/export/home/nmvi1.0,6
      +5=/export/home/nmv1.1,5
      +5_big=/export/home/nmv1.1_big,5
      + 6=/export/home/nmvi1.0
      +
      [compiler]
      ; Gnu fortran compiler
      name=g77
      - options=-W -static -O - -

      -
      -

      The header [nm_versions] must always be present. Each -line under the nm_version header corresponds to a NONMEM version. To -the left of the equal sign is a version identifier which is a number -or a text that identifies an installation of NONMEM. This is the -identifier you give to the PsN utilities using the option -nm_version. -To the right is the directory where the version of NONMEM is installed.

      - + options=-W -static -O

      +
      3. +
    3. +

      The header [nm_versions] must always be present. Each + line under the nm_version header corresponds to a NONMEM version. To + the left of the equal sign is an identifier which is a number + or a text that names an installation of NONMEM. This is what you give to the PsN utilities using the option -nm_version. + Next comes the directory where the version of NONMEM is installed. Last is a version number. This is new from PsN version 2.2.2 and is needed for the computation of conditional weighted residuals.

      +

      The [compiler] section is optional but necessary if you have -another compiler than g77. The compiler section has two -lines. One starting with name on the left of the equal sign -which is the compiler executable name. You should include a path if you -don't have the compiler directory in you environment path. The second -line is the options line where you specify compiler options, -such as optimizations. (Note that -lines starting with ; are ignored.)

      - -
      + another compiler than g77. The compiler section has two + lines. One starting with name on the left of the equal sign + which is the compiler executable name. You should include a path if you + don't have the compiler directory in you environment path. The second + line is the options line where you specify compiler options, + such as optimizations. (Note that + lines starting with ; are ignored.)

      +
    4. When the installation is done you can safely remove the PsN-Source diff --git a/html/index.php b/html/index.php index 823206a..e761828 100755 --- a/html/index.php +++ b/html/index.php @@ -1,324 +1,306 @@ - - - - PsN :: Home - - - - - - - - -
      -
      Welcome
      -
      -
      -
      -

      Latest News

      -

      The new version 2.2 is now available for download.

      -Perl-speaks-NONMEM -

      Perl-speaks-NONMEM (PsN) is a collection of Perl modules and programs aiding in the development - of non-linear mixed effect models using NONMEM. The functionality ranges from solutions to simpler - tasks such as parameter estimate extraction from output files, data file sub setting and resampling, - to advanced computer-intensive statistical methods. PsN includes stand-alone tools for the end-user - as well as development libraries for method developers. -

      -

      - Compared to previous versions of PsN 2, the ongoing work on PsN version 2.2 has brought some changes - to the overall structure. PsN is now logically divided into three parts. Note, however, that you still - only download and install one package. -

      -

      -Most of what was included in the 2.0 release of PsN is now included in PsN-Core. PsN-Core is built around NONMEM's model, data and output -files. Through PsN-Core -all parts of these files can be controlled in user-written Perl code.

      -

      -Using the functionality of PsN-Core we have implemented two new components: PsN-Toolkit and PsN-Utilities.

      -

      PsN-Toolkit is a collection - of computer intensive statistical methods for non-linear mixed effect modeling using NONMEM. The toolkit includes Perl modules for Bootstrapping, Jackknifing, Log-likelihood Profiling, Case-deletion Diagnostics and Stepwise Covariate Model building. Using a module from PsN-Toolkit requires some basic - knowledge of Perl and some understanding of the toolkit framework.

      -

      PsN-Utilities is a set of command line programs -that allows the user to use the functionality of PsN-Core and PsN-Toolkit without the need to write Perl code. Apart from programs for all the tools in PsN-Toolkit, PsN-Utilities includes utilities that perform simpler tasks, such as summarizing the output from NONMEM -runs. -

      -

      You can read more about the different parts under Documentation.

      -

      -Perl-speaks-NONMEM is copyright © 2000-2006 by Lars Lindbom -and Niclas -Jonsson. All rights reserved. -

      -

      -PsN is maintained by Pontus -Pihlgren and Lars -Lindbom. -

      -

      The site is designed and implemented -by Justin -Wilkins. -

      -

      -Perl-speaks-NONMEM is licensed under version 2 of the GNU General -Public License as published by the Free Software Foundation. -

      -

      References

      -
        -
      • Lindbom L, Pihlgren P, Jonsson EN.
        - PsN-Toolkit--a collection of computer intensive statistical methods for non-linear mixed effect modeling using NONMEM.
        - Comput Methods Programs Biomed. 2005 Sep;79(3):241-57.
        • -
      • Lindbom L, Ribbing J, Jonsson EN.
        - Perl-speaks-NONMEM (PsN)--a Perl module for NONMEM related programming.
        - Comput Methods Programs Biomed. 2004 Aug;75(2):85-94.
        • -
      -

      Acknowledgments

      -

      -NONMEM® is a registered trademark of GloboMax. The Perl camel -logo is -a registered trademark of O'Reilly Media, Inc. and is used with -permission. All logos and trademarks in this site are property of -their respective owners. -

      - - - - - -
      SourceForgePsN is hosted at SourceForge.
      -

       

      -

       

      -
      -
      - - - - - - - - - -
      - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
      Home
      Documentation
      Download
      Buglist
      Mailing List
      -
      - - - + + + + PsN :: Home + + + + + + + + +
      +
      Welcome
      +
      +
      +
      +

      Latest News

      +

      Version 2.2.4 is available here.

      +

      New features

      +

      - Simulated data suitable for creating mirror plots can be generated + by using a new options called --mirror_plots. The option takes a + value which defines the number of simulations that will be + generated.

      +

      - A new option, --iofv, enables the computation of individual + objective function values. These values are printed to a table file + called iotab*, where * is the same number as is defined for any + sdtab or patab tables. These names correspond to the normal Xpose + table file format. If no sdtab or patab table is found in the + NONMEM control stream, the iotab file is given number 1.

      +

      - A brand new user command update_inits that will put + final estimates from a NONMEM output file and put into a given + model file. This is quite experimental. PsN will reformat the model + file quite a lot and comments in the file might get lost. Your model + file will be copied with the addition of a ".org" extension, so you + wont lose your model.

      +

      - "--compute_cwres" option renamed to simply "--cwres"

      + +

      Bug Fixes

      +

      - As usual a few bugs were found and fixed between 2.2.3 and 2.2.4. Please read the change logs. These are also available in the README.txt file included in the release packege as usual. The most important bug fixes are that PsN now works properly on Windows 2000 and fixes to the Sun Grid Engine interface(Thanks to Jeroen Elassaiss and Stefan Verhoeven).

      +

      Perl-speaks-NONMEM

      +

      Perl-speaks-NONMEM (PsN) is a collection of Perl modules and programs aiding in the development + of non-linear mixed effect models using NONMEM. The functionality ranges from solutions to simpler + tasks such as parameter estimate extraction from output files, data file sub setting and resampling, + to advanced computer-intensive statistical methods. PsN includes stand-alone tools for the end-user + as well as development libraries for method developers. +

      +

      + Compared to previous versions of PsN 2, the ongoing work on PsN version 2.2 has brought some changes + to the overall structure. PsN is now logically divided into three parts. Note, however, that you still + only download and install one package. +

      +

      +Most of what was included in the 2.0 release of PsN is now included in PsN-Core. PsN-Core is built around NONMEM's model, data and output +files. Through PsN-Core +all parts of these files can be controlled in user-written Perl code.

      +

      +Using the functionality of PsN-Core we have implemented two new components: PsN-Toolkit and PsN-Utilities.

      +

      PsN-Toolkit is a collection + of computer intensive statistical methods for non-linear mixed effect modeling using NONMEM. The toolkit includes Perl modules for Bootstrapping, Jackknifing, Log-likelihood Profiling, Case-deletion Diagnostics and Stepwise Covariate Model building. Using a module from PsN-Toolkit requires some basic + knowledge of Perl and some understanding of the toolkit framework.

      +

      PsN-Utilities is a set of command line programs +that allows the user to use the functionality of PsN-Core and PsN-Toolkit without the need to write Perl code. Apart from programs for all the tools in PsN-Toolkit, PsN-Utilities includes utilities that perform simpler tasks, such as summarizing the output from NONMEM +runs. +

      +

      You can read more about the different parts under Documentation.

      +

      +Perl-speaks-NONMEM is copyright © 2000-2005 by Lars Lindbom +and Niclas +Jonsson and 2006-2007 by Lars Lindbom. All rights reserved. +

      +

      +PsN is maintained by Pontus +Pihlgren and Lars +Lindbom. +

      +

      The site is designed and implemented +by Justin +Wilkins. +

      +

      +Perl-speaks-NONMEM is licensed under version 2 of the GNU General +Public License as published by the Free Software Foundation. +

      +

      References

      +
        +
      • Lindbom L, Pihlgren P, Jonsson EN.
        + PsN-Toolkit--a collection of computer intensive statistical methods for non-linear mixed effect modeling using NON +MEM.
        + Comput Methods Programs Biomed. 2005 Sep;79(3):241-57.
        • +
      • Lindbom L, Ribbing J, Jonsson EN.
        + Perl-speaks-NONMEM (PsN)--a Perl module for NONMEM related programming.
        + Comput Methods Programs Biomed. 2004 Aug;75(2):85-94.
        • +
      + +

      Acknowledgments

      +

      +NONMEM® is a registered trademark of GloboMax. The Perl camel +logo is +a registered trademark of O'Reilly Media, Inc. and is used with +permission. All logos and trademarks in this site are property of +their respective owners. +

      + + + + + +
      SourceForgePsN is hosted at SourceForge.
      +

       

      +

       

      +
      +
      + + + + + + + + + +
      + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
      Home
      Documentation
      Download
      Buglist
      Mailing List
      +
      + + + diff --git a/html/llp_docs.php b/html/llp_docs.php index 72c23c9..8e83010 100755 --- a/html/llp_docs.php +++ b/html/llp_docs.php @@ -53,49 +53,9 @@ document[changeImages.arguments[i]].src = changeImages.arguments[i+1]; - + -
      - +

       

      diff --git a/html/scm_docs.php b/html/scm_docs.php index bc83e9b..453c53d 100755 --- a/html/scm_docs.php +++ b/html/scm_docs.php @@ -1,627 +1,459 @@ - - - - PsN :: Documentation :: scm - - - - - - - - - -
      -
      Stepwise -Covariate Modeling (scm)
      -
      -
      -
      -
      - -

      Valid states for the parameter-covariate relations

      - -

      The SCM tool in PsN is written to be flexible in the shape of the -parameter-covariate relations. Currently there is a fixed set of -available forms of relations but there is ongoing development aiming -at full flexibility for the user with regard to relation-shape -definitions.

      - -

      The avaialable shapes or states for the -parameter-covariate relations for continuous covariates are (1) not -included, (2) included as a linear relation and (3) included as a -piece-wise linear relation with two slopes.

      - -

      Categorical covariates are either (1) not included or (2) included with an extra parameter added for each but the most common category.

      -

      The figures, 1, 2 and 3, are used when specifying included relations below.

      - - -

      SCM configuration file options

      - -General considerations - -

      The syntax of the simple options is just the name of the option -followed by the equal sign and a valid value for the option. For -example:

      - -

      retries=5

      -

      This states that a maximum of 5 retries will be used in the executions.

      -Some options accept Perl code: -

      base_criteria_values={ 1 => ofv => -9571.234}

      -Other take comma-separated lists: -

      continuous_covariates=AGE,WT,ALBU,BILI,AST,ALT,ALKP,POT,MAG,CA,NA,TOTP

      - -Some options don't accept a value on the command line. They set the -behavior of the utility simply by being specified or not. In the -configuration file these options need to be followed by 1 or a 0. A 1 -indicates that it is used and a 0 that it is not used. - -

      fix=1

      -You can place a comment anywhere in the file by prefixing the line with a semicolon: -

      ;ask_if_fail=0

      -If an option is long you can break the line with a backslash, just a linebreak will NOT work: -

      categorical_covariates=SEX,ARR,SHD,NYHA,RACE,ALCO,SMOK,BETA,CGLY,LDIU,\
      - TDIU,CANT,ACE,NITR,ACOA,POTA,TWOC,TREA,TWOD,INDU,\
      - TUBS,ALQT,ANTP,ANTD,ANTM,ANTH,ANTB,SULP,ANT1,CLIA,\
      - CLIB,CLIC,CL3,CIME,METF,FENM,PROC,HEP1,HEP2,HEP3

      - -

      Simple options

      - -The simple options are the SCM specific command line options and the -command line options common to all PsN Utilties. - -

      List options

      - -List options are options that take enumerations of values like the -categorical_covariates option: - -

      categorical_covariates=SEX,ARR,SHD,NYHA,RAC

      - -The list options valid in the configuration file are: - -

      categorical_covariates

      - -The categorical_covariates list option is -neccessary for the scm to know which type of NONMEM code to generate -for the parameter covariate relation. - -

      continuous_covariates

      - -The categorical_covariates list option is -neccessary for the scm to know which type of NONMEM code to generate -for the parameter covariate relation. - -

      do_not_drop

      - -If the dataset used with the scm is to wide, scm will drop the data -columns that correponds to covariate which is not used in a particular -step. However, if you have used that covariate in code that is not -affected by the scm, NONMEM will produce an error. Therefore it is -neccessary for you to specify which those covariates are, using the -do_not_drop list option. - -

      extra_files

      - -If you need some extraordinary file to be copied into the NONMEM -execution directory, specify them with the extra_files list option. - -

      Code options

      - -Code options are almost like a Simple option. With the difference that -the right side is evaluted by the perl interpretor. If you don't -understand the difference, it is probably not important. However, it -is available for a reason. The SCM have hooks for custom goodness of -fit functions. Those functions may need some special input data. And -for flexibility, we allow that data to be just about anything, the -side effect is that it becomes to complex for the configuration file -structure. The solution is to allow options that are small Perl -programs. Those options we refer to as code options. - -

      base_criteria_values

      - -The base_criteria_values overrides default -criterias for the goodness of fit function used. The default goodness -of fit function takes its base_criteria_value to be the ofv of the -initial model. If you would like to change the base ofv to 400 you -would write: - -
      base_criteria_values= { 1 => { ofv => 400 } }
      - -

      Composite options

      - -

      The composite options can not be specified on the command line. An -exception is the extra_data_files option -which is valid for all utilities. A composite option section starts -with an option header followed by a set of lines with a left side, an -equal sign and a right side.

      - -
      - -
      extra_data_files
      - -
      The -extra_data_files option is used - to specify files with extra data. As NONMEM only accepts 20 - columns, it is sometimes necessary to split data files. The left - side of each line holds the extra data file name. The right side - is a comma-separated list of the column headers. -

      [extra_data_files]
      - sub.dta=ID,GRP,AGE,SEX,ARR,SHD,NYHA,LVEF,WT,HT,RACE,ALCO,SMOK,BETA
      -

      -
      - -
      test_relations
      - -
      The -test_relations option is used - to specify what parameter-covariate relations the scm utility - will test. The left side of each row is a parameter name and the - right side is a comma separated list of covariates. - -

      [test_relations]
      - CL=AGE,SEX,ARR,SHD,NYHA,WT,RACE,ALCO,SMOK,BETA,\
      - ALBU,BILI,AST,ALT,ALKP,POT,MAG,CA,NA,TOTP, \
      - ANT1,CLIA,CLIB,CLIC,CL3,CIME,METF,PROC

      - V=SEX,WT,ALBU,TOTP,LDIU,TDIU,ACOA,SULP

      -
      - -
      included_relations
      - -
      If you want to start the search with some relations included you - can add them to the included_relations. The left side of each - row holds the name of a parameter and the right side a list of - covariates. For each covariate you can specify a relation - state. The general syntax is COV-X - where COV is you covariate and X is the state. If the state is - omitted it will be set to 2. - -

      [included_relations]
      - CL=TDIU-2,CLIA-2,SEX-2
      - V=WT,ACOA-2,SEX-3,TOTP
      -

      -
      - -
      valid_states
      - -
      You can specify what states you want the SCM to test. By default - it tests three states. 1 - not included, 2 - linear, 3 - - piecewise linear and 4 - exponential parameterization. The - valid states are specified for separately continuous and - categorical covariates. - -

      [valid_states]
      - categorical=1,2
      - continuous=1,2,3

      -
      - -
      code
      - -
      If you are not satisfied with the state specific NONMEM code - generated by the scm, you can overide it using the code - option. It lets you specify code for specific parameters, - specific paramater - covariate relations and even specific - states. Each line under the [code] - header defines code for one parameter - covariate - state - relation. Like this: - -

      [code]
      - CL:WT-1=THETA(1)

      - - This will change the default code for state 1 of the CL - WT - relation. You can also use wildcards to redefine the code for - multiple relations: - -

      [code]
      - CL:*-1=THETA(1)

      - - In this example you will redefine the code for all covaritates - in a relation with CL. - -

      [code]
      - *:*-1=THETA(1)

      - - This example will redefine the code for all relations in state - 1. Notice that you can not use a wildcard for the state. -

      - Generally it is useful to include the parameter and/or the - covariate in the code. So to make it possible when using - wildcards you can use the generic variables PARM and COV which - will be replaced with the parameter and covariate in question - for the current relation. - -

      -
      lower_bounds, upper_bounds and inits
      - -
      The inits and bounds option lets you set initial estimates for - thetas introduced in the code and their corresponding bounds. A - hockeystick relation introduces two new thetas, so to set the - thetas value and bounds for the CL - WT relation you would - write: - -

      [lower_bounds]
      - CL:WT-2=0.1, 0.05

      - -

      [upper_bounds]
      - CL:WT-2=0.3, 0.08

      - -

      [inits]
      - CL:WT-2=0.15,0.07

      - - This would correspond to the following lines in a NONMEM modelfile: - -

      THETA(1) = 0.1,0.15,0.4
      - THETA(2) = 0.05,0.07,0.08

      - - Just like in the code option you can use wildcards. Notice that - if you specify to few or to many values, those you exclude will - be replaced by defaults and the extraneous will be ignored. - -
      - - -
      - -

      -

      Output

      -

      Result file

      -Currently, the logfile is the best place to look for the results from the scm utility. -

       

      -

      -
      -

       

      -
      - - - - - - - - - -
      - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
      Home
      Documentation
      Download
      Buglist
      Mailing List
      -
      - - - + + + + PsN :: Documentation :: scm + + + + + + + + + +
      +
      Stepwise +Covariate Modeling (scm)
      +
      +
      +
      +
      + +

      Valid states for the parameter-covariate relations

      + +

      The SCM tool in PsN is written to be flexible in the shape of the +parameter-covariate relations. Currently there is a fixed set of +available forms of relations but there is ongoing development aiming +at full flexibility for the user with regard to relation-shape +definitions.

      + +

      The avaialable shapes or states for the +parameter-covariate relations for continuous covariates are (1) not +included, (2) included as a linear relation and (3) included as a +piece-wise linear relation with two slopes.

      + +

      Categorical covariates are either (1) not included or (2) included with an extra parameter added for each but the most common category.

      +

      The figures, 1, 2 and 3, are used when specifying included relations below.

      + + +

      SCM configuration file options

      + +General considerations + +

      The syntax of the simple options is just the name of the option +followed by the equal sign and a valid value for the option. For +example:

      + +

      retries=5

      +

      This states that a maximum of 5 retries will be used in the executions.

      +Some options accept Perl code: +

      base_criteria_values={ ofv => -9571.234}

      +Other take comma-separated lists: +

      continuous_covariates=AGE,WT,ALBU,BILI,AST,ALT,ALKP,POT,MAG,CA,NA,TOTP

      + +Some options don't accept a value on the command line. They set the +behavior of the utility simply by being specified or not. In the +configuration file these options need to be followed by 1 or a 0. A 1 +indicates that it is used and a 0 that it is not used. + +

      fix=1

      +You can place a comment anywhere in the file by prefixing the line with a semicolon: +

      ;ask_if_fail=0

      +If an option is long you can break the line with a backslash, just a linebreak will not work: +

      categorical_covariates=SEX,ARR,SHD,NYHA,RACE,ALCO,SMOK,BETA,CGLY,LDIU,\
      + TDIU,CANT,ACE,NITR,ACOA,POTA,TWOC,TREA,TWOD,INDU,\
      + TUBS,ALQT,ANTP,ANTD,ANTM,ANTH,ANTB,SULP,ANT1,CLIA,\
      + CLIB,CLIC,CL3,CIME,METF,FENM,PROC,HEP1,HEP2,HEP3

      + +

      Simple options

      + +The simple options are the SCM specific command line options and the +command line options common to all PsN Utilties. + +

      List options

      + +List options are options that take enumerations of values like the +categorical_covariates option: + +

      categorical_covariates=SEX,ARR,SHD,NYHA,RAC

      + +The list options valid in the configuration file are: + +

      categorical_covariates

      + +The categorical_covariates list option is +neccessary for the scm to know which type of NONMEM code to generate +for the parameter covariate relation. + +

      continuous_covariates

      + +The categorical_covariates list option is +neccessary for the scm to know which type of NONMEM code to generate +for the parameter covariate relation. + +

      do_not_drop

      + +If the dataset used with the scm is to wide, scm will drop the data +columns that correponds to covariate which is not used in a particular +step. However, if you have used that covariate in code that is not +affected by the scm, NONMEM will produce an error. Therefore it is +neccessary for you to specify which those covariates are, using the +do_not_drop list option. + +

      extra_files

      + +If you need some extraordinary file to be copied into the NONMEM +execution directory, specify them with the extra_files list option. + +

      Code options

      + +Code options are almost like a Simple option. With the difference that +the right side is evaluted by the perl interpretor. If you don't +understand the difference, it is probably not important. However, it +is available for a reason. The SCM have hooks for custom goodness of +fit functions. Those functions may need some special input data. And +for flexibility, we allow that data to be just about anything, the +side effect is that it becomes to complex for the configuration file +structure. The solution is to allow options that are small Perl +programs. Those options we refer to as code options. + +

      base_criteria_values

      + +The base_criteria_values overrides default +criterias for the goodness of fit function used. The default goodness +of fit function takes its base_criteria_value to be the ofv of the +initial model. If you would like to change the base ofv to 400 you +would write: + +

      base_criteria_values= { ofv => 400 } }

      +

      Composite options

      + +

      The composite options can not be specified on the command line. An +exception is the extra_data_files option +which is valid for all utilities. A composite option section starts +with an option header followed by a set of lines with a left side, an +equal sign and a right side.

      + +
      + +
      extra_data_files
      + +
      The -extra_data_files option is used + to specify files with extra data. As NONMEM only accepts 20 + columns, it is sometimes necessary to split data files. The left + side of each line holds the extra data file name. The right side + is a comma-separated list of the column headers. +

      [extra_data_files]
      + sub.dta=ID,GRP,AGE,SEX,ARR,SHD,NYHA,LVEF,WT,HT,RACE,ALCO,SMOK,BETA
      +

      +
      + +
      test_relations
      + +
      The -test_relations option is used + to specify what parameter-covariate relations the scm utility + will test. The left side of each row is a parameter name and the + right side is a comma separated list of covariates. + +

      [test_relations]
      + CL=AGE,SEX,ARR,SHD,NYHA,WT,RACE,ALCO,SMOK,BETA,\
      + ALBU,BILI,AST,ALT,ALKP,POT,MAG,CA,NA,TOTP, \
      + ANT1,CLIA,CLIB,CLIC,CL3,CIME,METF,PROC

      + V=SEX,WT,ALBU,TOTP,LDIU,TDIU,ACOA,SULP

      +
      + +
      included_relations
      + +
      If you want to start the search with some relations included you + can add them to the included_relations. The left side of each + row holds the name of a parameter and the right side a list of + covariates. For each covariate you can specify a relation + state. The general syntax is COV-X + where COV is you covariate and X is the state. If the state is + omitted it will be set to 2. + +

      [included_relations]
      + CL=TDIU-2,CLIA-2,SEX-2
      + V=WT,ACOA-2,SEX-3,TOTP
      +

      +
      + +
      valid_states
      + +
      You can specify what states you want the SCM to test. By default + it tests three states. 1 - not included, 2 - linear, 3 - + piecewise linear and 4 - exponential parameterization. The + valid states are specified for separately continuous and + categorical covariates. + +

      [valid_states]
      + categorical=1,2
      + continuous=1,2,3

      +
      + +
      code
      + +
      If you are not satisfied with the state specific NONMEM code + generated by the scm, you can overide it using the code + option. It lets you specify code for specific parameters, + specific paramater - covariate relations and even specific + states. Each line under the [code] + header defines code for one parameter - covariate - state + relation. Like this: + +

      [code]
      + CL:WT-1=THETA(1)

      + + This will change the default code for state 1 of the CL - WT + relation. You can also use wildcards to redefine the code for + multiple relations: + +

      [code]
      + CL:*-1=THETA(1)

      + + In this example you will redefine the code for all covaritates + in a relation with CL. + +

      [code]
      + *:*-1=THETA(1)

      + + This example will redefine the code for all relations in state + 1. Notice that you can not use a wildcard for the state. +

      + Generally it is useful to include the parameter and/or the + covariate in the code. So to make it possible when using + wildcards you can use the generic variables PARM and COV which + will be replaced with the parameter and covariate in question + for the current relation. + +

      +
      lower_bounds, upper_bounds and inits
      + +
      The inits and bounds option lets you set initial estimates for + thetas introduced in the code and their corresponding bounds. A + hockeystick relation introduces two new thetas, so to set the + thetas value and bounds for the CL - WT relation you would + write: + +

      [lower_bounds]
      + CL:WT-2=0.1, 0.05

      + +

      [upper_bounds]
      + CL:WT-2=0.3, 0.08

      + +

      [inits]
      + CL:WT-2=0.15,0.07

      + + This would correspond to the following lines in a NONMEM modelfile: + +

      THETA(1) = 0.1,0.15,0.4
      + THETA(2) = 0.05,0.07,0.08

      + + Just like in the code option you can use wildcards. Notice that + if you specify to few or to many values, those you exclude will + be replaced by defaults and the extraneous will be ignored. + +
      + + +
      + +

      +

      Output

      +

      Result file

      +Currently, the logfile is the best place to look for the results from the scm utility. +

       

      +

      +
      +

       

      +
      + + + + + + + + + +
      + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
      Home
      Documentation
      Download
      Buglist
      Mailing List
      +
      + + + diff --git a/lib/common_options.pm b/lib/common_options.pm index ea7e927..9824bae 100644 --- a/lib/common_options.pm +++ b/lib/common_options.pm @@ -15,9 +15,9 @@ my @tool_options = ( "abort_on_fail", "condition_number_limit:f", "correlation_limit:f", # "cpu_time:i", + "crash_restarts:i", "directory:s", "drop_dropped", - "expected_run_time", # "eigen_values", # "grid_batch_size:i", # "grid_poll_interval:i", @@ -33,6 +33,7 @@ my @tool_options = ( "abort_on_fail", "lsf_ttl:s", "min_retries:i", "missing_data_token:s", + "msfo_resume", "near_bound_sign_digits:i", "near_zero_boundary_limit:f", "nice:i", @@ -50,9 +51,10 @@ my @tool_options = ( "abort_on_fail", "retries:i", # "run_on_nordugrid", "run_on_lsf", - "run_on_ud", - "run_on_umbrella", + "run_on_ud", "run_on_sge", + "sge_resource:s", + "sge_queue:s", "seed:s", "shrinkage", "significant_digits_rerun:f", @@ -60,11 +62,11 @@ my @tool_options = ( "abort_on_fail", "summarize|summary", "threads:i", "tweak_inits:i", - "umbrella_timeout", "ud_native_retrieve", "ud_sleep:i", "verbose!", "wrap_data", + "unwrap_table_files", "near_bound_sign_digits:i", "near_zero_boundary_limit:f", "sign_digits_off_diagonals:i", @@ -80,9 +82,12 @@ my @model_options = ("extra_data_files:s@", "extra_files:s", "extra_output:s", "sde", - "compute_cwres", - "outputfile:s", + "cwres", + "mirror_plots:i", + "iofv", + "mirror_from_lst!", "nm_version:s", + "outputfile:s", ); my @script_options = ( "debug:i", @@ -134,7 +139,7 @@ sub set_globals { print "Using database ".$PsN::config -> {'_'} -> {'project'}."\n"; if ( defined $options{'password'} ) { $PsN::config -> {'_'} -> {'password'} = $options{'password'}; - } elsif( defined $PsN::config -> {'_'} -> {'default_password'} ) { + } elsif( defined $PsN::config -> {'_'} -> {'password'} ) { $PsN::config -> {'_'} -> {'password'} = $PsN::config -> {'_'} -> {'default_password'}; } else { @@ -180,13 +185,13 @@ sub print_help { my $option_help; - $option_help .= "[ -h | -? ] [ --help ]\n\t" . ' ' x (1+length($command)); + $option_help .= "[ -h | -? ] [ --help ]\n" . ' ' x (1+length($command)); my @loop_array; if( $command eq 'execute' ){ @loop_array = sort(@get_opt_strings); } else { - @loop_array = (sort(keys %{$required}), sort(keys %{$optional})); + @loop_array = (keys %{$required}, keys %{$optional}); } foreach my $help( @loop_array ) { @@ -213,7 +218,7 @@ sub print_help { unless( $is_required{$help} ){ $option_help .= " ]"; } - $option_help .= "\n\t".' ' x (1+length($command)); + $option_help .= "\n".' ' x (1+length($command)); } return $option_help; @@ -659,6 +664,7 @@ EOF PsN connects with United Devices Grid MP. With -run_on_ud PsN will submit to the UD grid with parameters defined in the "uduserconf" file. EOF + $help_hash{-retries} = <<'EOF';

      -retries='integer'

      @@ -669,6 +675,15 @@ EOF -retries option is 6. EOF + $help_hash{-crash_restarts} = <<'EOF'; +

      -crash_restarts='integer'

      + + If a NONMEM outputfile is produced but PsN is unable to read it + properly it is assumed that NONMEM crashed, probably due to + something in the operating system, and PsN will start the run + again. But PsN will not consider it a retry and will not change + initial estimates. The default value is 4. +EOF $help_hash{-significant_digits_rerun} = <<'EOF';

      -significant_digits_rerun='number'

      @@ -798,7 +813,10 @@ EOF close( EXAMPLES ); open(SYNOPSIS, '>', 'html/' . $command . '_synopsis.php' ); - print SYNOPSIS $help_hash{Pre_help_message} . "
      \t$command " . common_options::print_help($command,$required_options, $optional_options)."
      " ; + print SYNOPSIS $help_hash{Pre_help_message},"\n"; + print SYNOPSIS "

      Synopsis

      \n"; + print SYNOPSIS "\n"; + print SYNOPSIS "
      $command " . common_options::print_help($command,$required_options, $optional_options)."\n
      \n" ; close( SYNOPSIS ); open(OPTIONS, '>', 'html/' . $command . '_options.php' ); @@ -816,7 +834,7 @@ EOF if( exists $help_hash{'-'.$option}){ $opt_help .= $help_hash{'-'.$option}."\n\n"; } else { - $opt_help .= "

      -$option

      No help available for '$option'

      "; + $opt_help .= "

      -$option

      No help available for '$option'

      "; } } print OPTIONS $help_hash{Options} . $opt_help; diff --git a/lib/data_subs.pm b/lib/data_subs.pm index eebb43c..042ccbe 100644 --- a/lib/data_subs.pm +++ b/lib/data_subs.pm @@ -725,7 +725,12 @@ start count_ind { # Returns the number of individuals in the data set. $self -> synchronize; - $num = scalar @{$self -> {'individuals'}}; + if( defined $self -> individuals() ) { + $num = scalar @{$self -> individuals()}; + } else { + debug -> die( message => "No individuals found in file ". + $self -> filename() ); + } } end count_ind @@ -1088,6 +1093,7 @@ start wrap if ( defined $self -> {'secondary_columns'} ); @primary_columns = @{$self -> {'primary_columns'}} if ( defined $self -> {'primary_columns'} ); + $self -> synced(0); } end wrap # }}} wrap diff --git a/lib/debug_subs.pm b/lib/debug_subs.pm index 8e2a17c..ac31504 100644 --- a/lib/debug_subs.pm +++ b/lib/debug_subs.pm @@ -48,8 +48,6 @@ start include statements use Carp; use Carp qw(cluck); use Text::Wrap; -use File::Find 'find'; -use File::Spec::Functions; # These variables are used for mapping level names to numbers. @@ -88,7 +86,6 @@ start level } else { return $the_instance -> {'level'}; } - #$self -> psn_in_inc; return; } end level @@ -236,13 +233,12 @@ start warn $subroutine. " : " . $message . "\n"; } else { my $level_name_length = length( $level_name ); - my $indent = ' ' x (4); - $text = "\n". $level_name . ': ' . $package . '->' . $subroutine. " :"; - $text .= wrap($prefix, $prefix . $indent, "\n" . $message ); - $text .= "\n"; + my $indent = ' ' x ($level_name_length + 4); + $text = wrap($prefix, $prefix . $indent, $level_name . ': ' . $package . '->' . + $subroutine. " : " . $message . "\n" ); } - if( $the_instance -> {'warn_with_trace'} ){ + if( $the_instance -> {'warn_with_trace'} >= $level ){ cluck( $text ); } else { print STDERR ( $text ); @@ -276,7 +272,7 @@ start die my $arr = scalar(@longmess); $prefix = ' ' x ($arr-2); } - if( $the_instance -> {'level'} > 1 ){ + if( $the_instance -> {'level'} > 0 ){ $! = 1; confess( $prefix . $message ); } else { @@ -287,15 +283,3 @@ start die } end die # }}} - -# {{{ psn_in_inc -start psn_in_inc -foreach my $plib ( @INC ) { - if ( $the_instance -> {'level'} > 1 ) { - find( { wanted => sub { print canonpath($_),"\n" if /PsN.*\.pm\z/ }, no_chdir => 1 }, $plib ) if ( $plib ne '.' ); - } elsif ( $the_instance -> {'level'} == 1 ) { - find( { wanted => sub { print canonpath($_),"\n" if /PsN\.pm\z/ }, no_chdir => 1 }, $plib ) if ( $plib ne '.' ); - } -} -end psn_in_inc -# }}} psn_in_inc diff --git a/lib/model/.cvsignore b/lib/model/.cvsignore index e4b6896..bb73cd5 100644 --- a/lib/model/.cvsignore +++ b/lib/model/.cvsignore @@ -1,2 +1,6 @@ +cwres_module.pm +iofv_module.pm +mirror_plot_module.pm +nonparametric_module.pm +shrinkage_module.pm problem.pm -k diff --git a/lib/model/cwres_module_subs.pm b/lib/model/cwres_module_subs.pm index 6de0aeb..d28cfac 100644 --- a/lib/model/cwres_module_subs.pm +++ b/lib/model/cwres_module_subs.pm @@ -7,6 +7,8 @@ start new $this -> {'nm_version'} = 'default'; } + my $mirror_name = $this -> {'mirror_plots'} ? 'sim' : ''; + my ($nmdir,$version) = split(/,/ , $PsN::config -> { 'nm_versions' } -> { $this -> {'nm_version'} } ); if( not defined $version ){ @@ -61,10 +63,22 @@ start new if( defined $sd_ref ) { foreach my $tabname ( @{$sd_ref} ) { if( $tabname =~ /[sd,pa]tab(\d+)/ ) { - $this -> sdno($1); + my $sdno = $1; + if( $sdno eq '' ){ + $sdno = 1; + } + $this -> sdno($sdno); for( my $i = 0; $i <= $#cwtab_names; $i++ ) { - if( $cwtab_names[$i] =~ /(\w+)\.(\d+)/ ) { - $cwtab_names[$i] = $1.$this -> sdno().'.'.$2; + + # This regular expression is probably quite unneccessary. It + # matches evertyhing before the first 'dot' in a filename, + # the dot, and the rest of the name(dots included). We can + # then inject a number before the first dot. It also handles + # no dots, the number will then be injected at the end of + # the filename. + + if( $cwtab_names[$i] =~ /([^\.]+)(\.{0,1})(.*)/ ) { + $cwtab_names[$i] = $1.$sdno.$mirror_name.$2.$3; } } $this -> cwtab_names( \@cwtab_names); @@ -114,10 +128,13 @@ start new # {{{ fortran code push( @{$code}, ('" IF (ICALL.EQ.0) THEN', - '"C open files here, if necessary') ); - for( my $i = 0; $i <= $#cwtab_names; $i++ ) { - push( @{$code}, '" OPEN(5'.$i.',FILE=\''.$cwtab_names[$i].'\')' ); - } + '"C open files here, if necessary', + '" OPEN(50,FILE=\'cwtab'.$this -> sdno().$mirror_name.'.est\')') ); + + #for( my $i = 0; $i <= $#cwtab_names; $i++ ) { + #push( @{$code}, '" OPEN(5'.$i.',FILE=\''.$cwtab_names[$i].'\')' ); + #} + push( @{$code}, # {{{ fortan code ('" ENDIF', @@ -125,6 +142,7 @@ start new '" MODE=0', '" CALL PASS(MODE)', '" MODE=1', + '" WRITE (50,*) \'ETAS\'', '" 20 CALL PASS(MODE)', '" IF (MODE.EQ.0) GO TO 30', '" IF (NEWIND.NE.2) THEN', @@ -133,16 +151,14 @@ start new '" ENDIF', '" GO TO 20', '" 30 CONTINUE', - '" WRITE (51,99) OBJECT', - '" WRITE (52,99) (THETAF(J),J=1,NTH)', - '" WRITE (53,99) (SETH(J), J=1,NTH)', + '" WRITE (50,*) \'THETAS\'', + '" WRITE (50,99) (THETAF(J),J=1,NTH)', + '" WRITE (50,*) \'OMEGAS\'', '" DO 7000 I=1,NETA', - '" WRITE (54,99) (OMEGAF(I,J),J=1,NETA)', - '" 7000 WRITE (55,99) (SEOM(I,J), J=1,NETA)', + '" 7000 WRITE (50,99) (OMEGAF(I,J),J=1,NETA)', + '" WRITE (50,*) \'SIGMAS\'', '" DO 7999 I=1,NEPS', - '" WRITE (56,99) (SIGMAF(I,J),J=1,NEPS)', - '" 7999 WRITE (57,99) (SESIG(I,J), J=1,NEPS)', - '" WRITE (58,98) IERE,IERC', + '" 7999 WRITE (50,99) (SIGMAF(I,J),J=1,NEPS)', '" ENDIF', '" 99 FORMAT (20E15.7)', '" 98 FORMAT (2I8)', @@ -153,60 +169,37 @@ start new } } elsif ( $version == 6 ) { + + my $code; + if( $have_advan ){ unless( $prob -> infns ){ $prob -> add_records( type => 'infn', record_strings => [] ); } - my $code = $prob -> infns -> [0] -> code; + $code = $prob -> infns -> [0] -> code; - push( @{$code}, 'IF (ICALL.EQ.3) THEN' ); - for( my $i = 0; $i <= $#cwtab_names; $i++ ) { - push( @{$code}, ' OPEN(5'.$i.',FILE=\''.$cwtab_names[$i].'\')' ); - } - push( @{$code}, - # {{{ fortran code - (' DO WHILE(DATA)', - ' IF (NEWIND.LE.1) WRITE (50,*) ETA', - ' ENDDO', - ' WRITE (51,*) OBJECT', - ' WRITE (52,*) THETA', - ' WRITE (53,*) SETHET', - ' WRITE (54,*) OMEGA(BLOCK)', - ' WRITE (55,*) SEOMEG(BLOCK)', - ' WRITE (56,*) SIGMA(BLOCK)', - ' WRITE (57,*) SESIGM(BLOCK)', - ' WRITE (58,*) IERE,IERC', - 'ENDIF') - - # }}} - ); } else { - my $code = $prob -> preds -> [0] -> code; - - push( @{$code}, 'IF (ICALL.EQ.3) THEN' ); - for( my $i = 0; $i <= $#cwtab_names; $i++ ) { - push( @{$code}, '" OPEN(5'.$i.',FILE=\''.$cwtab_names[$i].'\')' ); - } - push( @{$code}, - # {{{ fortran code - (' DO WHILE(DATA)', - ' IF (NEWIND.LE.1) WRITE (50,*) ETA', - ' ENDDO', - ' WRITE (51,*) OBJECT', - ' WRITE (52,*) THETA', - ' WRITE (53,*) SETHET', - ' WRITE (54,*) OMEGA(BLOCK)', - ' WRITE (55,*) SEOMEG(BLOCK)', - ' WRITE (56,*) SIGMA(BLOCK)', - ' WRITE (57,*) SESIGM(BLOCK)', - ' WRITE (58,*) IERE,IERC', - 'ENDIF') - # }}} - ); + $code = $prob -> preds -> [0] -> code; + } + + push( @{$code}, + 'IF (ICALL.EQ.3) THEN', + ' OPEN(50,FILE=\'cwtab'.$this -> sdno().$mirror_name.'.est\')', + ' WRITE (50,*) \'ETAS\'', + ' DO WHILE(DATA)', + ' IF (NEWIND.LE.1) WRITE (50,*) ETA', + ' ENDDO', + ' WRITE (50,*) \'THETAS\'', + ' WRITE (50,*) THETA', + ' WRITE (50,*) \'OMEGAS\'', + ' WRITE (50,*) OMEGA(BLOCK)', + ' WRITE (50,*) \'SIGMAS\'', + ' WRITE (50,*) SIGMA(BLOCK)', + 'ENDIF' ); } @@ -269,12 +262,12 @@ start new record_strings => ['ID ', join(' ',@table_row), "IPRED DV $mdv NOPRINT ". - "ONEHEADER FILE=cwtab".$this -> sdno()] ); - $prob -> add_records( type => 'table', - record_strings => ['ID ', - join(' ',@table_row), - "IPRED DV $mdv NOPRINT ". - "ONEHEADER FILE=cwtab".$this -> sdno().'.cwres'] ); + "ONEHEADER FILE=cwtab".$this -> sdno().'.deriv'] ); +# $prob -> add_records( type => 'table', +# record_strings => ['ID ', +# join(' ',@table_row), +# "IPRED DV $mdv NOPRINT ". +# "ONEHEADER FILE=cwtab".$this -> sdno().'.cwres'] ); } end new @@ -287,7 +280,8 @@ start post_process my ($advan,$junk) = $self -> {'problem'} -> _option_val_pos( record_name => 'subroutine', name => 'ADVAN', exact_match => 0); - my @cwtab_names = @{$self -> cwtab_names}; + + my $mirror_name = $self -> {'mirror_plots'} ? 'sim' : ''; if( $self -> {'nm_version'} == 5 and scalar(@{$advan}) > 0 ){ @@ -318,16 +312,19 @@ EOF print INFN " DATA NTH,NETA,NEPS/$ntheta,$neta,$neps/\n"; print INFN " IF (ICALL.EQ.0) THEN\nC open files here, if necessary\n"; - for( my $i = 0; $i <= $#cwtab_names; $i++ ) { - print INFN ' OPEN(5'.$i.',FILE=\''.$cwtab_names[$i].'\')'."\n"; - } - + print INFN " OPEN(50,FILE='cwtab".$self -> sdno().$mirror_name.".est')\n"; + +# for( my $i = 0; $i <= $#cwtab_names; $i++ ) { +# print INFN ' OPEN(5'.$i.',FILE=\''.$cwtab_names[$i].'\')'."\n"; +# } + print INFN << "EOF"; ENDIF IF (ICALL.EQ.3) THEN MODE=0 CALL PASS(MODE) MODE=1 + WRITE (50,*) 'ETAS' 20 CALL PASS(MODE) IF (MODE.EQ.0) GO TO 30 IF (NEWIND.NE.2) THEN @@ -336,16 +333,14 @@ EOF ENDIF GO TO 20 30 CONTINUE - WRITE (51,99) OBJECT - WRITE (52,99) (THETAF(J),J=1,NTH) - WRITE (53,99) (SETH(J), J=1,NTH) + WRITE (50,*) 'THETAS' + WRITE (50,99) (THETAF(J),J=1,NTH) + WRITE (50,*) 'OMEGAS' DO 7000 I=1,NETA - WRITE (54,99) (OMEGAF(I,J),J=1,NETA) - 7000 WRITE (55,99) (SEOM(I,J), J=1,NETA) + 7000 WRITE (50,99) (OMEGAF(I,J),J=1,NETA) + WRITE (50,*) 'SIGMAS' DO 7999 I=1,NEPS - WRITE (56,99) (SIGMAF(I,J),J=1,NEPS) - 7999 WRITE (57,99) (SESIG(I,J), J=1,NEPS) - WRITE (58,98) IERE,IERC + 7999 WRITE (50,99) (SIGMAF(I,J),J=1,NEPS) ENDIF 99 FORMAT (20E15.7) 98 FORMAT (2I8) diff --git a/lib/model/problem/error_subs.pm b/lib/model/problem/error_subs.pm index e2f91a6..cb10c04 100644 --- a/lib/model/problem/error_subs.pm +++ b/lib/model/problem/error_subs.pm @@ -1,2 +1,47 @@ start new end new + +start _format_record + { + if ( defined $self -> {'verbatim_first'} + or defined $self -> {'code'} + or defined $self -> {'verbatim_last'} ) { + my @class_names = split('::',ref($self)); + my $fname = uc(pop(@class_names)); + @formatted = "\$".$fname; + } + if ( defined $self -> {'verbatim_first'} ) { + push( @formatted, '"FIRST' ); + push( @formatted, @{$self -> {'verbatim_first'}} ); + } + my @cont; + if ( defined $self -> {'secondary_columns'} ) { + for ( my $i = 0; $i < scalar @{$self -> {'secondary_columns'}}; $i++ ) { + my $k = 1; + for ( my $j = 0; $j < scalar @{$self -> {'secondary_columns'}[$i]}; $j++ ) { + if ( $self -> {'secondary_columns'}[$i][$j][0] eq 'ID' or + $self -> {'secondary_columns'}[$i][$j][0] eq 'CONT' ) { + $k++; + next; + } + next if ( $self -> {'secondary_columns'}[$i][$j][0] =~ /XX(\d+)/ ); + my $str = "\" ".$self -> {'secondary_columns'}[$i][$j][0].'1 = EVTREC('.($i+1). + ','.($k++).")\n"; + $str = $str.$self -> {'secondary_columns'}[$i][$j][0].' = '. + $self -> {'secondary_columns'}[$i][$j][0].'1'."\n"; + push( @cont, $str ); + } + } + } + + push( @formatted, @cont ); + if ( defined $self -> {'code'} ) { + push( @formatted, @{$self -> {'code'}} ); + } + if ( defined $self -> {'verbatim_last'} ) { + push( @formatted, '"LAST' ); + push( @formatted, @{$self -> {'verbatim_last'}} ); + } + } +end _format_record + diff --git a/lib/model/problem/record_subs.pm b/lib/model/problem/record_subs.pm index e5d06ca..e761ba5 100644 --- a/lib/model/problem/record_subs.pm +++ b/lib/model/problem/record_subs.pm @@ -32,9 +32,11 @@ start remove_option my @options = @{$self -> {'options'}}; my @new_options = (); foreach my $option ( @options ) { - if ( $option -> name ne $name ) { - push( @new_options, $option ); - } + next if ( $option -> name eq $name ); + + next if ( $fuzzy_match and index( $name, $option -> name ) > -1 ); + + push( @new_options, $option ); } $self -> {'options'} = \@new_options; diff --git a/lib/model/problem_subs.pm b/lib/model/problem_subs.pm index d7366b5..f037123 100644 --- a/lib/model/problem_subs.pm +++ b/lib/model/problem_subs.pm @@ -44,10 +44,11 @@ start new $this -> {'ignore_missing_files'} ); } - if( $this -> {'compute_cwres'} ){ + if( $this -> {'cwres'} ){ $this -> add_cwres_module( 'init_data' => { problem => $this, - nm_version => $this -> {'nm_version'} } ); + nm_version => $this -> {'nm_version'}, + mirror_plots => $this -> {'mirror_plots'} } ); } @@ -191,7 +192,7 @@ start drop_dropped my @options = @{$opt_ref}; my @keep; foreach my $option ( @options ) { - push ( @keep, $option ) if ( not $option -> value eq 'DROP' or $option -> value eq 'SKIP' or + push ( @keep, $option ) if ( not ($option -> value eq 'DROP' or $option -> value eq 'SKIP') or $option -> name =~ /DAT(E|1|2|3)/ ); } $input -> options( \@keep ); @@ -349,13 +350,30 @@ start indexes my $accessor = $parameter_type.'s'; if( defined $self -> {$accessor} ){ + + # If we hit a "SAME" parameter we need to remember the + # previous parameter size. ( calculated as "row - previous_row" ) + + my $previous_row = 0; + foreach my $record ( @{$self -> $accessor} ) { # If not a block or if the block is of size 1 use normal numbering - if ( ( ! defined $record -> size ) or - ( $record -> size < 2 )) { - if( $record -> same() ) { - push( @indexes, $row++ ); - } elsif ( defined $record -> options ) { + + if( $record -> same ) { + + if( $previous_row == 0 ){ + 'debug' -> die( message => "You can't have an $parameter_type estimate defined as SAME if it's the first estimate" ); + } + my $size = $row - $previous_row; + $previous_row = $row; + for( 1..$size ){ + push( @indexes, $row++); + } + + } elsif ( ( ! defined $record -> size ) or + ( $record -> size < 2 )) { + if ( defined $record -> options ) { + $previous_row = $row; foreach my $option ( @{$record -> options} ) { push( @indexes, $row++ ); } @@ -372,6 +390,7 @@ start indexes } } } + $previous_row = $row; $row += $size; } } @@ -402,7 +421,18 @@ start nomegas foreach my $omega ( @{$self -> {'omegas'}} ) { my $size = $omega -> size; if( defined $size ) { - $nomegas += $size; + + # If the record has a size, it is of block form with diagonal of + # length given by $size. The actual values in the model file is + # then the arithmetic sum: (n*(n+1))/2 + + if( $with_correlations ){ + $nomegas += ($size*($size+1))/2; + + # But we really only want the diagonal elements here: + } else { + $nomegas += $size; + } } else { $nomegas += scalar @{$omega -> options}; } @@ -419,7 +449,21 @@ start nsigmas foreach my $sigma ( @{$self -> {'sigmas'}} ) { my $size = $sigma -> size; if( defined $size ) { - $nsigmas += $size; + + # If the record has a size, it is of block form with diagonal of + # length given by $size. The actual values in the model file is + # then the arithmetic sum: (n*(n+1))/2 + + if( $with_correlations ){ + + $nsigmas += ($size*($size+1))/2; + + } else { + + # But we really only want the diagonal elements here: + + $nsigmas += $size; + } } else { $nsigmas += scalar @{$sigma -> options}; } @@ -476,9 +520,13 @@ start secondary_columns if ( defined $parm ) { if ( defined $self -> pks ) { $self -> pks -> [0] -> secondary_columns($parm); - } else { + } + if ( defined $self -> preds ) { $self -> preds -> [0] -> secondary_columns($parm); } + if ( defined $self -> errors ) { + $self -> errors -> [0] -> secondary_columns($parm); + } # This is a lengthy bit of code to create a reasonable header and secondary # columns for the table files. When the data file of the problem is wrapped # using the CONT data item, the rows of the table files will be duplicated (or @@ -651,7 +699,7 @@ if( defined $self -> {$accessor} ) { record_strings => ["$option_name=$option_value"] ); } else { 'debug' -> warn( level => 2, - message => "No records of typ $accessor and add_option ". + message => "No records of type $accessor and add_option ". "set not to add one" ); } } @@ -671,11 +719,12 @@ unless( $self -> can($accessor) ){ if( defined $self -> {$accessor} ) { my @records = @{$self -> {$accessor}}; foreach my $record ( @records ) { - $record -> remove_option( name => $option_name ); + $record -> remove_option( name => $option_name, + fuzzy_match => $fuzzy_match ); } } else { 'debug' -> warn( level => 2, - message => "No records of typ $accessor" ); + message => "No records of type $accessor" ); } end remove_option @@ -811,7 +860,8 @@ start _init_attr if ( $record -> same() ) { # SAME == true: Nothing to be done. Just move forward to next $OMEGA but # increase counter first - $opt_num += $prev_size*($prev_size+1)/2; + + $opt_num += $prev_size; } else { foreach my $option ( @{$record -> options} ) { if ( scalar @parameter_numbers > 0 ) { @@ -821,6 +871,9 @@ start _init_attr if ( $attribute eq 'init' ) { push( @diagnostics, $option -> check_and_set_init( new_value => $num_val{$num} ) ); + } elsif( $attribute eq 'fix' and defined $record -> size() ){ + # size() tells us this is a block and we must fix on record level. + $record -> fix( $num_val{$num} ); } else { $option -> $attribute( $num_val{$num} ); } @@ -830,42 +883,66 @@ start _init_attr if ( $attribute eq 'init' ) { push( @diagnostics, $option -> check_and_set_init( new_value => shift( @new_values ) ) ); + } elsif( $attribute eq 'fix' and defined $record -> size() ){ + # size() tells us this is a block and we must fix on record level. + $record -> fix( shift( @new_values ) ); } else { $option -> $attribute( shift( @new_values ) ); } } $opt_num++; } - $prev_size = $record -> size; + if( $parameter_type eq 'theta' ){ + $prev_size = scalar @{$record -> options}; + } else { + my $size = $record -> size; + if( defined $size ) { + $prev_size = ($size*($size+1))/2; + } else { + $prev_size = scalar @{$record -> options}; + } + } } } # If $add_if_absent is set, any parameters that were not found above are # added below: + my @nums = sort {$a<=>$b} keys %found; + my $new_record = "model::problem::$parameter_type" -> new(); + my $do_add_record; foreach my $num ( @nums ) { if ( $add_if_absent and not $found{$num} ) { + $do_add_record = 1; unless($num == $opt_num) { 'debug' -> die( message => "Attempt to add a parameter with higher number ($num) than the number\n". "of parameters + 1 ($opt_num)\n" ); } # Get the last record of $parameter_type - my $last_record = $records[$#records]; + # my $new_record = $records[$#records]; my $option_class; if( $parameter_type eq 'theta' ){ $option_class = 'model::problem::record::theta_option'; } else { $option_class = 'model::problem::record::init_option'; } -# print "NUM: $num did not find: ",$found{$num},"\n"; + # Push a new option to this last record my $option = $option_class -> new; if ( $attribute eq 'init' ) { $option -> check_and_set_init( new_value => $num_val{$num} ); + } elsif( $attribute eq 'fix' and defined $new_record -> size() ){ + + # size() tells us this is a block and we must fix on + # record level. This will never happen, as we can't + # add BLOCKS, at least not like this. + + $new_record -> fix( $num_val{$num} ); } else { $option -> $attribute( $num_val{$num} ); } - push( @{$last_record -> {'options'}}, $option ); + push( @{$new_record -> {'options'}}, $option ); + # So we've added a parameter. Possible to add more, # lets increase the highest found: $opt_num++; @@ -877,6 +954,11 @@ start _init_attr @parameter_values = @diagnostics; } + if( $do_add_record ){ + push( @records, $new_record ); + $self -> {$accessor} = \@records ; + } + # }}} Update values } else { # {{{ Retrieve values @@ -1108,7 +1190,7 @@ start _option_val_pos @records = @{$self -> {$accessor}} ; } else { 'debug' -> warn( level => 2, - message => "No records of typ $accessor" ); + message => "No records of type $accessor" ); @records = (); } my @options = (); diff --git a/lib/model_subs.pm b/lib/model_subs.pm index 1a8df48..6e6e157 100644 --- a/lib/model_subs.pm +++ b/lib/model_subs.pm @@ -215,6 +215,25 @@ start new target => $this -> {'target'}, model_id => $this -> {'model_id'} ) ); } + + # Adding mirror_plots module here, since it can add + # $PROBLEMS. Also it needs to know wheter an lst file exists + # or not. + + if( $this -> {'mirror_plots'} > 0 ){ + my $mirror_plot_module = model::mirror_plot_module -> new( base_model => $this, + nr_of_mirrors => $this -> {'mirror_plots'}, + cwres => $this -> {'cwres'}, + mirror_from_lst => $this -> {'mirror_from_lst'}); + push( @{$this -> {'mirror_plot_modules'}}, $mirror_plot_module ); + } + + if( $this -> {'iofv'} > 0 ){ + my $iofv_module = model::iofv_module -> new( base_model => $this, + nm_version => $this -> {'nm_version'}); + push( @{$this -> {'iofv_modules'}}, $iofv_module ); + } + } end new @@ -738,7 +757,7 @@ start set_records $problems[$i-1] -> set_records( 'type' => $type, 'record_strings' => \@record_strings ); } else { - 'debug' -> die( "Problem number $i does not exist." ); + 'debug' -> die( message => "Problem number $i does not exist." ); } } # else { @@ -1175,7 +1194,7 @@ start datas my @new_datas = @{$parm}; # Check that new_headers and problems match 'debug' -> die( message => "The number of problems $nprobs and". - " new data ". $#new_datas+1 ." don't match in ". + " new data ". ($#new_datas+1) ." don't match in ". $self -> full_name ) unless ( $#new_datas + 1 == $nprobs ); if ( defined $self -> {'problems'} ) { for( my $i = 0; $i < $nprobs; $i++ ) { @@ -2345,6 +2364,11 @@ start is_option_set } foreach my $option ( @options ) { $found = 1 if ( defined $option and $option -> name eq $name ); + if( $fuzzy_match ){ + if( index( $name, $option -> name ) > -1 ){ + $found = 1; + } + } } } end is_option_set @@ -2669,6 +2693,7 @@ start labels @index = @{$self -> indexes( parameter_type => $parameter_type, parameter_numbers => \@parameter_numbers, problem_numbers => \@problem_numbers )}; + for ( my $i = 0; $i <= $#labels; $i++ ) { for ( my $j = 0; $j < scalar @{$labels[$i]}; $j++ ) { $idx = $index[$i][$j]; @@ -3035,7 +3060,6 @@ end min # {{{ name_val - =head2 fractions Usage: @@ -3338,7 +3362,7 @@ start nomegas my @problems = @{$self -> {'problems'}}; foreach my $i ( @problem_numbers ) { if ( defined $problems[ $i-1 ] ) { - push( @nomegas, $problems[ $i-1 ] -> nomegas ); + push( @nomegas, $problems[ $i-1 ] -> nomegas( with_correlations => $with_correlations )); } else { 'debug' -> die( "Problem number $i does not exist." ); } @@ -3407,7 +3431,7 @@ unless( $#problem_numbers >= 0 ){ my @problems = @{$self -> {'problems'}}; foreach my $i ( @problem_numbers ) { if ( defined $problems[ $i-1 ] ) { - push( @nsigmas, $problems[ $i-1 ] -> nsigmas ); + push( @nsigmas, $problems[ $i-1 ] -> nsigmas( with_correlations => $with_correlations )); } else { 'debug' -> die( "Problem number $i does not exist." ); } @@ -4371,7 +4395,21 @@ my ( $name_ref, $junk ) = $self -> record_name => 'estimation', problem_numbers => \@problem_numbers, new_values => \@new_names ); -@names = @{$name_ref}; + + +my ( $nonp_name_ref, $junk ) = $self -> + _option_val_pos( name => 'MSFO', + record_name => 'nonparametric', + problem_numbers => \@problem_numbers, + new_values => \@new_names ); + +if( length( @{$name_ref} > 0 ) ){ + push( @names, @{$name_ref} ); +} + +if( length( @{$nonp_name_ref} ) ){ + push( @names, @{$nonp_name_ref} ); +} end msfo_names @@ -4817,7 +4855,7 @@ start update_inits # } # }}} - + for ( my $i = 0; $i <= $#own_labels; $i++ ) { if( $from_output -> have_user_defined_prior ){ @@ -5055,6 +5093,8 @@ start wrap_data { my $default_wrap = 18; + $self -> drop_dropped(1); + my ( @wrap_columns, @cont_columns ); if ( not defined $wrap_column ) { for( my $i = 0; $i < scalar @{$self -> {'problems'}}; $i++ ) { @@ -5295,6 +5335,11 @@ start _write $data -> _write; } } + + if( $self -> {'iofv_modules'} ){ + $self -> {'iofv_modules'} -> [0] -> post_process; + } + } end _write @@ -5632,14 +5677,15 @@ start _read_problems ignore_missing_files => $self -> {'ignore_missing_files'}, ignore_missing_output_files => $self -> {'ignore_missing_output_files'}, sde => $self -> {'sde'}, - compute_cwres => $self -> {'compute_cwres'}, + cwres => $self -> {'cwres'}, + mirror_plots => $self -> {'mirror_plots'}, nm_version => $self -> {'nm_version'}, prob_arr => \@problem_lines, extra_data_file_name => $extra_data_files[$prob_num], extra_data_header => $extra_data_headers[$prob_num], shrinkage_module => $sh_mod ); push( @problems, $prob ); - if ( $self -> compute_cwres() ) { + if ( $self -> cwres() ) { my @eo; if ( defined $self -> extra_output() ) { @eo = @{$self -> extra_output()}; @@ -5677,9 +5723,11 @@ foreach my $i ( @problem_numbers ) { if ( defined $problems[ $i-1 ] ) { my $found = $self -> is_option_set( 'problem_number' => $i, 'record' => $record_name, - 'name' => $option_name ); + 'name' => $option_name, + 'fuzzy_match' => $fuzzy_match ); $problems[$i-1] -> remove_option( record_name => $record_name, - option_name => $option_name ) if ( $found ); + option_name => $option_name, + fuzzy_match => $fuzzy_match ) if ( $found ); $problems[$i-1] -> add_option( record_name => $record_name, option_name => $option_name, option_value => $option_value ); @@ -5724,7 +5772,8 @@ my @problems = @{$self -> {'problems'}}; foreach my $i ( @problem_numbers ) { if ( defined $problems[ $i-1 ] ) { $problems[$i-1] -> remove_option( record_name => $record_name, - option_name => $option_name ); + option_name => $option_name, + fuzzy_match => $fuzzy_match); } } @@ -5817,7 +5866,10 @@ start subroutine_files } } } - + + # BUG , nonmem6 might not require the file to be named .f And I've + # seen examples of files named .txt + @fsubs = keys %fsubs; if( @fsubs > 0 ){ for( my $i = 0; $i <= $#fsubs; $i ++ ){ diff --git a/lib/nonmem_subs.pm b/lib/nonmem_subs.pm index 244080a..3599114 100644 --- a/lib/nonmem_subs.pm +++ b/lib/nonmem_subs.pm @@ -50,7 +50,6 @@ start new unless( defined $nmdir ){ print "Unknown NONMEM version ",$this -> {'version'}," specified.\n"; - print "Configuration file is ",$PsN::config_file,"\n"; my @nmkeys = keys %{$PsN::config -> { 'nm_versions' }}; if ( $#nmkeys == 0 and defined $nmkeys[0] and @@ -125,29 +124,27 @@ start compile } my $modelfile = $self -> {'modelfile'}; + if( $Config{osname} eq 'MSWin32' ){ run3( "$nmdir/tr/nmtran.exe", $modelfile, \$self -> {'nmtran_message'}, \$self -> {'nmtran_message'} ); } else { run3( 'nice -' . $self -> {'nice'} . " $nmdir/tr/nmtran.exe", $modelfile, \$self -> {'nmtran_message'}, \$self -> {'nmtran_message'} ); } - #$self -> {'nmtran_message'} = `nice -19 $nmdir/tr/nmtran.exe < $modelfile 2>&1`; - open( NMMSG, '>compilation_output.txt' ); print( NMMSG $self -> {'nmtran_message'}, "\n" ); - + unless(-e 'FREPORT'){ close(NMMSG); $self -> {'error_message'} = "NMtran failed: \n" . $self -> {'nmtran_message'} ; return 0; } - + my $nmlink_message; - run3( "$nmlink", undef, \$nmlink_message, \$nmlink_message ); print( NMMSG $nmlink_message, "\n" ); - + my $fsub; if(-e 'FSUBS'){ @@ -160,7 +157,7 @@ start compile } } - if( defined $self -> {'fsubs'} ){ + if( defined $self -> {'fsubs'} and $self -> {'version'} != 6 ){ foreach my $sub ( @{$self -> {'fsubs'}} ){ $fsub .= " $sub"; } @@ -179,33 +176,37 @@ start compile my $compile_message; my $compile_command; + my @nmlib; + + if( $Config{osname} eq 'MSWin32' ){ + @nmlib = ("$nm\\NONMEM.obj","$nm\\BLKDAT.obj","$nm\\nonmem.lib"); + } else { + @nmlib = ("$nm/NONMEM.o", "$nm/BLKDAT.o", "$nm/nonmem.a"); + } + $compile_command = "g77 " . $self -> {'compiler_options'} . " -ononmem$version$adaptive $includes $fsub @link @nmlib 2>&1"; + if( $self -> {'compiler'} eq 'g77' ){ - my @nmlib = ("$nm/NONMEM.o", "$nm/BLKDAT.o", "$nm/nonmem.a"); $compile_command = "g77 " . $self -> {'compiler_options'} . " -ononmem$version$adaptive $includes $fsub @link @nmlib 2>&1"; } elsif( $self -> {'compiler'} eq 'df' or $self -> {'compiler'} eq 'fl32' ){ - - my @nmlib = ("$nm\\nonmem.obj", "$nm\\blkdat.obj", "$nm\\nonmem.lib"); $compile_command = $self -> {'compiler'}." " . $self -> {'compiler_options'} . " /Fenonmem $fsub @nmlib @link"; - + } elsif( $self -> {'compiler'} eq 'ifort' ){ - my @nmlib = ("$nm/NONMEM.o", "$nm/BLKDAT.o", "$nm/nonmem.a"); - $compile_command = "ifort " . $self -> {'compiler_options'} . " -ononmem$version$adaptive $includes $fsub @link @nmlib 2>&1"; + + $compile_command = "ifort " . $self -> {'compiler_options'} . " -ononmem$version$adaptive $includes $fsub @link @nmlib 2>&1"; + } run3( "$compile_command", undef, \$compile_message, \$compile_message ); print( NMMSG $compile_message, "\n" ); - + + close( NMMSG ); unless(-e "nonmem.exe" or -e "nonmem$version$adaptive" ) { $self -> {'error_message'} = "Fortran Compilation failed: \n" . $compile_message ; - close( NMMSG ); return 0; - } else { - print( NMMSG "Compilation done\n" ); - close( NMMSG ); } } end compile @@ -249,7 +250,7 @@ start execute } run3( "nice -n " . $self -> {'nice'} . " ./nonmem$version$adaptive", "FCON", $outputfile ); if ( -e "/proc/self/lock" ) { - open (OUTFILE,">/proc/self/lock") || die "Could not lock myself!\n"; + open (OUTFILE,">/proc/self/lock") || die "Could not unlock myself!\n"; print OUTFILE "1"; close (OUTFILE); } diff --git a/lib/output/problem/subproblem_subs.pm b/lib/output/problem/subproblem_subs.pm index 3a4674d..9144f52 100644 --- a/lib/output/problem/subproblem_subs.pm +++ b/lib/output/problem/subproblem_subs.pm @@ -196,7 +196,14 @@ start _compute_comegas_or_csigmas # undefined). my $row_size = 1; for( my $i = 0; $i <= $#raw_omegas_or_sigmas; $i+=$row_size ){ - my $omega_or_sigma_value = sqrt( $raw_omegas_or_sigmas[$i] ); + my $omega_or_sigma_value = undef; + if( $raw_omegas_or_sigmas[$i] >= 0 ) { + $omega_or_sigma_value = sqrt( $raw_omegas_or_sigmas[$i] ); + } else { + ui -> print( category => 'all', + message => "Warning: cannot take the square root of ". + "$omega_or_sigma with value ".$raw_omegas_or_sigmas[$i] ); + } push( @{$self -> {'c' . $omega_or_sigma .'s'}}, $omega_or_sigma_value ); $row_size ++; } diff --git a/lib/output/problem_subs.pm b/lib/output/problem_subs.pm index 94d5c57..90cfc91 100644 --- a/lib/output/problem_subs.pm +++ b/lib/output/problem_subs.pm @@ -491,7 +491,7 @@ while( $_ = @{$self -> {'lstfile'}}[ $start_pos++ ] ) { } } -unless( ( $self -> estimation_step_run() * $est_allowed ) or +unless( ( $self -> estimation_step_initiated() * $est_allowed ) or ( $self -> covariance_step_run() * $cov_allowed ) or ( $self -> nonparametric_step_run() * $nonp_allowed ) or ( $self -> tables_step_run() * $tables_allowed ) ) { @@ -617,20 +617,24 @@ while( $_ = @{$self -> {'lstfile'}}[ $start_pos++ ] ) { if ( /^0COVARIANCE STEP OMITTED/ or /0TABLES STEP OMITTED/ or /1DOUBLE PRECISION PREDPP/ or - /0SEARCH WITH ESTIMATION STEP WILL NOT PROCEED/ ) { + /0SEARCH WITH ESTIMATION STEP WILL NOT PROCEED/ or + /^1/ or + /^0MINIMIZATION/ or + /^ PROBLEM NO\.:\s+\d/ ) { # This is ok, the nonp step was not used. last; } - if( /^ PROBLEM NO\.:\s+\d/ or - /^0MINIMIZATION/ ) { + +# if( /^ PROBLEM NO\.:\s+\d/ or +# /^0MINIMIZATION/ ) { # This should not happen, raise error - my $errmess = "Found $_ while searching for the (optional) ". - "nonparametric step indicator\n"; - debug -> warn( level => 1, - message => $errmess."$!" ); - $self -> parsing_error( message => $errmess."$!" ); - return; - } +# my $errmess = "Found $_ while searching for the (optional) ". +# "nonparametric step indicator\n"; +# debug -> warn( level => 1, +# message => $errmess."$!" ); +# $self -> parsing_error( message => $errmess."$!" ); +# return; +# } if ( ($start_pos + 1) == scalar @{$self -> {'lstfile'}} ) { #EOF This should not happen, raise error @@ -754,7 +758,6 @@ start _read_inits my $start_pos = $self -> {'lstfile_pos'}; my ( @thetas, @omegas, @sigmas ); my $thetarea = 0; - my $thetabounds = 0; my $omegarea = 0; my $sigmarea = 0; my $success = 0; @@ -808,10 +811,6 @@ start _read_inits $thetarea = 1; } - if ( $thetarea and /LOWER BOUND\s+INITIAL EST\s+UPPER BOUND/ ){ - $thetabounds = 1; - } - if ( /^0ESTIMATION STEP OMITTED/ or /^0SIMULATION STEP OMITTED/ ) { # We want to find this if we are currently reading the omega or sigma inits @@ -829,25 +828,16 @@ start _read_inits return; } if ( $thetarea and /^\s*-?\d*\.\d*/ ) { - if( $thetabounds ){ - my @T = split(' ',$_); - push(@{$self -> {'initthetas'}},eval($T[1])); - push(@{$self -> {'lower_theta_bounds'}},eval($T[0])); - push(@{$self -> {'upper_theta_bounds'}},eval($T[2])); - if ( $T[0] == $T[1] and $T[0] == $T[2] ) { - push(@{$self -> {'fixedthetas'}},1); - push(@{$self -> {'estimated_thetas'}},0); - } else { - push(@{$self -> {'fixedthetas'}},0); - push(@{$self -> {'estimated_thetas'}},1); - } + my @T = split(' ',$_); + push(@{$self -> {'initthetas'}},eval($T[1])); + push(@{$self -> {'lower_theta_bounds'}},eval($T[0])); + push(@{$self -> {'upper_theta_bounds'}},eval($T[2])); + if ( $T[0] == $T[1] and $T[0] == $T[2] ) { + push(@{$self -> {'fixedthetas'}},1); + push(@{$self -> {'estimated_thetas'}},0); } else { - my @T = split(' ',$_); - foreach my $theta( @T ){ - push( @{$self -> {'initthetas'}}, eval($theta)); - push(@{$self -> {'fixedthetas'}},0); - push(@{$self -> {'estimated_thetas'}},1); - } + push(@{$self -> {'fixedthetas'}},0); + push(@{$self -> {'estimated_thetas'}},1); } } if ( /^0INITIAL ESTIMATE OF OMEGA:/ ) { @@ -884,6 +874,7 @@ start _read_inits my $om_row = 1; while( ($start_pos + 1) < scalar @{$self -> {'lstfile'}} ) { if( $self -> {'lstfile'}[ $start_pos ] =~ /^0INITIAL ESTIMATE OF SIGMA/ or + $self -> {'lstfile'}[ $start_pos ] =~ /^0SIMULATION STEP OMITTED/ or $self -> {'lstfile'}[ $start_pos ] =~ /^0ESTIMATION STEP OMITTED/ or $self -> {'lstfile'}[ $start_pos ] =~ /^0OMEGA CONSTRAINED TO BE THIS INITIAL ESTIMATE/ ) { $start_pos--; @@ -892,7 +883,7 @@ start _read_inits # After ten rows of omegas NONMEM starts wrapping # lines. We then need to skip the first part of the - # wrapped lines. This nice little formula calculates + # wrapped lines. This nice littel formula calculates # how many lines to skip. my $skip_lines = ($om_row - $om_row % 10)/10; @@ -963,7 +954,7 @@ start _read_inits # After ten rows of sigmas NONMEM starts wrapping # lines. We then need to skip the first part of the - # wrapped lines. This nice little formula calculates + # wrapped lines. This nice littel formula calculates # how many lines to skip. my $skip_lines = ($sm_row - $sm_row % 10)/10; diff --git a/lib/output_subs.pm b/lib/output_subs.pm index 2966d31..cf3eb74 100644 --- a/lib/output_subs.pm +++ b/lib/output_subs.pm @@ -77,6 +77,8 @@ start new # be left on disk in an effort to preserve memory. The file # will be read if needed. + debug -> warn( level => 2, + message => "Initiating new\tNM::output object from file $parm{'filename'}" ); if ( defined $this -> {'filename'} and $this -> {'filename'} ne '' ) { ( $this -> {'directory'}, $this -> {'filename'} ) = OSspecific::absolute_path( $this -> {'directory'},$this->{'filename'} ); @@ -91,19 +93,7 @@ start new } } else { debug -> die( message => "No filename specified or filename equals empty string!" ); - } - if( defined $this -> problems() ) { - my $mes = $this -> parsing_error_message(); - foreach my $prob ( @{$this -> problems()} ) { - $mes .= $prob -> parsing_error_message(); - $this -> parsed_successfully($this -> parsed_successfully() * - $prob -> parsed_successfully()); - } - $this -> parsing_error_message( $mes ); - } - if( defined $this -> parsing_error_message() and - $this -> parsing_error_message() ne '' ) { - print $this -> full_name,":\n",$this -> parsing_error_message(); + $this->{'filename'} = 'tempfile'; } if( defined $this -> problems() ) { my $mes = $this -> parsing_error_message(); @@ -739,7 +729,7 @@ start _read_problems $self -> {'lstfile_pos'} = 0; -# {{{ Old db code. Keep for now +# Old db code. Keep for now # if ( $PsN::config -> {'_'} -> {'use_database'} and # $self -> {'register_in_database'} and # defined $self -> {'output_id'} ) { @@ -758,7 +748,6 @@ start _read_problems # $dbh -> disconnect; # } -# }}} my $problem_start; my $success = 0; @@ -1141,6 +1130,7 @@ start flush { $self -> {'problems'} = undef; + $self -> {'synced'} = 0; } end flush # }}} flush diff --git a/lib/psn.conf b/lib/psn.conf index 0ea7c61..6dbe76e 100644 --- a/lib/psn.conf +++ b/lib/psn.conf @@ -19,20 +19,21 @@ output_style = SPLUS ; enables automatic calculation of conditional weighted residuals ; (CWRES). This feature requires that Xpose version 4.0 preview ; release 5 is also installed. -;R = c:\Program Files\R\R-2.4.1\bin\R.exe +;R = c:\progra~1\R-2.5.0\bin\R + +; Path to perl. This default to "C:\Perl\bin\perl.exe" on windows and +; simply 'perl' on unix. +; perl = c:\Perl\bin\perl.exe ; These two lines should not be modified, they are used to disable ; experimental features of PsN. use_keyboard=0 -;use_database=1 +use_database=0 ; Further experimental features -;database_server=doris.biof.uu.se +;database_server=localhost ;default_project=psn ;default_user=psn -;min_fork_delay=0 -;ax_fork_delay=0 - ; Set the job polling interval to a value that is suitable to your ; system. The general rule is that the larger the distributed system ; gets, the longer the polling interval needs to be. Slow systems, @@ -81,9 +82,9 @@ default=c:\nmvi,6 ; this: ; name=c:\fortran\fl32 -;[compiler] +[compiler] ; Gnu fortran compiler -;name=g77 +name=g77 ;options=-W -static -O ;[compiler] @@ -96,16 +97,11 @@ default=c:\nmvi,6 ;name=fl32 ;options=/Ox /Op -[compiler] +;[compiler] ;;Digitial Visual Fortran 5.0 -name=df +;name=df ;options=/optimize:4 /fltconsistency /fpe:3 -options=/optimize:1 /fpe:0 - -[compiler] -; Intel Fortran -name=ifort -options=-O3 +;options=/optimize:1 /fpe:0 [default_options] ; Options valid for all tools diff --git a/lib/tool/.cvsignore b/lib/tool/.cvsignore index 734c4f2..efd76fc 100644 --- a/lib/tool/.cvsignore +++ b/lib/tool/.cvsignore @@ -5,3 +5,4 @@ modelfit.pm scm.pm xv_step.pm xv.pm +mc.pm diff --git a/lib/tool/bootstrap_subs.pm b/lib/tool/bootstrap_subs.pm index 4b09a6a..61f925f 100644 --- a/lib/tool/bootstrap_subs.pm +++ b/lib/tool/bootstrap_subs.pm @@ -895,6 +895,13 @@ start modelfit_analyze unshift( @{$param_names[$i]}, 'OFV' ); push( @{$diagnostic_names[$i]}, @tmp_names ); } + if( defined $PsN::config -> {'_'} -> {'R'} and + -e $PsN::lib_dir . '/R-scripts/bootstrap.R' ) { + # copy the bootstrap R-script + cp ( $PsN::lib_dir . '/R-scripts/bootstrap.R', $self -> {'directory'} ); + # Execute the script + system( $PsN::config -> {'_'} -> {'R'}." CMD BATCH bootstrap.R" ); + } } end modelfit_analyze diff --git a/lib/tool/cdd_subs.pm b/lib/tool/cdd_subs.pm index c05f9a8..39ce861 100644 --- a/lib/tool/cdd_subs.pm +++ b/lib/tool/cdd_subs.pm @@ -45,7 +45,7 @@ start new if ( $PsN::config -> {'_'} -> {'use_database'} ) { my( $found_log, $found_cdd_id ) = $this -> read_cdd_log; - $this -> register_in_database unless ( $found_cdd_id ); + $this -> register_cdd_in_database unless ( $found_cdd_id ); $this -> log_object unless ( $found_log and $found_cdd_id ); print "Found ",$this -> {'cdd_id'},"\n"; @@ -55,43 +55,35 @@ end new # }}} new -# {{{ register_in_database +# {{{ register_cdd_in_database -start register_in_database +start register_cdd_in_database { if ( $PsN::config -> {'_'} -> {'use_database'} ) { my $dbh = DBI -> connect("DBI:mysql:host=".$PsN::config -> {'_'} -> {'database_server'}. ";databse=".$PsN::config -> {'_'} -> {'project'}, $PsN::config -> {'_'} -> {'user'}, - $PsN::config -> {'_'} -> {'password'}, + $PsN::config -> {'_'} -> {'password'}, {'RaiseError' => 1}); my $sth; - # bins and case_column can be defined for more than one - # model. We nevertheless begin by implementing this. - print "CC: ".$self -> {'case_columns'}."\n"; - print("INSERT INTO ".$PsN::config -> {'_'} -> {'project'}. - ".cdd ( bins, case_column,xv ) ". - "VALUES ( '".$self -> {'bins'}. - "', '".$self -> {'case_columns'}. - "', '".$self -> {'cross_validate'}."' )\n"); + # bins and case_column can be defined for more than one model. Skip + # registration of these for now. +# $sth = $dbh -> prepare("INSERT INTO ".$PsN::config -> {'_'} -> {'project'}. +# ".cdd ( tool_id, bins, case_column ) ". +# "VALUES (".$self -> {'tool_id'}.", '".$self -> {'bins'}. +# "', '".$self -> {'case_column'}."' )"); $sth = $dbh -> prepare("INSERT INTO ".$PsN::config -> {'_'} -> {'project'}. - ".cdd ( bins, case_column,xv ) ". - "VALUES ( '".$self -> {'bins'}. - "', '".$self -> {'case_columns'}. - "', '".$self -> {'cross_validate'}."' )"); -# $sth = $dbh -> prepare("INSERT INTO ".$PsN::config -> {'_'} -> {'project'}. -# ".cdd ( tool_id ) ". -# "VALUES (".$self -> {'tool_id'}." )"); + ".cdd ( tool_id ) ". + "VALUES (".$self -> {'tool_id'}." )"); $sth -> execute; $self -> {'cdd_id'} = $sth->{'mysql_insertid'}; $sth -> finish; $dbh -> disconnect; - tool::register_in_database( $self, cdd_id => $self -> {'cdd_id'} ); } } -end register_in_database +end register_cdd_in_database -# }}} register_in_database +# }}} register_cdd_in_database # {{{ register_mfit_results @@ -1440,6 +1432,13 @@ start modelfit_analyze # experimental: to save memory $self -> {'prepared_models'}[$model_number-1]{'own'} = undef; + if( defined $PsN::config -> {'_'} -> {'R'} and + -e $PsN::lib_dir . '/R-scripts/cdd.R' ) { + # copy the cdd R-script + cp ( $PsN::lib_dir . '/R-scripts/cdd.R', $self -> {'directory'} ); + # Execute the script + system( $PsN::config -> {'_'} -> {'R'}." CMD BATCH cdd.R" ); + } } end modelfit_analyze diff --git a/lib/tool/llp_subs.pm b/lib/tool/llp_subs.pm index 2070563..c6d36ed 100644 --- a/lib/tool/llp_subs.pm +++ b/lib/tool/llp_subs.pm @@ -811,6 +811,14 @@ start modelfit_analyze $self -> {'raw_results'}[$model_number-1] = $self -> {'tools'} -> [0] -> raw_results if( defined $self -> {'tools'} -> [0] ); } + if( $self -> iteration() < 2 and + defined $PsN::config -> {'_'} -> {'R'} and + -e $PsN::lib_dir . '/R-scripts/llp.R' ) { + # copy the llp R-script + cp ( $PsN::lib_dir . '/R-scripts/llp.R', $self -> {'directory'} ); + # Execute the script + system( $PsN::config -> {'_'} -> {'R'}." CMD BATCH llp.R" ); + } } end modelfit_analyze diff --git a/lib/tool/mc_subs.pm b/lib/tool/mc_subs.pm index 1e4ef4a..2759f95 100644 --- a/lib/tool/mc_subs.pm +++ b/lib/tool/mc_subs.pm @@ -93,6 +93,8 @@ if ( not $done ) { my @sim_models; my @table_header; my @all_simulated_files; + my ( @orig_table_names, @alt_table_names ); + # Copy the model to a new simulation models for( my $sim_no = 1; $sim_no <= $self -> {'samples'}; $sim_no++ ) { my $sim_name = "mc-$sim_no.sim"; @@ -114,12 +116,16 @@ if ( not $done ) { copy_data => 0, copy_output => 0); $est_original -> drop_dropped; + my $tbl_nm_ref = $model -> table_names(); + @orig_table_names = @{$tbl_nm_ref} if( defined $tbl_nm_ref ); $est_alternative = $alternative -> copy( filename => $self -> {'directory'}.'m'.$model_number.'/'.$alt_name, target => 'mem', copy_data => 0, copy_output => 0); $est_alternative -> drop_dropped; + my $tbl_nm_ref = $alternative -> table_names(); + @alt_table_names = @{$tbl_nm_ref} if( defined $tbl_nm_ref ); } else { $sim_model = $sim_models[0] -> copy( filename => $self -> {'directory'}.'m'.$model_number.'/'.$sim_name, @@ -143,9 +149,27 @@ if ( not $done ) { $est_original -> ignore_missing_files( 1 ); $est_original -> outputfile( $self -> {'directory'}.'m'.$model_number.'/'.$orig_out ); $est_original -> ignore_missing_files( 0 ); + my @new_orig_table_names; + for( my $pr = 0; $pr <= $#orig_table_names; $pr++ ) { + for( my $tbl = 0; $tbl < scalar @{$orig_table_names[$pr]}; $tbl++ ) { + $new_orig_table_names[$pr][$tbl] = $orig_table_names[$pr][$tbl].'-'.$sim_no; + } + } + if( $#new_orig_table_names >= 0 ) { + $est_original -> table_names( new_names => \@new_orig_table_names ); + } $est_alternative -> ignore_missing_files( 1 ); $est_alternative -> outputfile( $self -> {'directory'}.'m'.$model_number.'/'.$alt_out ); $est_alternative -> ignore_missing_files( 0 ); + my @new_alt_table_names; + for( my $pr = 0; $pr <= $#alt_table_names; $pr++ ) { + for( my $tbl = 0; $tbl < scalar @{$alt_table_names[$pr]}; $tbl++ ) { + $new_alt_table_names[$pr][$tbl] = $alt_table_names[$pr][$tbl].'-'.$sim_no; + } + } + if( $#new_alt_table_names >= 0 ) { + $est_alternative -> table_names( new_names => \@new_alt_table_names ); + } if( $self -> shrinkage() ) { $est_original -> shrinkage_stats( enabled => 1 ); $est_alternative -> shrinkage_stats( enabled => 1 ); @@ -257,15 +281,19 @@ if ( not $done ) { filename => $orig_name, outputfile => $orig_out, extra_files => $model -> extra_files, - target => 'disk', - ignore_missing_files => 1 ); + target => 'mem', + ignore_missing_files => 1, + extra_output => $model -> extra_output(), + cwres => $model -> cwres() ); my $est_alternative = model -> new( directory => $model_dir, filename => $alt_name, outputfile => $alt_out, extra_files => $alternative -> extra_files, - target => 'disk', - ignore_missing_files => 1 ); + target => 'mem', + ignore_missing_files => 1, + extra_output => $model -> extra_output(), + cwres => $model -> cwres() ); push( @orig_est_models, $est_original ); push( @alt_est_models, $est_alternative ); my $nl = $j == $stored_samples ? "" : "\r"; diff --git a/lib/tool/modelfit_subs.pm b/lib/tool/modelfit_subs.pm index b24b801..50e7286 100644 --- a/lib/tool/modelfit_subs.pm +++ b/lib/tool/modelfit_subs.pm @@ -396,8 +396,14 @@ start prepare_raw_results # Find the model with most parameter of each kind and put the # number of the parameter in "max_hash". + # %max_hash should be initiated and filled in the constructor. + my ( %max_hash ); #, %category_names ); for ( my $i = 0; $i < scalar @{$self -> {'models'}}; $i++ ) { # models level + unless( $self -> {'models'} -> [$i] -> outputs -> [0] -> parsed ){ + $self -> {'models'} -> [$i] -> outputs -> [0] -> abort_on_fail(0); + $self -> {'models'} -> [$i] -> outputs -> [0] -> _read_problems; + } # Some column in "raw_results_header" are meta-columns, they # will be replaced by several columns. For example, the @@ -407,7 +413,7 @@ start prepare_raw_results # column of the raw_results_header to find the maximum # number of real columns. - foreach my $category ( @{$self -> {'raw_results_header'}},'npomega' ) { + foreach my $category ( @{$self -> {'raw_results_header'}},'npomega' ) { if ( $category eq 'setheta' or $category eq 'seomega' or $category eq 'sesigma' ){ next; } elsif ( $category eq 'theta' or $category eq 'omega' or $category eq 'sigma' or @@ -441,7 +447,6 @@ start prepare_raw_results my $labels; if( $self -> {'models'} -> [$i] -> outputs -> [0] -> parsed and $self -> {'models'} -> [$i] -> outputs -> [0] -> parsed_successfully ){ - $labels = $self -> {'models'} -> [$i] -> {'outputs'} -> [0] -> labels( parameter_type => $category ); } # we can't use labels directly since different models may have different @@ -470,13 +475,14 @@ start prepare_raw_results # {{{ values + # This method should be called with a model number and should + # update $self -> {'results_file'}. Through this it should be + # possible to print the raw_results_file (and other files as + # well) during the execution of the method. + my $pushed_rows = 0; # All rows pushed on result array in previous models my $raw_line_structure = ext::Config::Tiny -> new( ); for ( my $i = 0; $i < scalar @{$self -> {'models'}}; $i++ ) { # models level - unless( $self -> {'models'} -> [$i] -> outputs -> [0] -> parsed ){ - $self -> {'models'} -> [$i] -> outputs -> [0] -> abort_on_fail(0); - $self -> {'models'} -> [$i] -> outputs -> [0] -> _read_problems; - } if( $self -> {'models'} -> [$i] -> outputs -> [0] -> parsed_successfully ){ my @probs = @{$self -> {'models'} -> [$i] -> {'outputs'} -> [0] -> problem_structure}; my $np = scalar @probs; # #probs @@ -513,7 +519,8 @@ start prepare_raw_results defined $prob -> [0] and ref $prob -> [0] eq 'ARRAY' and defined $self -> {'raw_results'} and defined $self -> {'raw_results'} -> [$i+$prob_num] ){ - $raw_line_structure -> { $i+$prob_num }{ $category } = scalar @{$self -> {'raw_results'} -> [$i+$prob_num]} . ",". scalar @{$prob -> [0]}; + my $num = scalar @{$self -> {'raw_results'} -> [$i+$prob_num]} . ",". scalar @{$prob -> [0]}; + $raw_line_structure -> { $i+$prob_num }{ $category } = $num; } $prob_num++; } @@ -602,7 +609,7 @@ start prepare_raw_results my $row = $pushed_rows+1; push( @{$self -> {'raw_results'} -> [$pushed_rows++]}, ($i+1). ",run failed - Could not parse the output file: ". - $self -> {'models'} -> [$i] -> {'outputs'} -> [0] -> filename ); + $self -> {'models'} -> [$i] -> {'outputs'} -> [0] -> filename ); } } $raw_line_structure -> write( 'raw_file_structure' ); @@ -745,7 +752,7 @@ start copy_model_and_input } else { debug -> die( message => 'No datafiles set in modelfile.' ); } - + # copy the msfi files if( defined $model -> msfi_names() ){ foreach my $msfi_files( @{$model -> msfi_names()} ){ @@ -757,6 +764,16 @@ start copy_model_and_input } } + # copy the msfo files if msfo_resume + if( $self -> {'msfo_resume'} ){ + foreach my $msfo_files( @{$model -> msfo_names()} ){ + foreach my $msfo_file( @{$msfo_files} ){ + my ( $dir, $filename ) = OSspecific::absolute_path($model -> directory, + $msfo_file ); + cp( $dir.$filename, $filename ); + } + } + } # Copy extra data files to temporary dirctory and set short version names my @new_extra_data_names; my @problems = @{$model -> problems}; @@ -843,8 +860,18 @@ start copy_model_and_output { my $outfilename = $model -> outputs -> [0] -> full_name; + my ($dir, $model_filename) = OSspecific::absolute_path($model -> directory, + $model -> filename ); + + # This is used with 'prepend_model_file_name' + my $dotless_model_filename = $model_filename; + $dotless_model_filename =~ s/\.[^.]+$//; + + # Copy $use_run files to files without numbers, to avoid confusion. + cp( "psn-$use_run.lst", 'psn.lst' ) if ( $use_run ne '' ); cp( "psn.lst", $outfilename ); + if( defined $model -> table_names ){ foreach my $table_files( @{$model -> table_names} ){ foreach my $table_file( @{$table_files} ){ @@ -852,29 +879,62 @@ start copy_model_and_output } } } + my $final_output = output -> new( filename => $outfilename, model_id => $model -> model_id ); $final_output -> register_in_database( model_id => $model -> model_id, force => 1 ); # If we are here, the model has been run $final_model -> outputs( [$final_output] ); + if( defined $model -> msfo_names() ){ foreach my $msfo_files( @{$model -> msfo_names()} ){ foreach my $msfo_file( @{$msfo_files} ){ my ( $dir, $filename ) = OSspecific::absolute_path($model -> directory, $msfo_file ); - cp( $filename, $dir.$filename ); + if( $self -> {'prepend_model_file_name'} ) { + cp( $filename, $dir.$dotless_model_filename.'.'.$filename ); + } else { + cp( $filename, $dir.$filename ); + } } } - $final_model -> msfo_names( new_names => $model -> msfo_names() ); + #$final_model -> msfo_names( new_names => $model -> msfo_names() ); } - + if( defined $model -> table_names ){ foreach my $table_files( @{$model -> table_names} ){ foreach my $table_file( @{$table_files} ){ my ( $dir, $filename ) = OSspecific::absolute_path($model -> directory, $table_file ); - cp( $filename, $dir.$filename ); + if( $self -> unwrap_table_files() ) { + open( TABLE, '<'.$filename ); + my @table = ; + close( TABLE ); + open( TABLE, '>'.$filename ); + my ( $j, $cont_column ) = ( 0, -1 ); + for( my $i = 0; $i <= $#table; $i++ ) { + print TABLE $table[$i] and next if( $i == 0 ); + chomp($table[$i]); + my @row = split(' ',$table[$i]); + if( $i == 1 ) { + for( my $k = 0; $k <= $#row; $k++ ) { + $cont_column = $k if( $row[$k] eq 'CONT' ); + } + } + for( my $k = 0; $k <= $#row; $k++ ) { + next if( $k == $cont_column ); + print TABLE sprintf( "%12s",$row[$k] ); + } + print TABLE "\n"; + } + close( TABLE ); + } + if( $self -> {'prepend_model_file_name'} ) { + cp( $filename, $dir.$dotless_model_filename.'.'.$filename ); + } else { + cp( $filename, $dir.$filename ); + } } } $final_model -> table_names( new_names => $model -> table_names ); @@ -882,9 +942,11 @@ start copy_model_and_output if( defined $model -> {'extra_output'} ){ foreach my $extra_out ( @{$model -> {'extra_output'}} ){ - my ( $dir, $filename ) = OSspecific::absolute_path($model -> directory, - $model -> filename ); - cp( $extra_out, $dir.$filename.'.'.$extra_out ); + if( $self -> {'prepend_model_file_name'} ) { + cp( $extra_out.'-'.$use_run, $dir.$dotless_model_filename.'.'.$extra_out ); + } else { + cp( $extra_out.'-'.$use_run, $dir.$extra_out ); + } } } @@ -895,11 +957,22 @@ start copy_model_and_output my ( $dir, $eta_filename ) = OSspecific::absolute_path($model -> directory, $problems[$i] -> shrinkage_module -> eta_tablename ); - cp( $final_problems[$i] -> shrinkage_module -> eta_tablename, $dir.$eta_filename ); + + if( $self -> {'prepend_model_file_name'} ) { + cp( $final_problems[$i] -> shrinkage_module -> eta_tablename, $dir.$dotless_model_filename.'.'.$eta_filename ); + } else { + cp( $final_problems[$i] -> shrinkage_module -> eta_tablename, $dir.$eta_filename ); + } + my ( $dir, $wres_filename ) = OSspecific::absolute_path($model -> directory, $problems[$i] -> shrinkage_module -> wres_tablename ); - cp( $final_problems[$i] -> shrinkage_module -> wres_tablename, $dir.$wres_filename ); + + if( $self -> {'prepend_model_file_name'} ) { + cp( $final_problems[$i] -> shrinkage_module -> wres_tablename, $dir.$dotless_model_filename.'.'.$wres_filename ); + } else { + cp( $final_problems[$i] -> shrinkage_module -> wres_tablename, $dir.$wres_filename ); + } } } @@ -976,23 +1049,17 @@ start set_msfo_to_msfi "defined in model file " . $candidate_model -> filename ); } - my $msfo = @{$msfo_ref}->[0][0]; - $msfo =~ /(\d+)\Z/; - my $msfi = $msfo; - if ( defined ( $1 ) ) { - cp( 'psn.lst', 'psn.lst.msfo'.($1) ); - cp( $candidate_model -> filename, $candidate_model -> filename.'.msfo'.($1) ); - } - my $new_num = defined ( $1 ) ? $1+1 : 1; - $msfo =~ s/\d*(\Z)/$new_num$1/; - - $candidate_model->_option_val_pos( name => 'MSFO', - record_name => 'estimation', - problem_numbers => [[0]], - new_values => [[$msfo]]); + my $msfi = $msfo_ref -> [0][0]; $candidate_model->set_records(type=>'msfi', - record_strings => ["$msfo"]); + record_strings => ["$msfi"]); + + $candidate_model->remove_option(record_name => 'estimation', + option_name => 'MSFO'); + $candidate_model->remove_records(type=>'theta'); + $candidate_model->remove_records(type=>'omega'); + $candidate_model->remove_records(type=>'sigma'); + } end set_msfo_to_msfi @@ -1287,7 +1354,7 @@ start lsf_submit my ($lsf_out, $lsf_err); for( my $i = 1; $i <= 5; $i++ ){ - my $str = "bsub -e stderr -o stdout -K " . + my $str = "bsub -e stderr -o stdout " . "-q " . $self -> {'lsf_queue'} . ($self -> {'lsf_project_name'} ? " -P " . $self -> {'lsf_project_name'} : ' ') . ($self -> {'lsf_job_name'} ? " -J " . $self -> {'lsf_job_name'} : ' ') . @@ -1300,18 +1367,23 @@ start lsf_submit " psn.mod psn.lst " . $self -> {'nice'} . " ". $nm_version . " " . - ($self -> {'no_remote_compile'} ? 0 : 1) . " " . - ($self -> {'no_remote_execution'} ? 0 : 1) . " " . + 1 . ' ' . + 1 . ' ' . $fsubs . " " . $self -> {'nm_directory'} . "\""; run3( $str, undef, \$lsf_out, \$lsf_err ); + if ($lsf_out=~/Job \<(\d+)\> is submitted to queue/) { + $jobId=$1; + } + unless( $lsf_err =~ /System call failed/ or $lsf_err =~ /Bad argument/ or $lsf_err =~ /Request aborted by esub/ or $lsf_err =~ /Bad user ID/ ) { last; } + print "$lsf_err\n"; if( $lsf_err =~ /Bad argument/ or $lsf_err =~ /Request aborted by esub/ or @@ -1328,8 +1400,25 @@ end lsf_submit # }}} +# {{{ lsf_monitor + +start lsf_monitor + { + my ($stdout, $stderr); + run3("bjobs $jobId",undef,\$stdout, \$stderr ); + + if ($stdout=~/DONE/m) { + return $jobId; # Return the jobId found. + } + + return 0; + } +end lsf_monitor + +# }}} + # {{{ ud_submit - + start ud_submit { my $script; @@ -1353,7 +1442,7 @@ start ud_submit } end ud_submit - + # }}} # {{{ ud_monitor @@ -1583,8 +1672,8 @@ start sge_submit my $fsubs = join( ',' , @{$model -> subroutine_files} ); if( system( 'qsub -cwd -b y ' . - ($self -> {'sge_resource'} ? ' ' : '-l '.$self -> {'sge_resource'}.' '), - ($self -> {'sge_queue'} ? ' ' : '-q '.$self -> {'sge_queue'}.' '), + ($self -> {'sge_resource'} ? '-l '.$self -> {'sge_resource'}.' ' : ' ') . + ($self -> {'sge_queue'} ? '-q '.$self -> {'sge_queue'}.' ' : ' ') . ($PsN::config -> {'_'} -> {'remote_perl'} ? ' ' . $PsN::config -> {'_'} -> {'remote_perl'} : ' perl ') . " -I" . $PsN::lib_dir ."/../ " . $PsN::lib_dir . "/nonmem.pm" . @@ -1620,6 +1709,7 @@ start sge_monitor while( ){ if( /Following jobs do not exist:/ ){ # regexp to find finished jobs. close(JOBFILE); + unlink( "JobStat" ); return $jobId; # Return the jobId found. } } @@ -1638,7 +1728,7 @@ start run_nonmem { my $candidate_model = $queue_info -> {'candidate_model'}; my $tries = $queue_info -> {'tries'}; - my $compile_only = $queue_info -> {'compile_only'}; + # my $compile_only = $queue_info -> {'compile_only'}; my $model = $queue_info -> {'model'}; my $fprob = $candidate_model -> problems; @@ -1655,190 +1745,201 @@ start run_nonmem record_name => 'estimation', problem_numbers => [$i], new_values => [["msfo$i"]]); - } + } elsif( $self -> {'msfo_resume'} and -e $model_msfos -> [0][0] ){ + print "Resuming MSFO\n"; + $self -> set_msfo_to_msfi( candidate_model => $candidate_model ); + } } } $candidate_model->_write(); - my $nonmem; # We do not expect any values of rerun lower than 1 here. (a bug otherwise...) if( not -e 'psn-' . ( $tries + 1 ) . '.lst' or $self -> {'rerun'} >= 2 ){ # {{{ Compilation step (local or remote) - if( $queue_info -> {'modelfile_tainted'} ){ # We need to recompile - # -------------- Notes about local vs remote compilation/execution ----------------- - - # Here we check wheter compilation and/or exection will be done - # localy or remote. (Remote compilation is only supported on LSF - # so far). So, unless we are doing everything remotely, we will - # need a "nonmem" object. Which is created here. - - if( ( not $self -> {'run_local'} ) and - ( $self -> {'no_remote_compile'} or - $self -> {'no_remote_execution'} ) ){ - - $nonmem = nonmem -> new( adaptive => $self -> {'adaptive'}, - modelfile => 'psn.mod', - version => $nm_version, - nm_directory => $self -> {'nm_directory'}, - nice => $self -> {'nice'}, - show_version => not ( $self -> {'run_on_lsf'} ) ); - - $nonmem -> fsubs( $candidate_model -> subroutine_files ); - } +# if( $queue_info -> {'modelfile_tainted'} ){ # We need to recompile - if( ( $self -> {'run_local'} and $compile_only ) or - ( not $self -> {'run_local'} and $self -> {'no_remote_compile'} ) ){ - - # {{{ local compile requested - - my $fsubs = join( ',' , @{$model -> subroutine_files} ); - my ($stdin,$stdout,$stderr); - my $pid = fork(); - if( $pid == 0 ){ - exec( ($PsN::config -> {'_'} -> {'remote_perl'} ? ' ' . $PsN::config -> {'_'} -> {'remote_perl'} : ' perl ') . " -I" . - $PsN::lib_dir ."/../ " . - $PsN::lib_dir . "/nonmem.pm" . - " psn.mod psn.lst " . - $self -> {'nice'} . " ". - $nm_version . " " . - 1 . " " . # compilation - 0 . " " . # no execution - $fsubs . " " . - $self -> {'nm_directory'} ); - } - - $queue_info->{'compile_only'} = 1; - $queue_map->{$pid} = $run_no; +# if( ( $self -> {'run_local'} and $compile_only ) or +# ( not $self -> {'run_local'} and $self -> {'no_remote_compile'} ) ){ + +# # {{{ local compile requested + +# my $fsubs = join( ',' , @{$model -> subroutine_files} ); + +# my $pid = fork(); +# if( $pid == 0 ){ +# exec( ($PsN::config -> {'_'} -> {'remote_perl'} ? ' ' . $PsN::config -> {'_'} -> {'remote_perl'} : ' perl ') . " -I" . +# $PsN::lib_dir ."/../ " . +# $PsN::lib_dir . "/nonmem.pm" . +# " psn.mod psn.lst " . +# $self -> {'nice'} . " ". +# $nm_version . " " . +# 1 . " " . # compilation +# 0 . " " . # no execution +# $fsubs . " " . +# $self -> {'nm_directory'} ); +# } - #unless( $nonmem -> compile() ){ - # debug -> die( message => "NONMEM compilation of ". $model -> full_name . " failed:\n" . $nonmem -> error_message ); - #} +# $queue_info->{'compile_only'} = 1; +# $queue_map->{$pid} = $run_no; - #unless ( -e 'FCON.orig' ) { - # unless( $self -> {'clean'} ){ - - # Instead of cleaning its better to never create - # garbage :) However, if FCON pertubation becomes - # reality, we will need FCON.orig. - - # cp( 'FCON', 'FCON.orig' ); - # } - #} - - #if( $nonmem -> nmtran_message =~ /WARNING/s and $self -> {'verbose'} ){ - # ui -> print( category => 'all', - # message => "NONMEM Warning:\n" . $nonmem -> nmtran_message ); - #} - - $queue_info->{'modelfile_tainted'} = 0; +# $queue_info->{'modelfile_tainted'} = 0; - # }}} +# # }}} - } elsif ( $self -> run_on_lsf() and $self -> no_remote_execution() ) { +# } elsif ( $self -> run_on_lsf() and $self -> no_remote_execution() ) { - # lsf_submit will call the "nonmem" module that will - # figure out that we want to compile remotely. If we want - # to compile AND run remotely that will be done in the - # call to "lsf_submit" below. +# # lsf_submit will call the "nonmem" module that will +# # figure out that we want to compile remotely. If we want +# # to compile AND run remotely that will be done in the +# # call to "lsf_submit" below. - $self -> lsf_submit( model => $candidate_model, - nm_version => $nm_version, - work_dir => $self -> {'directory'} . "/NM_run$run_no/"); +# $self -> lsf_submit( model => $candidate_model, +# nm_version => $nm_version, +# work_dir => $self -> {'directory'} . "/NM_run$run_no/"); - #### OBS! Måste skrivas om så att lsf_submit inte väntar på att jobbet ska komma tillbaks +# #### OBS! Måste skrivas om så att lsf_submit inte väntar på att jobbet ska komma tillbaks - } - } +# } +# } # }}} # {{{ Execution step (local or remote) - if( ( $self -> {'run_local'} and not $compile_only ) or - ( not $self -> {'run_local'} and $self -> {'no_remote_execution'} ) ){ + if( $self -> {'run_local'} ) { + # and not $compile_only ) or + # ( not $self -> {'run_local'} and $self -> {'no_remote_execution'} ) ){ - # Normal local execution + # Normal local execution - open( NMMSG , "; - my $compilation_message = join( '',@tmp ); + # {{{ read and print compilation problems - unless( $compilation_message =~ /Compilation done/ ){ - debug -> warn( message => "NONMEM compilation of ". $model -> full_name . " failed:\n" . $compilation_message ); - } else { - if( $compilation_message =~ /WARNING/s and $self -> {'verbose'} ){ - ui -> print( category => 'all', - message => "NONMEM Warning:\n" . $compilation_message ); - } - } + # I'm removing this as compilation/execution is done in one sweep these days. + +# open( NMMSG , "; +# my $compilation_message = join( '',@tmp ); + +# unless( $compilation_message =~ /Compilation done/ ){ +# debug -> warn( message => "NONMEM compilation of ". $model -> full_name . " failed:\n" . $compilation_message ); +# } else { +# if( $compilation_message =~ /WARNING/s and $self -> {'verbose'} ){ +# ui -> print( category => 'all', +# message => "NONMEM Warning:\n" . $compilation_message ); +# } +# } + + # }}} + my $fsubs = join( ',' , @{$model -> subroutine_files} ); - my ($stdin,$stdout,$stderr); - my $pid = fork(); - if( $pid == 0 ){ - exec( ($PsN::config -> {'_'} -> {'remote_perl'} ? ' ' . $PsN::config -> {'_'} -> {'remote_perl'} : ' perl ') . " -I" . - $PsN::lib_dir ."/../ " . - $PsN::lib_dir . "/nonmem.pm" . - " psn.mod psn.lst " . - $self -> {'nice'} . " ". - $nm_version . " " . - 0 . " " . # no compilation - 1 . " " . # execution - $fsubs . " " . - $self -> {'nm_directory'} ); - } + + my $command_line_options = " -I" . + $PsN::lib_dir ."/../ " . + $PsN::lib_dir . "/nonmem.pm" . + " psn.mod psn.lst " . + $self -> {'nice'} . " ". + $nm_version . " " . + 1 . " " . # compilation + 1 . " " . # execution + $fsubs . " " . + $self -> {'nm_directory'} ; + + if( $Config{osname} eq 'MSWin32' ){ + + my $perl_bin = ($PsN::config -> {'_'} -> {'perl'} ? $PsN::config -> {'_'} -> {'perl'} : 'C:\Perl\bin\perl.exe'); + + require Win32::Process; + require Win32; + sub ErrorReport{ print Win32::FormatMessage( Win32::GetLastError() ); } + my $proc; + Win32::Process::Create($proc,$perl_bin,$perl_bin . $command_line_options,0,$Win32::Process::NORMAL_PRIORITY_CLASS,'.') || die ErrorReport(); + + $queue_info->{'winproc'}=$proc; + $queue_map->{$proc->GetProcessID()} = $run_no; + + } else { #Asume *nix + + my $perl_bin = ($PsN::config -> {'_'} -> {'perl'} ? $PsN::config -> {'_'} -> {'perl'} : ' perl '); + + my $pid = fork(); + if( $pid == 0 ){ + exec( $perl_bin . $command_line_options ); + die "Unable to start nonmem.pm\n"; + } + $queue_map->{$pid} = $run_no; - $queue_info->{'compile_only'} = 0; - $queue_map->{$pid} = $run_no; + } + # $queue_info->{'compile_only'} = 0; + } elsif( $self -> {'run_on_lsf'} ) { # lsf_submit will call the "nonmem" module that will figure # out that we want to run remotely. If we are also compiling # remotely, it will be done from here as well. - $self -> lsf_submit( model => $candidate_model, - nm_version => $nm_version, - work_dir => $self -> {'directory'} . "/NM_run$run_no/"); - #### OBS! Måste skrivas om så att lsf_submit inte väntar på att jobbet ska komma tillbaks + my $jobId = $self -> lsf_submit( model => $candidate_model, + nm_version => $nm_version, + work_dir => $self -> {'directory'} . "/NM_run$run_no/"); + + $queue_map->{$jobId} = $run_no; } elsif( $self -> {'run_on_umbrella'} ) { # umbrella_submit will submit a request for execution # using the umbrella system. - #sleep(1); - $self -> umbrella_submit( queue_info => $queue_info, - model => $candidate_model, - nm_version => $nm_version, - prepare_jobs => 1); - } elsif ( $self -> run_on_ud() and not $self -> no_remote_execution() ) { + #sleep(1); + $self -> umbrella_submit( queue_info => $queue_info, + model => $candidate_model, + nm_version => $nm_version, + prepare_jobs => 1); + } elsif ( $self -> run_on_ud() ) { + debug -> warn( level => 1, + message => "Submitting to the UD system" ); my $jobId = $self -> ud_submit( model => $candidate_model ); - - $queue_info->{'compile_only'} = 0; + + #$queue_info->{'compile_only'} = 0; $queue_map->{$jobId} = $run_no; - - } elsif ( $self -> run_on_sge() and not $self -> no_remote_execution() ){ + + } elsif ( $self -> run_on_sge() ) { my $jobId = $self -> sge_submit( model => $candidate_model, nm_version => $nm_version ); - - $queue_info->{'compile_only'} = 0; + + #$queue_info->{'compile_only'} = 0; $queue_map->{$jobId} = $run_no; } # }}} - + } elsif( $self -> {'rerun'} >= 1 ){ - + # We are not forcing a rerun, but we want to recheck the # output files for errors. Therefore we put a fake entry in - # queue_map to trigger "restart_nonmem()". - - $queue_info->{'compile_only'} = 0; - $queue_map->{'rerun_'.$run_no} = $run_no; #Fake pid + # queue_map to trigger "restart_nonmem()". We also need to + # copy psn-x.lst to psn.lst to make sure that we don't stop + # the next time we enter run_nonmem" + + cp( 'psn-'.($tries+1).'.lst','psn.lst' ); + if( defined $model -> table_names ){ + foreach my $table_files( @{$model -> table_names} ){ + foreach my $table_file( @{$table_files} ){ + cp( $table_file.'-'.( $tries+1 ), $table_file ); + } + } + } + if( defined $model -> extra_output() ){ + foreach my $file( @{$model -> extra_output()} ){ + cp( $file.'-'.( $tries+1 ), $file ); + } + } + $queue_info->{'compile_only'} = 0; + $queue_map->{'rerun_'.$run_no} = $run_no; #Fake pid } # end of "not -e psn-$tries.lst or rerun" + } end run_nonmem @@ -1860,7 +1961,7 @@ start restart_needed # estimates. Two modelfit members govern the pertubation process; # "retries" which is a number giving the maximum number of retries # when nonmem failes and "min_retries" which is the number of runs - # we want to do to get a global minima. If min_retries is 3 and + # we want to do to get a global minima. If min_retries is 2 and # retries is 5 we will stop after 3 runs if we have reached a # successful minimization but continue until 5 if necessary. @@ -1879,7 +1980,7 @@ start restart_needed unless( defined $parm{'queue_info'} ){ # The queue_info must be defined here! 'debug' -> die( message => "Internal run queue corrupt\n" ); - } + } my $queue_info_ref = $parm{'queue_info'}; my $run_results = $queue_info_ref -> {'run_results'}; my $tries = \$queue_info_ref -> {'tries'}; @@ -1948,22 +2049,23 @@ start restart_needed $run_results -> [${$tries}] -> {'failed'} = 1; return(0); } + if( not $output_file -> parsed_successfully() and - $queue_info_ref -> {'crashes'} <= $self -> restart_crashes() ) { + $queue_info_ref -> {'crashes'} <= $self -> crash_restarts() ) { # If the output file could not be parsed successfully, this is # a sign of a crashed run. This is not a NONMEM error as such # but probably an operating system problem. To handle this, we # mark this for rerunning but do not increase the $tries # variable but instead increase $crashes and check whether - # this value is below or equal to $restart_crashes. + # this value is below or equal to $crash_restarts. debug -> warn( level => 1, message => "Restarting crashed run ". $output_file -> full_name(). "\n".$output_file -> parsing_error_message() ); $queue_info_ref -> {'crashes'}++; - cp( 'psn-'.($tries+1).'.mod', 'psn.mod' ); - unlink( 'psn-'.($tries+1).'.lst' ); + cp( 'psn-'.(${$tries}+1).'.mod', 'psn.mod' ); + unlink( 'psn-'.(${$tries}+1).'.lst' ); return(1); # Return a one (1) to make run() rerun the # model. By returning here, we avoid the # perturbation of the initial estimates later on in @@ -2008,13 +2110,10 @@ start restart_needed if ( /\s*MAX. NO. OF FUNCTION EVALUATIONS EXCEEDED\s*/) { if( $self -> {'use_implicit_msfo'} ) { + # Msfo to msfi will also clear initial estimates $self -> set_msfo_to_msfi( candidate_model => $candidate_model ); } - - $candidate_model->remove_records(type=>'theta'); - $candidate_model->remove_records(type=>'omega'); - $candidate_model->remove_records(type=>'sigma'); - + $candidate_model -> _write; ${$modelfile_tainted} = 1; $marked_for_rerun = 1; @@ -2189,13 +2288,13 @@ start restart_needed last; } else { sleep(($lsttry+1)**2); - print "The lst-file is not present, trying ",(5-$lsttry)," times more\n"; + print " The lst-file is not present, trying ".(5-$lsttry)." times more\n"; } # Did the lst file exist? } # The loop trying to read the lst file unless( $lstsuccess ) { # psn.lst doesn't exist. $run_results -> [${$tries}] -> {'failed'} = 1; } - + } end restart_needed @@ -2495,49 +2594,6 @@ end print_finish_message # }}} -# {{{ _print_done_file - -start _print_done_file -{ - open( DONE, ">./NM_run". ($run+1) ."/done" ); - - my @seed = random_get_seed; - my @evals = @{$evals_ref} if ( defined $evals_ref ); - - print DONE "This file indicates that PsN has run nonmem", - "and tried its best to get it through using\n", - ($tries+1), " number of attempts\n"; - - print DONE "evals: @evals\n"; - - print DONE "seed: @seed\n"; - - close( DONE ); -} -end _print_done_file - -# }}} - -# {{{ _read_done_file - -start _read_done_file -{ - open( DONE, "./NM_run". ($run+1) ."/done" ); - my @rows = ; - close( DONE ); - my $junk; - my ( $tries, $junk ) = split(' ',$rows[1],3); - my @evals = split(' ',$rows[2]); - my @seed = split(' ',$rows[3]); - shift( @evals ); # get rid of 'evals'-word - shift( @seed ); # get rid of 'seed'-word - random_set_seed( @seed ); - return \@evals; -} -end _read_done_file - -# }}} - # {{{ run start run @@ -2598,8 +2654,6 @@ start run # {{{ Local execution - my @evals; - # %queue_map is a mapping from nonmem.pm pID to run number. my %queue_map; @@ -2619,9 +2673,10 @@ start run # We loop while there is content in the queue (which shrinks and grows) # and while we have jobs running (represented in the queue_info) - my $qno = 0; + + my $poll_count = 0; while( @queue or (scalar keys %queue_map > 0) ){ - $qno++; + $poll_count++; # We start by looking for jobs that have been started and # finished. If we find one, we set $pid to that job. @@ -2632,18 +2687,18 @@ start run foreach my $check_pid( keys %queue_map ){ if( $check_pid =~ /^rerun_/ ){ - - # A pid that starts with "rerun_" is a rerun and is always + + # A pid that starts with "rerun" is a rerun and is always # "finished". $pid = $check_pid; last; } - + # Diffrent environments have diffrent ways of reporting # job status. Here we check which environment we are in # and act accordingly. - + if( $self -> {'run_on_ud'} ){ $pid = $self -> ud_monitor( jobId => $check_pid ); @@ -2652,35 +2707,71 @@ start run $self -> ud_retrieve( jobId => $check_pid, run_no => $queue_map{$check_pid} ); } - + } elsif( $self -> {'run_on_sge'} ) { - + $pid = $self -> sge_monitor( jobId => $check_pid ); + + } elsif( $self -> {'run_on_lsf'} ) { + + $pid = $self -> lsf_monitor( jobId => $check_pid ); } else { # Local process poll - $pid = waitpid($check_pid,WNOHANG); - if( $pid == -1 ){ + + if( $Config{osname} eq 'MSWin32' ){ + + my $exit_code; + + # GetExitCode is supposed to return a value indicating + # if the process is still running, however it seems to + # allways return 0. $exit_code however is update and + # seems to be nonzero if the process is running. + + $queue_info{$queue_map{$check_pid}}{'winproc'}->GetExitCode($exit_code); + + if( $exit_code == 0 ){ + $pid = $check_pid; + } + + } else { + + $pid = waitpid($check_pid,WNOHANG); - my $run = $queue_map{$check_pid}; + # Waitpid will return $check_pid if that process has + # finished and 0 if it is still running. - $queue_info{$run}{'nr_wait'}++; - if( $queue_info{$run}{'nr_wait'} > 10 ){ - debug -> die(message => "Nonmem run was lost\n"); + if( $pid == -1 ){ + # If waitpid return -1 the child has probably finished + # and has been "Reaped" by someone else. We treat this + # case as the child has finished. If this happens many + # times in the same NM_runX directory, there is probably + # something wrong and we die(). (I [PP] suspect that we + # never/seldom reach 10 reruns in one NM_runX directory) + + my $run = $queue_map{$check_pid}; + + $queue_info{$run}{'nr_wait'}++; + if( $queue_info{$run}{'nr_wait'} > 10 ){ + debug -> die(message => "Nonmem run was lost\n"); + } + $pid = $check_pid; } - $pid = $check_pid; + } + } last if $pid; - + } # }}} + + if( not $pid ){ - if( !$pid ){ + # No process has finished. - # No pid has finished. Check if queue is empty or we have - # $threads number of jobs running. + # {{{ Return to polling if queue is empty or we have max number of jobs running. if( (scalar @queue == 0) or scalar keys %queue_map >= $threads ){ @@ -2697,7 +2788,7 @@ start run if ( defined $PsN::config -> {'_'} -> {'polls_per_moshog_request'} and $self -> {'adaptive'} and $self -> {'threads'} > 1 and - not ( $qno % $PsN::config -> {'_'} -> {'polls_per_moshog_request'} ) ) { + not ( $poll_count % $PsN::config -> {'_'} -> {'polls_per_moshog_request'} ) ) { $make_request = 1; } @@ -2710,20 +2801,17 @@ start run $threads += $moshog_client -> granted(); } - - next; + + next; # Return to polling for finished jobs. } - } - # restart_needed will set $modelfile_tainted to 1 if a - # recompile is necessary. (run_nonmem will then recompile) - my $modelfile_tainted = 0; - - if( not $pid ) { + # }}} + # This is where we initiate a new job: + my $run = shift( @queue ); - # {{{ check for no run conditions. + # {{{ check for no run conditions. (e.g. job already run) if ( -e $self -> {'models'} -> [$run] -> outputs -> [0] -> full_name and @@ -2736,8 +2824,8 @@ start run # use execute to create a summary or run table). mkdir( "./NM_run" . ($run+1) ); - open( DONE, ">./NM_run". ($run+1) ."/done" ); - print DONE "This is faked\n 0 attepmts\nevals: 0\nseed: 1 1\n" ; + open( DONE, ">./NM_run". ($run+1) ."/done.1" ); + print DONE "This is faked\nseed: 1 1\n" ; close( DONE ); # TODO Must copy tablefiles if they exist. @@ -2748,12 +2836,6 @@ start run # TODO Should check for tablefiles. - # We read the "done" file to get seed and number of - # evaluations (which could be faked). - - my $evals_ref = $self -> _read_done_file(); - @evals = @{$evals_ref}; - my $modulus = (($#models+1) <= 10) ? 1 : (($#models+1) / 10)+1; if ( $run % $modulus == 0 or $run == 0 or $run == $#models ) { @@ -2762,11 +2844,12 @@ start run unless( $self -> {'parent_threads'} > 1 or $self -> {'verbose'} ); } - $queue_info{$run}{'candidate_model'} = model -> new( filename => "./NM_run".($run+1)."/psn.mod", - target => 'disk', - ignore_missing_files => 1, - quick_reload => 1 ); - + $queue_info{$run}{'candidate_model'} = + model -> new( filename => "./NM_run".($run+1)."/psn.mod", + target => 'disk', + ignore_missing_files => 1, + quick_reload => 1, + cwres => $models[$run] -> cwres() ); $self -> print_finish_message( candidate_model => $queue_info{$run}{'candidate_model'}, run => $run ); @@ -2774,16 +2857,16 @@ start run push( @{$self -> {'ntries'}}, 0 ); # push( @{$self -> {'evals'}}, \@evals ); - next; # We are done with this model. It has already been run. + next; # We are done with this model. It has already been run. Go back to polling. } # }}} + + # {{{ delay code (to avoid overload of new processes) if( $threads > 1 ){ - # {{{ delay code - if( $run > 1 ){ my $start_sleep = Time::HiRes::time(); @@ -2814,13 +2897,10 @@ start run Time::HiRes::usleep($min_sleep); } } - - # }}} } - - # Make sure that each process gets a unique random sequence: - random_set_seed_from_phrase(random_uniform_integer(1,0,10000*($run+1))); + + # }}} # {{{ Call to run_nonmem @@ -2832,14 +2912,16 @@ start run $self -> create_sub_dir( subDir => '/NM_run'.($run+1) ); chdir( 'NM_run'.($run+1) ); - + + # Initiate queue_info entry (unless its a restart) + unless( exists $queue_info{$run} ){ $queue_info{$run}{'candidate_model'} = $self -> copy_model_and_input( model => $models[$run], source => '../' ); $queue_info{$run}{'model'} = $models[$run]; $queue_info{$run}{'modelfile_tainted'} = 1; $queue_info{$run}{'tries'} = 0; $queue_info{$run}{'crashes'} = 0; - $queue_info{$run}{'compile_only'} = 1; + # $queue_info{$run}{'compile_only'} = 1; $queue_info{$run}{'run_results'} = []; # {{{ printing progress @@ -2867,11 +2949,6 @@ start run my %options_hash = %{$self -> _get_run_options(run_id => $run)}; - # This call will request a compile of nonmem. HOWEVER, if we - # are running on some remote system, it is perfectly ok for - # execution to be requested here as well. In that case the - # 'compile_only' flag must be cleared, which it is. - $self -> run_nonmem ( run_no => $run, nm_version => $options_hash{'nm_version'}, queue_info => $queue_info{$run}, @@ -2887,37 +2964,47 @@ start run # }}} - } else { + } else { + + # {{{ Here, a process has finished and we check for restarts. my $run = $queue_map{$pid}; - # This pid is finished and will not be needed again. - delete( $queue_map{$pid} ); - my $candidate_model = $queue_info{$run}{'candidate_model'}; - my %options_hash = %{$self -> _get_run_options(run_id => $run)}; - my $work_dir = 'NM_run' . ($run+1) ; chdir( $work_dir ); - if( $queue_info{$run}{'compile_only'} ){ - # If a pid finishes with the 'compile_only' flag set. It - # is time to run nonmem. - $queue_info{$run}{'compile_only'} = 0; - $self -> run_nonmem ( run_no => $run, - nm_version => $options_hash{'nm_version'}, - queue_info => $queue_info{$run}, - queue_map => \%queue_map ); - chdir( '..' ); - } else { +# if( $queue_info{$run}{'compile_only'} ){ +# $queue_info{$run}{'compile_only'} = 0; +# $self -> run_nonmem ( run_no => $run, +# nm_version => $options_hash{'nm_version'}, +# queue_info => $queue_info{$run}, +# queue_map => \%queue_map ); +# +# delete( $queue_map{$pid} ); +# chdir( '..' ); +# } else { $self -> compute_cwres( queue_info => $queue_info{$run}, run_no => $run ); + + $self -> compute_iofv( queue_info => $queue_info{$run}, + run_no => $run ); + + # Make sure that each process gets a unique random sequence: + my $tmpseed = defined $self->seed() ? $self->seed() : random_uniform_integer(1,1,99999999); + my $tmptry = exists $queue_info{$run}{'tries'} ? $queue_info{$run}{'tries'} : 0; + random_set_seed(($tmpseed+100000*($run+1)),($tmptry+1)); + + my %options_hash = %{$self -> _get_run_options(run_id => $run)}; + my $do_restart = $self -> restart_needed( %options_hash, queue_info => $queue_info{$run}, run_no => $run ); + if( $do_restart ){ unshift(@queue, $run); + delete($queue_map{$pid}); chdir( '..' ); } else { $self -> select_best_model(run_no => $run, @@ -2969,27 +3056,27 @@ start run unless ( $self -> {'parent_threads'} > 1 or $self -> {'verbose'} ); } } - + # }}} chdir( '..' ); - # {{{ cleaning and create "done" file - + # {{{ cleaning and done file + if( $self -> {'clean'} >= 3 ){ unlink( <$work_dir/*> ); unless( rmdir( $work_dir ) ){debug -> warn( message => "Unable to remove $work_dir directory: $! ." )}; sleep(2); } else { - unless( $queue_info{$run}{'tries'} > 0 ){ - $self -> _print_done_file( run => $run, - tries => $queue_info{$run}{'tries'}, - evals_ref => \@evals ); - } +# unless( $queue_info{$run}{'tries'} > 0 ){ +# $self -> _print_done_file( run => $run, +# tries => $queue_info{$run}{'tries'}, +# evals_ref => \@evals ); +# } } - + # }}} $self -> print_finish_message( candidate_model => $candidate_model, @@ -3001,11 +3088,14 @@ start run # push( @{$self -> {'evals'}}, \@evals ); } - # Here we are finished with this model and can forget - # all we know about its run. delete( $queue_info{$run} ); + delete( $queue_map{$pid} ); } - } + + next; +# } + # }}} + } } # queue loop ends here @@ -3559,10 +3649,10 @@ if( defined $PsN::config -> {'_'} -> {'R'} ) { foreach my $prob ( @{$probs} ) { if( $prob -> {'cwres_modules'} ){ my $sdno = $prob -> {'cwres_modules'} -> [0] -> sdno(); - + my $sim = $prob -> {'cwres_modules'} -> [0] -> mirror_plots ? ',sim.suffix="sim"' : ''; # Create the short R-script to compute the CWRES. open( RSCRIPT, ">compute_cwres.R" ); - print RSCRIPT "library(xpose4)\ncompute.cwres(\"cwtab$sdno\")\n"; + print RSCRIPT "library(xpose4)\ncompute.cwres($sdno $sim)\n"; close( RSCRIPT ); # Execute the script @@ -3571,6 +3661,22 @@ if( defined $PsN::config -> {'_'} -> {'R'} ) { } } } + end compute_cwres # }}} + +# {{{ compute_iofv + +start compute_iofv +{ + my $queue_info_ref = defined $parm{'queue_info'} ? $parm{'queue_info'} : {}; + my $model = $queue_info_ref -> {'model'}; + + if( defined $model -> iofv_modules ){ + $model -> iofv_modules -> [0] -> post_run_process; + } +} +end compute_iofv + +# }}} diff --git a/lib/tool/scm/config_file_subs.pm b/lib/tool/scm/config_file_subs.pm index 84bfadc..95cb4b4 100644 --- a/lib/tool/scm/config_file_subs.pm +++ b/lib/tool/scm/config_file_subs.pm @@ -269,11 +269,18 @@ start _check_various if( ($parm eq '*' or $cov eq '*') ) { if( length( $slave ) > 0 ){ - unless( defined $self -> {'relations'} -> {$parameter} -> {$covariate} -> {$master} -> {$state} -> {$slave} ){ + unless( exists $self -> {'relations'} -> {$parameter} and + exists $self -> {'relations'} -> {$parameter} -> {$covariate} and + exists $self -> {'relations'} -> {$parameter} -> {$covariate} -> {$master} and + exists $self -> {'relations'} -> {$parameter} -> {$covariate} -> {$master} -> {$state} and + exists $self -> {'relations'} -> {$parameter} -> {$covariate} -> {$master} -> {$state} -> {$slave} ){ @{$self -> {'relations'} -> {$parameter} -> {$covariate} -> {$master} -> {$state} -> {$slave}} = @bounds; } } else { - unless( defined $self -> {'relations'} -> {$parameter} -> {$covariate} -> {$master} -> {$state} ){ + unless( exists $self -> {'relations'} -> {$parameter} and + exists $self -> {'relations'} -> {$parameter} -> {$covariate} and + exists $self -> {'relations'} -> {$parameter} -> {$covariate} -> {$master} and + exists $self -> {'relations'} -> {$parameter} -> {$covariate} -> {$master} -> {$state} ){ @{$self -> {'relations'} -> {$parameter} -> {$covariate} -> {$master} -> {$state}} = @bounds; } } diff --git a/lib/tool/scm_subs.pm b/lib/tool/scm_subs.pm index d90dd7a..48b8b01 100644 --- a/lib/tool/scm_subs.pm +++ b/lib/tool/scm_subs.pm @@ -419,7 +419,8 @@ start new } $this -> {'relations'}[$i]{$par}{$cov}{'state'} = 1; foreach my $state ( @valid_states ) { - if ( defined $this -> {'included_relations'}[$i]{$par}{$cov} and + if ( exists $this -> {'included_relations'}[$i]{$par} and + exists $this -> {'included_relations'}[$i]{$par}{$cov} and $this -> {'included_relations'}[$i]{$par}{$cov}{'state'} == $state ) { $this -> {'included_relations'}[$i]{$par}{$cov}{'code'} = $this -> {'relations'}[$i]{$par}{$cov}{'code'}{$state}; @@ -524,7 +525,8 @@ start new statistics => $this -> {'covariate_statistics'}[$i]{$cov}, missing_data_token => $this -> {'missing_data_token'}, fix => $this -> {'fix'}[$i] ); - if ( defined $this -> {'included_relations'}[$i]{$par}{$cov} and + if ( exists $this -> {'included_relations'}[$i]{$par} and + exists $this -> {'included_relations'}[$i]{$par}{$cov} and $this -> {'included_relations'}[$i]{$par}{$cov}{'state'} == $state ) { $this -> {'included_relations'}[$i]{$par}{$cov}{'code'} = $this -> {'relations'}[$i]{$par}{$cov}{'code'}{$state}; diff --git a/lib/tool/xv_step_subs.pm b/lib/tool/xv_step_subs.pm index 96c3f68..d643662 100644 --- a/lib/tool/xv_step_subs.pm +++ b/lib/tool/xv_step_subs.pm @@ -162,11 +162,16 @@ start modelfit_setup target => 'mem' ); # TODO set max evals to zero for prediction. - $model_copy_pred -> _option_val_pos( record_name => 'estimation', - exact_match => 0, - name => 'MAX', - new_values => [[0]], - problem_numbers => [[0]]); +# $model_copy_pred -> _option_val_pos( record_name => 'estimation', +# exact_match => 0, +# name => 'MAX', +# new_values => [[0]], +# problem_numbers => [[0]]); + + $model_copy_pred -> set_option( record_name => 'estimation', + fuzzy_match => 1, + option_name => 'MAX', + option_value => 0 ); # Make sure changes is reflected on disk. $model_copy_pred -> _write(); diff --git a/lib/tool_subs.pm b/lib/tool_subs.pm index 51a8448..7d927c4 100644 --- a/lib/tool_subs.pm +++ b/lib/tool_subs.pm @@ -288,22 +288,18 @@ start register_in_database $PsN::config -> {'_'} -> {'password'}, {'RaiseError' => 1}); my $sth; - if ( defined $self -> {'parent_tool_id'} ) { # print "INSERT INTO tool (parent_tool_id,name,date,directory) ". # "VALUES (".$self -> {'parent_tool_id'}.", '". # "$tool_name', '$date_time', '$dir_str' )\n"; $sth = $dbh -> prepare("INSERT INTO ".$PsN::config -> {'_'} -> {'project'}. - ".tool (parent_tool_id,execute_id,cdd_id,date,directory) ". + ".tool (parent_tool_id,name,date,directory) ". "VALUES (".$self -> {'parent_tool_id'}.", '". - "$execute_id', '$cdd_id', '$date_time', '$dir_str' )"); + "$tool_name', '$date_time', '$dir_str' )"); } else { - #print("INSERT INTO ".$PsN::config -> {'_'} -> {'project'}. - # ".tool (execute_id,cdd_id,date,directory) ". - # "VALUES ('$execute_id','$cdd_id', '$date_time', '$dir_str' )"); $sth = $dbh -> prepare("INSERT INTO ".$PsN::config -> {'_'} -> {'project'}. - ".tool (execute_id,cdd_id,date,directory) ". - "VALUES ('$execute_id','$cdd_id', '$date_time', '$dir_str' )"); + ".tool (name,date,directory) ". + "VALUES ('$tool_name', '$date_time', '$dir_str' )"); } $sth -> execute; $self -> {'tool_id'} = $sth->{'mysql_insertid'}; @@ -722,7 +718,7 @@ start run $threads = $#models + 1 if ( $threads > $#models + 1); # Currently parallel execution is not supported on windows platforms - # $threads = 1 if( $Config{osname} eq 'MSWin32' ); + $threads = 1 if( $Config{osname} eq 'MSWin32' ); # Create new forkmanager my $pm = ext::Parallel::ForkManager -> new($threads) if ( $threads > 1 ); diff --git a/test/lib/scripts_test.pm b/test/lib/scripts_test.pm index b5901a0..032d5b4 100644 --- a/test/lib/scripts_test.pm +++ b/test/lib/scripts_test.pm @@ -179,18 +179,22 @@ sub generic_tester { $options = $options." -debug_package=$debug_package" if defined $debug_package; $options = $options." -debug_subroutine=$debug_subroutine" if defined $debug_subroutine; - system( &os_format( $directory . "/../bin/" . $script . " ". $options ) ); + system( &os_format( $directory . "/../bin/$script -directory=$script".'_dir'. $options ) ); my $success; SKIP: { if( $generate_facit ){ - system("cp $directory/".$script."_dir1/".$script."_results.csv $directory/files/utilities/" . $script . "_results.csv.".$sub_test ); + system("cp $directory/".$script."_dir/".$script."_results.csv $directory/files/utilities/" . $script . "_results.csv.".$sub_test ); system('rm -rf ' . $script .'_dir*'); skip "Generating facit", 1; } - $success = ok( -e "$directory/".$script."_dir1/".$script."_results.csv", "$script results file existance test" ); + + # Remove any previous (failed) test results. + system('rm -rf ' . $script .'_dir'); + + $success = ok( -e "$directory/".$script."_dir/".$script."_results.csv", "$script results file existance test" ); skip "Skipping contents test", 1 unless( $success ); - open( TESTRES, "$directory/" . $script ."_dir1/".$script."_results.csv" ); + open( TESTRES, "$directory/" . $script ."_dir/".$script."_results.csv" ); open( ORIGRES, "$directory/files/utilities/" . $script . "_results.csv.".$sub_test ); my @test_array = ; my @orig_array = ; -- 2.11.4.GIT