var/mri4dvar/run_mri3d4dvar.csh_lsf

   1 #!/bin/tcsh -f
   2
   3 #set echo
   4 set nonomatch
   5 set TOP_DIR=/glade/p/mmm/liuz/cwb2016
   6 set JOB='LSF'
   7 set PROJID='NMMM0015'
   8 set QUEUE='regular'
   9 set OS=`uname -s`
  10 set WRFDA_DIR=$TOP_DIR/liuz_newcode/WRFDA_V38
  11 set MULTI_INC_TOOLS=$TOP_DIR/liuz_newcode/multi_inc_tools_new
  12 set DATA_DIR=$TOP_DIR/20150614case/2015061400_ztd30min
  13
  14 set RUN_STAGE1=true
  15 set RUN_STAGE2=true
  16 set RUN_STAGE3=true
  17
  18 #---------------- User Configuration --------------
  19 set VAR4D=$1 # false if 3DVAR
  20 set VAR4D_LBC=false
  21 set MULTI_INC=$2
  22 set use_cvt=$3
  23 set use_vp=$4
  24 set observerclocktime=00:30
  25 set minimizeclocktime=03:00
  26 set WORK_DIR=$TOP_DIR/20150614case/$5
  27 set THIN_FACTOR=($6 $7)
  28 set BE1=$TOP_DIR/20150614case/be.dat_$8
  29 set BE2=$TOP_DIR/20150614case/be.dat_$9
  30 set BE3=$TOP_DIR/20150614case/be.dat_2km
  31 #----------------- User Configuration -------------
  32
  33 mkdir -p $WORK_DIR; cd $WORK_DIR
  34
  35 if ( $VAR4D == true ) then
  36  ln -sf ${WRFDA_DIR}/run/RRTM_DATA_DBL      RRTM_DATA
  37  ln -sf ${WRFDA_DIR}/run/RRTMG_LW_DATA_DBL  RRTMG_LW_DATA
  38  ln -sf ${WRFDA_DIR}/run/RRTMG_SW_DATA_DBL  RRTMG_SW_DATA
  39  ln -sf ${WRFDA_DIR}/run/SOILPARM.TBL       .
  40  ln -sf ${WRFDA_DIR}/run/VEGPARM.TBL        .
  41  ln -sf ${WRFDA_DIR}/run/GENPARM.TBL        .
  42 endif
  43 ln -sf ${WRFDA_DIR}/run/LANDUSE.TBL        .
  44 ln -sf ${WRFDA_DIR}/var/da/da_wrfvar.exe   .
  45
  46 cp ${DATA_DIR}/wrfinput_d01                 orig_fg
  47 if ( $VAR4D == true ) then
  48  cp ${DATA_DIR}/wrfinput_d01                orig_wrfinput_d01
  49  cp ${DATA_DIR}/wrfbdy_d01                  orig_wrfbdy_d01
  50  cp ${DATA_DIR}/fg02                        orig_fg02
  51 endif
  52
  53 if ( $VAR4D == true ) then
  54  cp ${DATA_DIR}/namelist.input_cv7_4dvar orig_namelist.input
  55  ln -sf ${DATA_DIR}/ob*.ascii           .
  56 else
  57  cp ${DATA_DIR}/namelist.input_cv7_3dvar orig_namelist.input
  58  ln -sf ${DATA_DIR}/ob01.ascii           ./ob.ascii
  59 endif
  60
  61 #=============================================================================;
  62 # Purpose     : Script for running WRF-3D/4DVAR with Multi-resolution
  63 #
  64 # Assuming    : 1. All the necessary files required by 4DVAR run are
  65 #                  already under $RUN_DIR, such as be.dat, namelist.input, *.tbl,
  66 #                  fg, fg02, wrfbdy_d01, da_wrfvar.exe, da_update_bc, etc.
  67 #
  68 #               2. da_bilin.exe, da_bdy.exe, da_thin.exe, da_vp_bilin.exe
  69 #                   da_vp_split.exe located under ${MULTI_INC_TOOLS}
  70 #
  71 # How to run  : When everything is ready to go, call this script instead of
  72 #               da_wrfvar.exe for a Multi-incremental run
  73 #
  74 #               run_mri3d4dvar.csh
  75 #
  76 # Limitation  : Grids need to match
  77 #                         ( n - 1 ) mod m = 0
  78 #               where n is the x/y grid number of high resolution, m is the
  79 #               x/y grid number of low resolution. Default ratio is 1:3.
  80 #
  81 # Platform    : All the commands involved by this script are GNU/Linux
  82 #               commands on CentOS box. If involved this script other than
  83 #               CentOS, commands may not run as your expect, double check
  84 #               it before using.
  85 #
  86 #               Not fullly test with all platforms, use it at your own risk
  87 #
  88 # jliu@ucar.edu, MMM/NCAR, 01/13/2012
  89 #
  90 # Remove RUN_STAGE
  91 # Add the capability of different resolutions for different outer-loops
  92 # xinzhang@ucar.edu, MMM/NCAR, 11/25/2013
  93 #
  94 # Re-write script and add more comments to ease understanding
  95 # Unify Multi-Resolution Incremental 3DVAR and 4DVAR, i.e., MRI-3D/4DVAR
  96 # Zhiquan (Jake) Liu, liuz@ucar.edu, NCAR/MMM, August 2016
  97 #=============================================================================;
  98
  99
 100 #----------------------------User settings------------------------------------;
 101 #
 102 # Use these environment variables to override the default settings
 103 #
 104 # Variable Names     Default Value                 Description
 105 #
 106 # RUN_CMD            mpirun -np 16   Job submit command, "" for serial and OpenMP
 107 #
 108 # TIME_STEP_STAGE2   auto detect     Stage2 Time step for integration in integer
 109 #                                    seconds as large as 6*DX (in km) and must be
 110 #                                    exactly divisible by VAR4D_bin exactly.
 111 #
 112 # RADT_STAGE2        auto detect     Minutes between radiation physics calls for
 113 #                                    Multi_inc stage2. 1 minute per km of dx.
 114 #
 115 # THIN_FACTOR        3               Thinning ratio
 116 #
 117 # MULTI_INC          TRUE            TRUE/FALSE - Multi-incremental/full resolution
 118 #
 119 # MAX_OUTERLOOP      1               outerloop number for Multi-incremental run
 120 #
 121 if ( ! $?RUN_CMD          ) set  RUN_CMD="mpiexec -n "    # "" - Serial/OpenMP
 122 if ( ! $?NPROCS_NL        ) set  NPROCS_NL=1024            # Number of processing cores
 123 if ( ! $?NPROCS           ) set  NPROCS=(1024 1024)          # Number of processing cores
 124 if ( ! $?MAX_OUTERLOOP    ) set  MAX_OUTERLOOP=2          # Only available for Multi-incremental run
 125 if ( ! $?THIN_FACTOR      ) set  THIN_FACTOR=(1 1)        # default decimation factor
 126    @ n1 = $MAX_OUTERLOOP
 127    @ n2 = $#THIN_FACTOR
 128    @ n3 = $#NPROCS
 129    if ( $n1 > $n2 || $n1 > $n3 ) then
 130       echo "The dimension of THIN_FACTOR ($#THIN_FACTOR) should be equal to MAX_OUTERLOOP ($MAX_OUTERLOOP) "
 131       exit -1
 132    endif
 133
 134 #----------------------------End of User settings-----------------------------;
 135
 136 if ( ! $?MULTI_INC_TOOLS ) then
 137   if ( ${#argv} > 0 ) then
 138     set MULTI_INC_TOOLS=$1
 139   else
 140     set appname=${0:t}
 141     set  MULTI_INC_TOOLS=${0:h}
 142     if ( "$MULTI_INC_TOOLS" == "$appname" ) set MULTI_INC_TOOLS="."
 143   endif
 144 endif
 145
 146 if ( $MULTI_INC == true ) then
 147
 148     foreach f (da_thin.exe da_bilin.exe da_bdy.exe  \
 149                da_vp_bilin.exe da_vp_split.exe \
 150               nc_vpglobal.ncl nc_vphires.ncl nc_increment.ncl )
 151       if ( -e ${MULTI_INC_TOOLS}/$f ) then
 152         if ( "$MULTI_INC_TOOLS" != "." ) then
 153           ln -sf ${MULTI_INC_TOOLS}/$f .
 154         endif
 155       else
 156         echo "$f NOT exists" > FAIL
 157         exit -1
 158       endif
 159     end
 160
 161     set FILES_TO_CLEAN=(ana02 ana02_hires ana02_lores \
 162                         wrfvar_output wrfvar_output_hires wrfvar_output_lores \
 163                         FAIL .current_stage .last_stage .final_stage \
 164                         namelist.input \
 165                         rsl* gts* vp_* outerloop_*)
 166
 167     #rm -rf $FILES_TO_CLEAN
 168
 169     touch .current_stage .last_stage .final_stage
 170
 171     set N=1
 172     #if  ( $VAR4D == true ) then
 173       @ nloop = $MAX_OUTERLOOP + 1
 174     #else
 175     #  @ nloop = $MAX_OUTERLOOP
 176     #endif
 177
 178   while ( $N <= $nloop )
 179
 180    if ( $N == $nloop ) then
 181       set RUN_STAGE1=true   # only run omb for the last loop
 182       set RUN_STAGE2=false
 183       set RUN_STAGE3=false
 184       set istage=1  # 1 for observer; 0 for normal 3dvar mode
 185       set nouterloop=1
 186       set ninnerloop=0
 187    else
 188       set RUN_STAGE1=true
 189       set RUN_STAGE2=true
 190       set RUN_STAGE3=true
 191       set istage=1
 192       set nouterloop=1
 193       set ninnerloop=40
 194    endif
 195
 196 #---------------- User Configuration --------------
 197    if ( $N == 1 ) ln -sf ${BE1} be.dat
 198    if ( $N == 2 ) ln -sf ${BE2} be.dat
 199    if ( $N == 3 ) ln -sf ${BE3} be.dat
 200 #---------------- User Configuration --------------
 201
 202    if ( $N == 1 ) then
 203       cp orig_fg     o${N}s1_fg # stage1 for observer step
 204      if ( $VAR4D == true ) then
 205       cp orig_fg02   o${N}s1_fg02
 206      endif
 207    else # from 2nd loop, use previous loop's analysis
 208       @ NM1 = $N - 1
 209       cp ./o${NM1}s3/wrfvar_output_hires o${N}s1_fg
 210      if ( $VAR4D == true ) then
 211       cp orig_fg02   o${N}s1_fg02
 212      endif
 213    endif
 214
 215     if ( $RUN_STAGE1 == "true" ) then
 216
 217     echo "--------------------------------------"
 218     echo "| Run outerloop-$N : Stage1-Observer |"
 219     echo "--------------------------------------"
 220
 221       echo "--------------------------------------"
 222       echo "| 1.0 Set up namlist.input for stage1 |"
 223       echo "--------------------------------------"
 224     #-----------------------------------
 225     #   &wrfvar1
 226     #     multi_inc=1, # stage1 for omb only
 227     #---------------------------------------
 228     if ( $OS == "Darwin" ) then
 229        sed -e "/[Mm][Uu][Ll][Tt][Ii]_[Ii][Nn][Cc][ \t]*=/ c\ \
 230            multi_inc=${istage}," \
 231            -e "/[m][a][x]_[e][x][t]_[i][t][s][ \t]*=/ c\ \
 232            max_ext_its=${nouterloop}," \
 233            -e "/[n][t][m][a][x][ \t]*=/ c\ \
 234            ntmax=${ninnerloop}," \
 235            orig_namelist.input > o${N}s1_namelist.input
 236     else
 237        sed -e "/[Mm][Uu][Ll][Tt][Ii]_[Ii][Nn][Cc][ \t]*=/ c\multi_inc=${istage}," \
 238            -e "/[m][a][x]_[e][x][t]_[i][t][s][ \t]*=/ c\max_ext_its=${nouterloop}," \
 239            -e "/[n][t][m][a][x][ \t]*=/ c\ntmax=${ninnerloop}," \
 240            orig_namelist.input > o${N}s1_namelist.input
 241     endif
 242
 243
 244         if ( $VAR4D == true ) then
 245          echo "--------------------------------------------"
 246          echo "| 1.1 Generating boundary file for stage1 |"
 247          echo "--------------------------------------------"
 248
 249          da_bdy.exe -fg   o${N}s1_fg         \
 250                     -fg02 o${N}s1_fg02       \
 251                     -bdy  orig_wrfbdy_d01    \
 252                     -o    o${N}s1_wrfbdy_d01 \
 253                     >&    o${N}s1_da_bdy.log
 254          if ( ! -e o${N}s1_wrfbdy_d01 ) then
 255            echo "generating bdy file for outloop1 stage1 failed" > FAIL
 256            exit -1
 257          endif
 258         endif
 259
 260       echo "--------------------------------------------"
 261       echo "| 1.2 run WRF-VAR in observer mode |"
 262       echo "--------------------------------------------"
 263
 264         if ( -e namelist.input ) rm namelist.input
 265         ln -sf o${N}s1_namelist.input  namelist.input
 266         ln -sf o${N}s1_fg          fg
 267         if ( $VAR4D == true ) then
 268          ln -sf o${N}s1_fg02        fg02
 269          ln -sf o${N}s1_fg          wrfinput_d01
 270          ln -sf o${N}s1_wrfbdy_d01  wrfbdy_d01
 271         endif
 272
 273         echo "outerloop${N}_stage1" > .current_stage
 274         echo "outerloop${N}_stage1" > .final_stage
 275
 276 #--------------------------------------
 277         if ( $JOB == 'LSF' ) then
 278            if ( -e submit_lsf.csh ) rm -f submit_lsf.csh
 279 cat > submit_lsf.csh <<EOF
 280 #!/bin/csh
 281 #
 282 ### LSF batch script to run an MPI application
 283 #
 284 #BSUB -P $PROJID             # project code
 285 #BSUB -W $observerclocktime       # wall-clock time (hrs:mins)
 286 #BSUB -n $NPROCS_NL          # number of tasks in job
 287 #BSUB -R "span[ptile=16]"    # run 16 MPI tasks per node
 288 #BSUB -J o${N}s1           # job name
 289 #BSUB -o o${N}s1.%J.out    # output file name in which %J is replaced by the job ID
 290 #BSUB -e o${N}s1.%J.err    # error file name in which %J is replaced by the job ID
 291 #BSUB -q $QUEUE              # queue
 292
 293 #run the executable
 294 mpirun.lsf ./da_wrfvar.exe
 295 EOF
 296            bsub -K < submit_lsf.csh
 297         else
 298            $RUN_CMD $NPROCS_NL ${PWD}/da_wrfvar.exe < /dev/null
 299         endif
 300 #------------------------------------------
 301
 302         if ( "`tail  rsl.out.0000 | grep -o -m 1 successfully`" != "successfully" ) then
 303           echo "da_wrfvar stage1 failed " > FAIL
 304           exit -1
 305         endif
 306
 307       echo "------------------------------"
 308       echo "| 1.3 Save and clean output |"
 309       echo "------------------------------"
 310
 311         set out_storage=o${N}s1
 312         if ( ! -d $out_storage ) mkdir $out_storage; #mkdir $out_storage/RSL
 313         mv submit_lsf.csh fort.140 namelist.input namelist.output* $out_storage
 314         mv rsl.* $out_storage
 315         #mv o${N}s1.*.out o${N}s1.*.err $out_storage
 316         rm rej_obs_conv* gts_omb_oma_*.* filtered_obs.* unpert_obs*
 317         rm analysis_increments wrfvar_output
 318         mv gts_omb.* $out_storage # to be used in stage2
 319         mv gts_omb_oma_* filtered_obs_* qcstat_conv_01 $out_storage
 320         mv buddy_check check_max_iv jo cost_fn grad_fn statistics $out_storage
 321
 322         echo "outerloop${N}_stage1" > .last_stage
 323
 324       if ( "`cat .current_stage`" != "outerloop${N}_stage1" && \
 325            "`cat .last_stage`"    != "outerloop${N}_stage1"  ) then
 326         echo "outerloop${N}_stage1 was NOT done, aborted Stage2"
 327         exit -1
 328       endif
 329   endif # end if RUN_STAGE1
 330
 331 #---------------------------------
 332   if ( $RUN_STAGE2 == "true" ) then
 333
 334       echo "--------------------------------------"
 335       echo "| Run outerloop-$N : Stage2-Minimizer |"
 336       echo "--------------------------------------"
 337
 338       echo "--------------------------------------------"
 339       echo "| 2.0 Set up namelist.input for stage2 |"
 340       echo "--------------------------------------------"
 341
 342     #---------------------------------------
 343     #   &wrfvar1
 344     #    multi_inc=2, # stage2 for minimization
 345     #    dx=
 346     #    dy=${NL_DY}.0,
 347     #    e_we=$NL_E_WE,
 348     #    e_sn=$NL_E_SN,
 349     #    time_step=$TIME_STEP_STAGE2,
 350     #    var4d_bin=
 351     #------------------------------------------
 352     set   NL_DX=`grep -i -E "dx[ \t]*="   orig_namelist.input | sed -e 's/\t/ /g' -e 's/  *//g' -e 's/,//' -e 's/\r//g'| cut -c4-`
 353     set   NL_DY=`grep -i -E "dy[ \t]*="   orig_namelist.input | sed -e 's/\t/ /g' -e 's/  *//g' -e 's/,//' -e 's/\r//g'| cut -c4-`
 354     set NL_E_WE=`grep -i -E "e_we[ \t]*=" orig_namelist.input | sed -e 's/\t/ /g' -e 's/  *//g' -e 's/,//' -e 's/\r//g'| cut -c6-`
 355     set NL_E_SN=`grep -i -E "e_sn[ \t]*=" orig_namelist.input | sed -e 's/\t/ /g' -e 's/  *//g' -e 's/,//' -e 's/\r//g'| cut -c6-`
 356
 357     set   NL_DX=`echo $NL_DX | sed 's/\.[0-9]*//g'`
 358     set   NL_DY=`echo $NL_DY | sed 's/\.[0-9]*//g'`
 359     set NL_E_WE=`echo $NL_E_WE | sed 's/\.[0-9]*//g'`
 360     set NL_E_SN=`echo $NL_E_SN | sed 's/\.[0-9]*//g'`
 361
 362     @   NL_DX = $NL_DX * $THIN_FACTOR[$N]
 363     @   NL_DY = $NL_DY * $THIN_FACTOR[$N]
 364     @ NL_E_WE = ($NL_E_WE - 1) / $THIN_FACTOR[$N] + 1
 365     @ NL_E_SN = ($NL_E_SN - 1) / $THIN_FACTOR[$N] + 1
 366
 367     @   RADT_STAGE2 = $NL_DX / 1000
 368
 369     set NL_VAR4D_BIN=`grep -i -E "var4d_bin[ \t]*=" orig_namelist.input | \
 370                       sed -e 's/\t/ /g' -e 's/  *//g' \
 371                           -e 's/\.[0-9]*//g' -e 's/,//' \
 372                           -e 's/\r//g' | \
 373                       cut -c11-`
 374     @ TIME_STEP_STAGE2 = ( $NL_DX / 1000 ) * 6
 375     set i=$TIME_STEP_STAGE2
 376     while ( $i != 0 )
 377       @ i = $NL_VAR4D_BIN % $TIME_STEP_STAGE2
 378       @ TIME_STEP_STAGE2--
 379     end
 380     @ TIME_STEP_STAGE2++
 381
 382     if ( $OS == "Darwin" ) then
 383        sed -e '/[Mm][Uu][Ll][Tt][Ii]_[Ii][Nn][Cc]/d' \
 384            -e '/&[Ww][Rr][Ff][Vv][Aa][Rr]1$/ a\ \
 385            multi_inc=2,' \
 386            -e "/[Dd][Xx][ \t]*=/ c\ \
 387            dx=${NL_DX}.0," \
 388            -e "/[Dd][Yy][ \t]*=/ c\ \
 389            dy=${NL_DY}.0," \
 390            -e "/[Ee]_[Ww][Ee][ \t]*=/ c\ \
 391            e_we=$NL_E_WE,"\
 392            -e "/[Ee]_[Ss][Nn][ \t]*=/ c\ \
 393            e_sn=$NL_E_SN," \
 394            -e "/[Tt][Ii][Mm][Ee]_[Ss][Tt][Ee][Pp][ \t]*=/ c\ \
 395            time_step=$TIME_STEP_STAGE2," \
 396            -e "/[u][s][e]_[i][n][v][e][r][s][e]_[s][q][u][a][r][e][r][o][o][t][b][ \t]*=/ c\ \
 397            use_inverse_squarerootb=$use_vp," \
 398            -e "/[u][s][e]_[i][n][t][e][r][p][o][l][a][t][e]_[c][v][t][ \t]*=/ c\ \
 399            use_interpolate_cvt=$use_cvt," \
 400            orig_namelist.input > o${N}s2_namelist.input
 401     else
 402        sed -e '/[Mm][Uu][Ll][Tt][Ii]_[Ii][Nn][Cc]/d' \
 403            -e '/&[Ww][Rr][Ff][Vv][Aa][Rr]1$/ a\multi_inc=2,' \
 404            -e "/[Dd][Xx][ \t]*=/ c\dx=${NL_DX}.0," \
 405            -e "/[Dd][Yy][ \t]*=/ c\dy=${NL_DY}.0," \
 406            -e "/[Ee]_[Ww][Ee][ \t]*=/ c\e_we=$NL_E_WE," \
 407            -e "/[Ee]_[Ss][Nn][ \t]*=/ c\e_sn=$NL_E_SN," \
 408            -e "/[Tt][Ii][Mm][Ee]_[Ss][Tt][Ee][Pp][ \t]*=/ c\time_step=$TIME_STEP_STAGE2," \
 409            -e "/[u][s][e]_[i][n][v][e][r][s][e]_[s][q][u][a][r][e][r][o][o][t][b][ \t]*=/ c\use_inverse_squarerootb=$use_vp," \
 410            -e "/[u][s][e]_[i][n][t][e][r][p][o][l][a][t][e]_[c][v][t][ \t]*=/ c\use_interpolate_cvt=$use_cvt," \
 411            orig_namelist.input > o${N}s2_namelist.input
 412     endif
 413
 414       if ( $N > 1 ) then  # only do this step from outer loop 2
 415
 416          @ NM1 = $N - 1
 417
 418           if ( $use_vp == true || $use_cvt == true ) then
 419       echo "--------------------------------------------"
 420       echo "| 2.1 Scatter the global cvt or vp to PEs |"
 421       echo "--------------------------------------------"
 422
 423             ln -sf ./o${NM1}s3/vp_output.global_hires  .
 424             if ( ! -e vp_output.global_hires ) then
 425                echo "vp_output.global_hires is not found"
 426                exit -1
 427             endif
 428
 429 #-------------------------------------------
 430             if ( $JOB == 'LSF' ) then
 431                if ( -e submit_lsf.csh ) rm -f submit_lsf.csh
 432 cat > submit_lsf.csh <<EOF
 433 #!/bin/csh
 434 #
 435 ## LSF batch script to run an MPI application
 436 #
 437 #BSUB -P $PROJID             # project code
 438 #BSUB -W 00:05               # wall-clock time (hrs:mins)
 439 #BSUB -n $NPROCS[$N]         # number of tasks in job
 440 #BSUB -R "span[ptile=16]"    # run 16 MPI tasks per node
 441 #BSUB -J o${N}s2_split   # job name
 442 #BSUB -o o${N}s2_split.%J.out # output file name in which %J is replaced by the job ID
 443 #BSUB -e o${N}s2_split.%J.err # error file name in which %J is replaced by the job ID
 444 #BSUB -q $QUEUE              # queue
 445
 446 #run the executable
 447 mpirun.lsf ./da_vp_split.exe
 448 EOF
 449                bsub -K < submit_lsf.csh
 450             else
 451                $RUN_CMD $NPROCS[$N] ${PWD}/da_vp_split.exe < /dev/null
 452             endif
 453 #---------------------------------------------
 454            endif # end if ( $use_vp == true || $use_cvt == true )
 455
 456            if ( $use_vp == true || $use_cvt == true ) then
 457             foreach f ( vp_input.0* )
 458               if ( ! -e $f ) then
 459                 echo "$f NOT exists" > FAIL
 460                 exit -1
 461               endif
 462             end
 463            endif
 464
 465       endif     # end if N > 1
 466
 467      if ( $THIN_FACTOR[$N] > 1 ) then
 468       echo "--------------------------------------------"
 469       echo "| 2.2 Thin high-res guess to low-res |"
 470       echo "--------------------------------------------"
 471
 472         ./da_thin.exe -i o${N}s1_fg      \
 473                       -o o${N}s2_fg      \
 474                       -thin $THIN_FACTOR[$N] \
 475                       >& o${N}s2_thin_fg.log
 476         if ( ! -e o${N}s2_fg ) then
 477           echo "thinning fg failed" > FAIL
 478           exit -1
 479         endif
 480
 481         if ( $VAR4D == true ) then # 2nd level fg only for 4DVAR
 482          ./da_thin.exe -i o${N}s1_fg02    \
 483                        -o o${N}s2_fg02    \
 484                        -thin $THIN_FACTOR[$N] \
 485                        >>& o${N}s2_thin_fg02.log
 486          if ( ! -e o${N}s2_fg02 ) then
 487            echo "thinning fg02 failed" > FAIL
 488            exit -1
 489          endif
 490         endif
 491      else
 492         cp o${N}s1_fg   o${N}s2_fg
 493         if ( $VAR4D == true ) cp o${N}s1_fg02 o${N}s2_fg02
 494      endif
 495
 496      if ( $VAR4D == true ) then
 497       echo "--------------------------------------------"
 498       echo "| 2.3 Generating boundary file for stage2 |"
 499       echo "--------------------------------------------"
 500       da_bdy.exe -fg   o${N}s2_fg         \
 501                  -fg02 o${N}s2_fg02         \
 502                  -bdy  orig_wrfbdy_d01     \
 503                  -o    o${N}s2_wrfbdy_d01   \
 504                  >&    o${N}s2_bdy.log
 505       if ( ! -e o${N}s2_wrfbdy_d01 ) then
 506         echo "generating bdy file for outerloop$N stage2 failed" > FAIL
 507         exit -1
 508       endif
 509      endif
 510
 511       echo "--------------------------------------------"
 512       echo "| 2.4 Run WRF-VAR minimization at low-res |"
 513       echo "--------------------------------------------"
 514
 515       if ( -e namelist.input ) rm -f  namelist.input
 516       ln -sf  o${N}s2_namelist.input  namelist.input
 517       ln -sf  ./o${N}s1/gts_omb.*  .
 518       ln -sf  o${N}s2_fg          fg
 519       if ( $VAR4D == true ) then
 520        ln -sf  o${N}s2_fg02        fg02
 521        ln -sf  o${N}s2_fg          wrfinput_d01
 522        ln -sf  o${N}s2_wrfbdy_d01  wrfbdy_d01
 523       endif
 524
 525       echo "outerloop${N}_stage2" > .current_stage
 526       echo "outerloop${N}_stage2" > .last_stage
 527
 528 #---------------------------------
 529       if ( $JOB == 'LSF' ) then
 530          if ( -e submit_lsf.csh ) rm -f submit_lsf.csh
 531 cat > submit_lsf.csh <<EOF
 532 #!/bin/csh
 533 #
 534 ### LSF batch script to run an MPI application
 535 #
 536 #BSUB -P $PROJID             # project code
 537 #BSUB -W $minimizeclocktime  # wall-clock time (hrs:mins)
 538 #BSUB -n $NPROCS[$N]         # number of tasks in job
 539 #BSUB -R "span[ptile=16]"    # run 16 MPI tasks per node
 540 #BSUB -J o${N}s2           # job name
 541 #BSUB -o o${N}s2.%J.out    # output file name in which %J is replaced by the job ID
 542 #BSUB -e o${N}s2.%J.err    # error file name in which %J is replaced by the job ID
 543 #BSUB -q  $QUEUE             # queue
 544
 545 #run the executable
 546 mpirun.lsf ./da_wrfvar.exe
 547 EOF
 548          bsub -K < submit_lsf.csh
 549       else
 550          $RUN_CMD $NPROCS[$N] ${PWD}/da_wrfvar.exe < /dev/null
 551       endif
 552 #------------------------------------
 553
 554       if ( "`tail  rsl.out.0000 | grep -o -m 1 successfully`" != "successfully" ) then
 555         echo "da_wrfvar stage2_$N failed " > FAIL
 556         exit -1
 557       endif
 558
 559       echo "------------------------------"
 560       echo "| 2.5 Save and clean output |"
 561       echo "------------------------------"
 562
 563       if ( $use_vp == true ) ncl nc_vpglobal.ncl
 564       ncl nc_increment.ncl
 565
 566       set out_storage=o${N}s2
 567       if ( ! -d $out_storage ) mkdir $out_storage; mkdir $out_storage/RSL
 568       rm unpert* gts_omb.* gts_omb_oma_*.* filtered_obs.*
 569       mv buddy_check check_max_iv $out_storage
 570       mv rsl.* rej_obs* qcstat_conv_01 $out_storage #/RSL
 571       mv cost_fn grad_fn jo fort.* statistics $out_storage
 572       mv wrfvar_output namelist.output* $out_storage
 573       mv gts_omb_oma_*  filtered_obs_* submit_lsf.csh $out_storage
 574       mv analysis_increments*  $out_storage
 575       if ( $use_vp == true || $use_cvt == true ) then
 576         if ( $N > 1 ) rm vp_output.global_hires
 577         mv vp_input.* vv_input.* vv_after* vp_output.global* $out_storage
 578       endif
 579
 580   endif # end if RUN_STAGE2
 581
 582 #-----------------------------------
 583   if ( $RUN_STAGE3 == "true" ) then
 584
 585       echo "--------------------------------------"
 586       echo "| Run outerloop${N} : Stage3-Regrid |"
 587       echo "--------------------------------------"
 588
 589     @ NP1 = $N + 1
 590
 591     if ( "$VAR4D_LBC" == "t" || "$VAR4D_LBC" == "T" ) then
 592       # this is not well tested yet. turned it off
 593       echo "------------------------------------------------------------"
 594       echo "| 3.1 regridding in model space for 2nd time level (fg02) |"
 595       echo "------------------------------------------------------------"
 596       #  only if var4d_lbc=true.
 597       #  NOTE: interpolate to original resolution in model space
 598
 599         ./da_bilin.exe  -fg_lores o${N}s2_fg02 \
 600                         -fg_hires o${N}s1_fg02 \
 601                         -an_lores ana02       \
 602                         -ns       $THIN_FACTOR[$N]   \
 603                         -o        ana02_hires >& o${N}s2_bilin_fg02.log
 604         if ( ! -e ana02_hires ) then
 605           echo "regridding increment failed" > FAIL
 606           exit -1
 607         endif
 608         mv ana02  ana02_lores
 609     else
 610       echo "# Skipped, VAR4D_LBC=$VAR4D_LBC \n"
 611     endif
 612
 613       echo "------------------------------------------------------------"
 614       echo "| 3.1 regridding in model space for 1st time level (fg) |"
 615       echo "------------------------------------------------------------"
 616     #  wrfvar_output_hires = fg_stage1 + S (wrfvar_output_lores - fg_stage2)
 617     #----------------------------------------------------------------------------
 618     if ( $THIN_FACTOR[$N] > 1 ) then # regrid to model resolution
 619       ./da_bilin.exe  -fg_lores o${N}s2_fg          \
 620                       -fg_hires o${N}s1_fg          \
 621                       -an_lores ./o${N}s2/wrfvar_output    \
 622                       -ns       $THIN_FACTOR[$N] \
 623                       -o        wrfvar_output_hires >& o${N}s3_da_bilin.log
 624       if ( ! -e wrfvar_output_hires ) then
 625         echo "regridding increment failed" > FAIL
 626         exit -1
 627       endif
 628      else    # if DA res. is same as model res., no need for interpolation
 629       cp ./o${N}s2/wrfvar_output wrfvar_output_hires
 630      endif
 631
 632     if ( $N < $MAX_OUTERLOOP ) then # no need to do this for the last loop
 633     #------------------------------------------------------
 634     # interpolate vp to next outer loop's resolution
 635     #-------------------------------------------------------------
 636      if ( $use_vp == true || $use_cvt == true ) then
 637         if ( $THIN_FACTOR[$N] != $THIN_FACTOR[$NP1] ) then # only do this if res diff for two loops
 638            echo "---------------------------------------------"
 639            echo "| 3.2 regridding in control variable space  |"
 640            echo "---------------------------------------------"
 641            if ( $N < $MAX_OUTERLOOP ) then
 642                @ ratio = $THIN_FACTOR[$N] / $THIN_FACTOR[$NP1]
 643            else
 644                @ ratio = $THIN_FACTOR[$N] # / 1
 645            endif
 646
 647            if ( -e vp_output.global )  rm -f vp_output.global
 648            ln -sf ./o${N}s2/vp_output.global .
 649            ./da_vp_bilin.exe -ratio $ratio >& o${N}s3_vp_bilin.log
 650            if ( ! -e vp_output.global_hires ) then
 651               echo "vp_output.global_hires is not generated" > FAIL
 652               exit -1
 653            endif
 654         else # if resolution same, no interpolation needed.
 655            cp ./o${N}s2/vp_output.global vp_output.global_hires
 656         endif
 657         ncl nc_vphires.ncl
 658      endif
 659
 660       echo "---------------------------------"
 661       echo "| 3.3 Save and Clean results  |"
 662       echo "---------------------------------"
 663
 664       set out_storage=o${N}s3
 665       if ( ! -d $out_storage ) mkdir $out_storage
 666
 667       mv  wrfvar_output_hires $out_storage
 668       rm  namelist.input
 669
 670       if ( $use_vp == true || $use_cvt == true ) then
 671         mv vp_output.global_hires* $out_storage
 672         rm vp_output.global
 673       endif
 674
 675    endif # $N < $MAX_OUTERLOOP
 676
 677   endif ## end if RUN_STAGE3=true
 678
 679     @ N++
 680
 681   end # End of outerloop
 682
 683   rm fg fg02 wrfinput_d01 wrfbdy_d01
 684
 685 else  ## if NOT multi-resolution incremental 3D/4DVAR, no stop outer loop
 686
 687         ln -sf ${BE3}     be.dat
 688         ln -sf orig_fg    fg
 689         if ( $VAR4D == true ) then
 690           ln -sf     orig_wrfinput_d01  wrfinput_d01
 691           ln -sf     orig_wrfbdy_d01    wrfbdy_d01
 692         endif
 693
 694     if ( $OS == "Darwin" ) then
 695        sed -e "/[u][s][e]_[i][n][v][e][r][s][e]_[s][q][u][a][r][e][r][o][o][t][b][ \t]*=/ c\ \
 696            use_inverse_squarerootb=$use_vp," \
 697            -e "/[u][s][e]_[i][n][t][e][r][p][o][l][a][t][e]_[c][v][t][ \t]*=/ c\ \
 698            use_interpolate_cvt=$use_cvt," \
 699            orig_namelist.input > namelist.input
 700     else
 701        sed -e "/[u][s][e]_[i][n][v][e][r][s][e]_[s][q][u][a][r][e][r][o][o][t][b][ \t]*=/ c\use_inverse_squarerootb=$use_vp," \
 702            -e "/[u][s][e]_[i][n][t][e][r][p][o][l][a][t][e]_[c][v][t][ \t]*=/ c\use_interpolate_cvt=$use_cvt," \
 703            orig_namelist.input > namelist.input
 704     endif
 705
 706 #---------------------------------
 707     if ( $JOB == 'LSF' ) then
 708        if ( -e submit_lsf.csh ) rm -f submit_lsf.csh
 709 cat > submit_lsf.csh <<EOF
 710 #!/bin/csh
 711 #
 712 ### LSF batch script to run an MPI application
 713 #
 714 #BSUB -P $PROJID             # project code
 715 #BSUB -W 5:00 # ban  00:20                 # wall-clock time (hrs:mins)
 716 #BSUB -n 1024          # number of tasks in job
 717 #BSUB -R "span[ptile=16]"    # run 16 MPI tasks per node
 718 #BSUB -J wrfvar               # job name
 719 #BSUB -o wrfvar.%J.out        # output file name in which %J is replaced by the job ID
 720 #BSUB -e wrfvar.%J.err        # error file name in which %J is replaced by the job ID
 721 #BSUB -q premium  # regular               # queue
 722
 723 #run the executable
 724 mpirun.lsf ./da_wrfvar.exe
 725 EOF
 726        bsub -K < submit_lsf.csh
 727     else
 728        set N=1
 729        $RUN_CMD $NPROCS[$N] ${PWD}/da_wrfvar.exe < /dev/null
 730     endif
 731 #-----------------------------------
 732
 733       ln -sf $MULTI_INC_TOOLS/nc_increment.ncl .
 734       ncl nc_increment.ncl
 735
 736       set out_storage=$WORK_DIR
 737       if ( ! -d $out_storage ) mkdir $out_storage; mkdir $out_storage/RSL
 738       rm unpert* gts_omb_oma_*.* filtered_obs.*
 739       mv rsl.* rej_obs* qcstat_conv_01 $out_storage/RSL
 740
 741 endif
 742