var/mri4dvar/run_mri3d4dvar.csh_pbs

   1 #!/bin/tcsh -f
   2
   3 #set echo
   4 set nonomatch
   5 set TOP_DIR=/glade/p/mmm/liuz/cy_code/cwb2017
   6 set JOB='PBS' # LSF or PBS
   7 set PROJID='NMMM0015'
   8 set QUEUE='economy'
   9 set OS=`uname -s`
  10 set WRFDA_DIR=$TOP_DIR/../WRF_Github/wrf_myfork
  11 set MULTI_INC_TOOLS=$WRFDA_DIR/var/mri4dvar
  12 set DATA_DIR=$TOP_DIR/case_2015061400/2015061400_ztd30min_cloud_radar
  13
  14 set RUN_STAGE1=true
  15 set RUN_STAGE2=true
  16 set RUN_STAGE3=true
  17
  18 #---------------- User Configuration --------------
  19 set VAR4D=$1 # false if 3DVAR
  20 set VAR4D_LBC=false
  21 set MULTI_INC=$2
  22 set use_cvt=$3
  23 set use_vp=$4
  24 set observerclocktime=00:10:00
  25 set minimizeclocktime=00:35:00
  26 set WORK_DIR=$TOP_DIR/case_2015061400/$5
  27 set THIN_FACTOR=($6 $7)
  28 set BE1=$TOP_DIR/2km_domain/be.dat_$8
  29 set BE2=$TOP_DIR/2km_domain/be.dat_$9
  30 set BE3=$TOP_DIR/2km_domain/be.dat_2km
  31 #----------------- User Configuration -------------
  32
  33 mkdir -p $WORK_DIR; cd $WORK_DIR
  34
  35 if ( $VAR4D == true ) then
  36  ln -sf ${WRFDA_DIR}/run/RRTM_DATA_DBL      RRTM_DATA
  37  ln -sf ${WRFDA_DIR}/run/RRTMG_LW_DATA_DBL  RRTMG_LW_DATA
  38  ln -sf ${WRFDA_DIR}/run/RRTMG_SW_DATA_DBL  RRTMG_SW_DATA
  39  ln -sf ${WRFDA_DIR}/run/SOILPARM.TBL       .
  40  ln -sf ${WRFDA_DIR}/run/VEGPARM.TBL        .
  41  ln -sf ${WRFDA_DIR}/run/GENPARM.TBL        .
  42 endif
  43 ln -sf ${WRFDA_DIR}/run/LANDUSE.TBL        .
  44 ln -sf ${WRFDA_DIR}/var/da/da_wrfvar.exe   .
  45
  46 cp ${DATA_DIR}/wrfinput_d01                 orig_fg
  47 if ( $VAR4D == true ) then
  48  cp ${DATA_DIR}/wrfinput_d01                orig_wrfinput_d01
  49  cp ${DATA_DIR}/wrfbdy_d01                  orig_wrfbdy_d01
  50  cp ${DATA_DIR}/fg02                        orig_fg02
  51 endif
  52
  53 if ( $VAR4D == true ) then
  54  cp ${DATA_DIR}/namelist.input_cv7_4dvar orig_namelist.input
  55  ln -sf ${DATA_DIR}/ob*.ascii           .
  56  ln -sf ${DATA_DIR}/ob*.radar           .
  57 else
  58  cp ${DATA_DIR}/namelist.input_cv7_3dvar orig_namelist.input
  59  ln -sf ${DATA_DIR}/ob01.ascii           ./ob.ascii
  60 endif
  61
  62 #=============================================================================;
  63 # Purpose     : Script for running WRF-3D/4DVAR with Multi-resolution
  64 #
  65 # Assuming    : 1. All the necessary files required by 4DVAR run are
  66 #                  already under $RUN_DIR, such as be.dat, namelist.input, *.tbl,
  67 #                  fg, fg02, wrfbdy_d01, da_wrfvar.exe, da_update_bc, etc.
  68 #
  69 #               2. da_bilin.exe, da_bdy.exe, da_thin.exe, da_vp_bilin.exe
  70 #                   da_vp_split.exe located under ${MULTI_INC_TOOLS}
  71 #
  72 # How to run  : When everything is ready to go, call this script instead of
  73 #               da_wrfvar.exe for a Multi-incremental run
  74 #
  75 #               run_mri3d4dvar.csh
  76 #
  77 # Limitation  : Grids need to match
  78 #                         ( n - 1 ) mod m = 0
  79 #               where n is the x/y grid number of high resolution, m is the
  80 #               x/y grid number of low resolution. Default ratio is 1:3.
  81 #
  82 # Platform    : All the commands involved by this script are GNU/Linux
  83 #               commands on CentOS box. If involved this script other than
  84 #               CentOS, commands may not run as your expect, double check
  85 #               it before using.
  86 #
  87 #               Not fullly test with all platforms, use it at your own risk
  88 #
  89 # jliu@ucar.edu, MMM/NCAR, 01/13/2012
  90 #
  91 # Remove RUN_STAGE
  92 # Add the capability of different resolutions for different outer-loops
  93 # xinzhang@ucar.edu, MMM/NCAR, 11/25/2013
  94 #
  95 # Re-write script and add more comments to ease understanding
  96 # Unify Multi-Resolution Incremental 3DVAR and 4DVAR, i.e., MRI-3D/4DVAR
  97 # Zhiquan (Jake) Liu, liuz@ucar.edu, NCAR/MMM, August 2016
  98 #=============================================================================;
  99
 100
 101 #----------------------------User settings------------------------------------;
 102 #
 103 # Use these environment variables to override the default settings
 104 #
 105 # Variable Names     Default Value                 Description
 106 #
 107 # RUN_CMD            mpirun -np 16   Job submit command, "" for serial and OpenMP
 108 #
 109 # TIME_STEP_STAGE2   auto detect     Stage2 Time step for integration in integer
 110 #                                    seconds as large as 6*DX (in km) and must be
 111 #                                    exactly divisible by VAR4D_bin exactly.
 112 #
 113 # RADT_STAGE2        auto detect     Minutes between radiation physics calls for
 114 #                                    Multi_inc stage2. 1 minute per km of dx.
 115 #
 116 # THIN_FACTOR        3               Thinning ratio
 117 #
 118 # MULTI_INC          TRUE            TRUE/FALSE - Multi-incremental/full resolution
 119 #
 120 # MAX_OUTERLOOP      1               outerloop number for Multi-incremental run
 121 #
 122 if ( ! $?RUN_CMD          ) set  RUN_CMD="mpiexec -n "    # "" - Serial/OpenMP
 123 if ( ! $?NPROCS_NL        ) set  NPROCS_NL=512           # Number of processing cores
 124 if ( ! $?NNODES_NL        ) set  NNODES_NL=16       # Number of nodes
 125 if ( ! $?NPROCS           ) set  NPROCS=(512 512)       # Number of processing cores
 126 if ( ! $?NNODES           ) set  NNODES=(16 16)       # Number of nodes
 127 if ( ! $?MAX_OUTERLOOP    ) set  MAX_OUTERLOOP=2          # Only available for Multi-incremental run
 128 if ( ! $?THIN_FACTOR      ) set  THIN_FACTOR=(1 1)        # default decimation factor
 129    @ n1 = $MAX_OUTERLOOP
 130    @ n2 = $#THIN_FACTOR
 131    @ n3 = $#NPROCS
 132    if ( $n1 > $n2 || $n1 > $n3 ) then
 133       echo "The dimension of THIN_FACTOR ($#THIN_FACTOR) should be equal to MAX_OUTERLOOP ($MAX_OUTERLOOP) "
 134       exit -1
 135    endif
 136
 137 #----------------------------End of User settings-----------------------------;
 138
 139 if ( ! $?MULTI_INC_TOOLS ) then
 140   if ( ${#argv} > 0 ) then
 141     set MULTI_INC_TOOLS=$1
 142   else
 143     set appname=${0:t}
 144     set  MULTI_INC_TOOLS=${0:h}
 145     if ( "$MULTI_INC_TOOLS" == "$appname" ) set MULTI_INC_TOOLS="."
 146   endif
 147 endif
 148
 149 if ( $MULTI_INC == true ) then
 150
 151     foreach f (da_thin.exe da_bilin.exe da_bdy.exe  \
 152                da_vp_bilin.exe da_vp_split.exe \
 153               nc_vpglobal.ncl nc_vphires.ncl nc_increment.ncl )
 154       if ( -e ${MULTI_INC_TOOLS}/$f ) then
 155         if ( "$MULTI_INC_TOOLS" != "." ) then
 156           ln -sf ${MULTI_INC_TOOLS}/$f .
 157         endif
 158       else
 159         echo "$f NOT exists" > FAIL
 160         exit -1
 161       endif
 162     end
 163
 164     set FILES_TO_CLEAN=(ana02 ana02_hires ana02_lores \
 165                         wrfvar_output wrfvar_output_hires wrfvar_output_lores \
 166                         FAIL .current_stage .last_stage .final_stage \
 167                         namelist.input \
 168                         rsl* gts* vp_* outerloop_*)
 169
 170     #rm -rf $FILES_TO_CLEAN
 171
 172     touch .current_stage .last_stage .final_stage
 173
 174     set N=1
 175     #if  ( $VAR4D == true ) then
 176       @ nloop = $MAX_OUTERLOOP + 1
 177     #else
 178     #  @ nloop = $MAX_OUTERLOOP
 179     #endif
 180
 181   while ( $N <= $nloop )
 182
 183    if ( $N == $nloop ) then
 184       set RUN_STAGE1=true   # only run omb for the last loop
 185       set RUN_STAGE2=false
 186       set RUN_STAGE3=false
 187       set istage=1  # 1 for observer; 0 for normal 3dvar mode
 188       set nouterloop=1
 189       set ninnerloop=0
 190    else
 191       set RUN_STAGE1=true
 192       set RUN_STAGE2=true
 193       set RUN_STAGE3=true
 194       set istage=1
 195       set nouterloop=1
 196       set ninnerloop=30
 197    endif
 198
 199 #---------------- User Configuration --------------
 200    if ( $N == 1 ) ln -sf ${BE1} be.dat
 201    if ( $N == 2 ) ln -sf ${BE2} be.dat
 202    if ( $N == 3 ) ln -sf ${BE3} be.dat
 203 #---------------- User Configuration --------------
 204
 205    if ( $N == 1 ) then
 206       cp orig_fg     o${N}s1_fg # stage1 for observer step
 207      if ( $VAR4D == true ) then
 208       cp orig_fg02   o${N}s1_fg02
 209      endif
 210    else # from 2nd loop, use previous loop's analysis
 211       @ NM1 = $N - 1
 212       cp ./o${NM1}s3/wrfvar_output_hires o${N}s1_fg
 213      if ( $VAR4D == true ) then
 214       cp orig_fg02   o${N}s1_fg02
 215      endif
 216    endif
 217
 218     if ( $RUN_STAGE1 == "true" ) then
 219
 220     echo "--------------------------------------"
 221     echo "| Run outerloop-$N : Stage1-Observer |"
 222     echo "--------------------------------------"
 223
 224       echo "--------------------------------------"
 225       echo "| 1.0 Set up namlist.input for stage1 |"
 226       echo "--------------------------------------"
 227     #-----------------------------------
 228     #   &wrfvar1
 229     #     multi_inc=1, # stage1 for omb only
 230     #---------------------------------------
 231     if ( $OS == "Darwin" ) then
 232        sed -e "/[Mm][Uu][Ll][Tt][Ii]_[Ii][Nn][Cc][ \t]*=/ c\ \
 233            multi_inc=${istage}," \
 234            -e "/[m][a][x]_[e][x][t]_[i][t][s][ \t]*=/ c\ \
 235            max_ext_its=${nouterloop}," \
 236            -e "/[n][t][m][a][x][ \t]*=/ c\ \
 237            ntmax=${ninnerloop}," \
 238            orig_namelist.input > o${N}s1_namelist.input
 239     else
 240        sed -e "/[Mm][Uu][Ll][Tt][Ii]_[Ii][Nn][Cc][ \t]*=/ c\multi_inc=${istage}," \
 241            -e "/[m][a][x]_[e][x][t]_[i][t][s][ \t]*=/ c\max_ext_its=${nouterloop}," \
 242            -e "/[n][t][m][a][x][ \t]*=/ c\ntmax=${ninnerloop}," \
 243            orig_namelist.input > o${N}s1_namelist.input
 244     endif
 245
 246
 247         if ( $VAR4D == true ) then
 248          echo "--------------------------------------------"
 249          echo "| 1.1 Generating boundary file for stage1 |"
 250          echo "--------------------------------------------"
 251
 252          da_bdy.exe -fg   o${N}s1_fg         \
 253                     -fg02 o${N}s1_fg02       \
 254                     -bdy  orig_wrfbdy_d01    \
 255                     -o    o${N}s1_wrfbdy_d01 \
 256                     >&    o${N}s1_da_bdy.log
 257          if ( ! -e o${N}s1_wrfbdy_d01 ) then
 258            echo "generating bdy file for outloop1 stage1 failed" > FAIL
 259            exit -1
 260          endif
 261         endif
 262
 263       echo "--------------------------------------------"
 264       echo "| 1.2 run WRF-VAR in observer mode |"
 265       echo "--------------------------------------------"
 266
 267         if ( -e namelist.input ) rm namelist.input
 268         ln -sf o${N}s1_namelist.input  namelist.input
 269         ln -sf o${N}s1_fg          fg
 270         if ( $VAR4D == true ) then
 271          ln -sf o${N}s1_fg02        fg02
 272          ln -sf o${N}s1_fg          wrfinput_d01
 273          ln -sf o${N}s1_wrfbdy_d01  wrfbdy_d01
 274         endif
 275
 276         echo "outerloop${N}_stage1" > .current_stage
 277         echo "outerloop${N}_stage1" > .final_stage
 278
 279 #--------------------------------------
 280         if ( $JOB == 'LSF' ) then
 281            if ( -e submit_lsf.csh ) rm -f submit_lsf.csh
 282 cat > submit_lsf.csh <<EOF
 283 #!/bin/csh
 284 #
 285 ### LSF batch script to run an MPI application
 286 #
 287 #BSUB -P $PROJID             # project code
 288 #BSUB -W $observerclocktime       # wall-clock time (hrs:mins)
 289 #BSUB -n $NPROCS_NL          # number of tasks in job
 290 #BSUB -R "span[ptile=16]"    # run 16 MPI tasks per node
 291 #BSUB -J o${N}s1           # job name
 292 #BSUB -o o${N}s1.%J.out    # output file name in which %J is replaced by the job ID
 293 #BSUB -e o${N}s1.%J.err    # error file name in which %J is replaced by the job ID
 294 #BSUB -q $QUEUE              # queue
 295
 296 #run the executable
 297 mpirun.lsf ./da_wrfvar.exe
 298 EOF
 299            bsub -K < submit_lsf.csh
 300
 301         else #if ( $JOB == 'PBS' )
 302            if ( -e submit_pbs.csh ) rm -f submit_pbs.csh
 303 cat > submit_pbs.csh <<EOF
 304 #!/bin/csh
 305 #
 306 ### PBS batch script to run an MPI application
 307 #
 308 #PBS -A $PROJID
 309 #PBS -l walltime=$observerclocktime
 310 #PBS -l select=16:ncpus=36:mpiprocs=36:mem=110G
 311 #PBS -N o${N}s1
 312 #PBS -j oe
 313 #PBS -q $QUEUE
 314
 315 #run the executable
 316 mpirun ./da_wrfvar.exe
 317 EOF
 318         set jobid = `qsub  submit_pbs.csh`
 319
 320         #else
 321         #   $RUN_CMD $NPROCS_NL ${PWD}/da_wrfvar.exe < /dev/null
 322         endif
 323 #------------------------------------------
 324
 325     set i = 1
 326     while ( $i <= 1000 )
 327       sleep 30
 328       echo "sleep 30" >> log
 329       if ( "`qstat | grep -o -m 1 $jobid`" != $jobid ) then
 330         if ( "`tail  rsl.out.0000 | grep -o -m 1 successfully`" != "successfully" ) then
 331           echo "da_wrfvar stage1 failed " > FAIL
 332           exit -1
 333         else
 334           break
 335         endif
 336       else
 337         @ i++
 338       endif
 339     end
 340     echo "done" >> log
 341
 342       echo "------------------------------"
 343       echo "| 1.3 Save and clean output |"
 344       echo "------------------------------"
 345
 346         set out_storage=o${N}s1
 347         if ( ! -d $out_storage ) mkdir $out_storage
 348         mkdir $out_storage/RSL
 349         mv submit_pbs.csh fort.140 namelist.input namelist.output* $out_storage
 350         mv rsl.* $out_storage/RSL
 351         #mv o${N}s1.*.out o${N}s1.*.err $out_storage
 352         rm rej_obs_conv*
 353         mv gts_omb.* $out_storage # to be used in stage2
 354         mv buddy_check check_max_iv jo cost_fn grad_fn statistics $out_storage
 355
 356         echo "outerloop${N}_stage1" > .last_stage
 357
 358       if ( "`cat .current_stage`" != "outerloop${N}_stage1" && \
 359            "`cat .last_stage`"    != "outerloop${N}_stage1"  ) then
 360         echo "outerloop${N}_stage1 was NOT done, aborted Stage2"
 361         exit -1
 362       endif
 363   endif # end if RUN_STAGE1
 364
 365 #---------------------------------
 366   if ( $RUN_STAGE2 == "true" ) then
 367
 368       echo "--------------------------------------"
 369       echo "| Run outerloop-$N : Stage2-Minimizer |"
 370       echo "--------------------------------------"
 371
 372       echo "--------------------------------------------"
 373       echo "| 2.0 Set up namelist.input for stage2 |"
 374       echo "--------------------------------------------"
 375
 376     #---------------------------------------
 377     #   &wrfvar1
 378     #    multi_inc=2, # stage2 for minimization
 379     #    dx=
 380     #    dy=${NL_DY}.0,
 381     #    e_we=$NL_E_WE,
 382     #    e_sn=$NL_E_SN,
 383     #    time_step=$TIME_STEP_STAGE2,
 384     #    var4d_bin=
 385     #------------------------------------------
 386     set   NL_DX=`grep -i -E "dx[ \t]*="   orig_namelist.input | sed -e 's/\t/ /g' -e 's/  *//g' -e 's/,//' -e 's/\r//g'| cut -c4-`
 387     set   NL_DY=`grep -i -E "dy[ \t]*="   orig_namelist.input | sed -e 's/\t/ /g' -e 's/  *//g' -e 's/,//' -e 's/\r//g'| cut -c4-`
 388     set NL_E_WE=`grep -i -E "e_we[ \t]*=" orig_namelist.input | sed -e 's/\t/ /g' -e 's/  *//g' -e 's/,//' -e 's/\r//g'| cut -c6-`
 389     set NL_E_SN=`grep -i -E "e_sn[ \t]*=" orig_namelist.input | sed -e 's/\t/ /g' -e 's/  *//g' -e 's/,//' -e 's/\r//g'| cut -c6-`
 390
 391     set   NL_DX=`echo $NL_DX | sed 's/\.[0-9]*//g'`
 392     set   NL_DY=`echo $NL_DY | sed 's/\.[0-9]*//g'`
 393     set NL_E_WE=`echo $NL_E_WE | sed 's/\.[0-9]*//g'`
 394     set NL_E_SN=`echo $NL_E_SN | sed 's/\.[0-9]*//g'`
 395
 396     @   NL_DX = $NL_DX * $THIN_FACTOR[$N]
 397     @   NL_DY = $NL_DY * $THIN_FACTOR[$N]
 398     @ NL_E_WE = ($NL_E_WE - 1) / $THIN_FACTOR[$N] + 1
 399     @ NL_E_SN = ($NL_E_SN - 1) / $THIN_FACTOR[$N] + 1
 400
 401     @   RADT_STAGE2 = $NL_DX / 1000
 402
 403     set NL_VAR4D_BIN=`grep -i -E "var4d_bin[ \t]*=" orig_namelist.input | \
 404                       sed -e 's/\t/ /g' -e 's/  *//g' \
 405                           -e 's/\.[0-9]*//g' -e 's/,//' \
 406                           -e 's/\r//g' | \
 407                       cut -c11-`
 408     @ TIME_STEP_STAGE2 = ( $NL_DX / 1000 ) * 6
 409     set i=$TIME_STEP_STAGE2
 410     while ( $i != 0 )
 411       @ i = $NL_VAR4D_BIN % $TIME_STEP_STAGE2
 412       @ TIME_STEP_STAGE2--
 413     end
 414     @ TIME_STEP_STAGE2++
 415
 416     if ( $OS == "Darwin" ) then
 417        sed -e '/[Mm][Uu][Ll][Tt][Ii]_[Ii][Nn][Cc]/d' \
 418            -e '/&[Ww][Rr][Ff][Vv][Aa][Rr]1$/ a\ \
 419            multi_inc=2,' \
 420            -e "/[Dd][Xx][ \t]*=/ c\ \
 421            dx=${NL_DX}.0," \
 422            -e "/[Dd][Yy][ \t]*=/ c\ \
 423            dy=${NL_DY}.0," \
 424            -e "/[Ee]_[Ww][Ee][ \t]*=/ c\ \
 425            e_we=$NL_E_WE,"\
 426            -e "/[Ee]_[Ss][Nn][ \t]*=/ c\ \
 427            e_sn=$NL_E_SN," \
 428            -e "/[Tt][Ii][Mm][Ee]_[Ss][Tt][Ee][Pp][ \t]*=/ c\ \
 429            time_step=$TIME_STEP_STAGE2," \
 430            -e "/[u][s][e]_[i][n][v][e][r][s][e]_[s][q][u][a][r][e][r][o][o][t][b][ \t]*=/ c\ \
 431            use_inverse_squarerootb=$use_vp," \
 432            -e "/[u][s][e]_[i][n][t][e][r][p][o][l][a][t][e]_[c][v][t][ \t]*=/ c\ \
 433            use_interpolate_cvt=$use_cvt," \
 434            orig_namelist.input > o${N}s2_namelist.input
 435     else
 436        sed -e '/[Mm][Uu][Ll][Tt][Ii]_[Ii][Nn][Cc]/d' \
 437            -e '/&[Ww][Rr][Ff][Vv][Aa][Rr]1$/ a\multi_inc=2,' \
 438            -e "/[Dd][Xx][ \t]*=/ c\dx=${NL_DX}.0," \
 439            -e "/[Dd][Yy][ \t]*=/ c\dy=${NL_DY}.0," \
 440            -e "/[Ee]_[Ww][Ee][ \t]*=/ c\e_we=$NL_E_WE," \
 441            -e "/[Ee]_[Ss][Nn][ \t]*=/ c\e_sn=$NL_E_SN," \
 442            -e "/[Tt][Ii][Mm][Ee]_[Ss][Tt][Ee][Pp][ \t]*=/ c\time_step=$TIME_STEP_STAGE2," \
 443            -e "/[u][s][e]_[i][n][v][e][r][s][e]_[s][q][u][a][r][e][r][o][o][t][b][ \t]*=/ c\use_inverse_squarerootb=$use_vp," \
 444            -e "/[u][s][e]_[i][n][t][e][r][p][o][l][a][t][e]_[c][v][t][ \t]*=/ c\use_interpolate_cvt=$use_cvt," \
 445            orig_namelist.input > o${N}s2_namelist.input
 446     endif
 447
 448       if ( $N > 1 ) then  # only do this step from outer loop 2
 449
 450          @ NM1 = $N - 1
 451
 452           if ( $use_vp == true || $use_cvt == true ) then
 453       echo "--------------------------------------------"
 454       echo "| 2.1 Scatter the global cvt or vp to PEs |"
 455       echo "--------------------------------------------"
 456
 457             ln -sf ./o${NM1}s3/vp_output.global_hires  .
 458             if ( ! -e vp_output.global_hires ) then
 459                echo "vp_output.global_hires is not found"
 460                exit -1
 461             endif
 462
 463 #-------------------------------------------
 464             if ( $JOB == 'LSF' ) then
 465                if ( -e submit_lsf.csh ) rm -f submit_lsf.csh
 466 cat > submit_lsf.csh <<EOF
 467 #!/bin/csh
 468 #
 469 ## LSF batch script to run an MPI application
 470 #
 471 #BSUB -P $PROJID             # project code
 472 #BSUB -W 00:05               # wall-clock time (hrs:mins)
 473 #BSUB -n $NPROCS[$N]         # number of tasks in job
 474 #BSUB -R "span[ptile=16]"    # run 16 MPI tasks per node
 475 #BSUB -J o${N}s2_split   # job name
 476 #BSUB -o o${N}s2_split.%J.out # output file name in which %J is replaced by the job ID
 477 #BSUB -e o${N}s2_split.%J.err # error file name in which %J is replaced by the job ID
 478 #BSUB -q $QUEUE              # queue
 479
 480 #run the executable
 481 mpirun.lsf ./da_vp_split.exe
 482 EOF
 483                bsub -K < submit_lsf.csh
 484
 485         else #if ( $JOB == 'PBS' )
 486            if ( -e submit_pbs.csh ) rm -f submit_pbs.csh
 487 cat > submit_pbs.csh <<EOF
 488 #!/bin/csh
 489 #
 490 ### PBS batch script to run an MPI application
 491 #
 492 #PBS -A $PROJID
 493 #PBS -l walltime=00:05:00
 494 #PBS -l select=16:ncpus=36:mpiprocs=36:mem=110G
 495 #PBS -N o${N}s2_split
 496 #PBS -j oe
 497 #PBS -q $QUEUE
 498
 499 #run the executable
 500 mpirun ./da_vp_split.exe -cloud_cv_options 3 -use_cv_w 1
 501 EOF
 502            set jobid = `qsub  submit_pbs.csh`
 503
 504             #else
 505             #   $RUN_CMD $NPROCS[$N] ${PWD}/da_vp_split.exe < /dev/null
 506             endif
 507 #---------------------------------------------
 508            endif # end if ( $use_vp == true || $use_cvt == true )
 509
 510     set i = 1
 511     while ( $i <= 1000 )
 512       sleep 20
 513       echo "sleep 20" >> log
 514       if ( "`qstat | grep -o -m 1 $jobid`" != $jobid ) then # if job finish
 515         if ( $use_vp == true || $use_cvt == true ) then
 516             foreach f ( vp_input.0* )
 517               if ( ! -e $f ) then
 518                 echo "$f NOT exists" > FAIL
 519                 exit -1
 520               endif
 521             end
 522             break
 523         endif
 524       else
 525         @ i++
 526       endif
 527     end
 528     echo "done" >> log
 529
 530       endif     # end if N > 1
 531
 532      if ( $THIN_FACTOR[$N] > 1 ) then
 533       echo "--------------------------------------------"
 534       echo "| 2.2 Thin high-res guess to low-res |"
 535       echo "--------------------------------------------"
 536
 537         ./da_thin.exe -i o${N}s1_fg      \
 538                       -o o${N}s2_fg      \
 539                       -thin $THIN_FACTOR[$N] \
 540                       >& o${N}s2_thin_fg.log
 541         if ( ! -e o${N}s2_fg ) then
 542           echo "thinning fg failed" > FAIL
 543           exit -1
 544         endif
 545
 546         if ( $VAR4D == true ) then # 2nd level fg only for 4DVAR
 547          ./da_thin.exe -i o${N}s1_fg02    \
 548                        -o o${N}s2_fg02    \
 549                        -thin $THIN_FACTOR[$N] \
 550                        >>& o${N}s2_thin_fg02.log
 551          if ( ! -e o${N}s2_fg02 ) then
 552            echo "thinning fg02 failed" > FAIL
 553            exit -1
 554          endif
 555         endif
 556      else
 557         cp o${N}s1_fg   o${N}s2_fg
 558         if ( $VAR4D == true ) cp o${N}s1_fg02 o${N}s2_fg02
 559      endif
 560
 561      if ( $VAR4D == true ) then
 562       echo "--------------------------------------------"
 563       echo "| 2.3 Generating boundary file for stage2 |"
 564       echo "--------------------------------------------"
 565       da_bdy.exe -fg   o${N}s2_fg         \
 566                  -fg02 o${N}s2_fg02         \
 567                  -bdy  orig_wrfbdy_d01     \
 568                  -o    o${N}s2_wrfbdy_d01   \
 569                  >&    o${N}s2_bdy.log
 570       if ( ! -e o${N}s2_wrfbdy_d01 ) then
 571         echo "generating bdy file for outerloop$N stage2 failed" > FAIL
 572         exit -1
 573       endif
 574      endif
 575
 576       echo "--------------------------------------------"
 577       echo "| 2.4 Run WRF-VAR minimization at low-res |"
 578       echo "--------------------------------------------"
 579
 580       if ( -e namelist.input ) rm -f  namelist.input
 581       ln -sf  o${N}s2_namelist.input  namelist.input
 582       ln -sf  ./o${N}s1/gts_omb.*  .
 583       ln -sf  o${N}s2_fg          fg
 584       if ( $VAR4D == true ) then
 585        ln -sf  o${N}s2_fg02        fg02
 586        ln -sf  o${N}s2_fg          wrfinput_d01
 587        ln -sf  o${N}s2_wrfbdy_d01  wrfbdy_d01
 588       endif
 589
 590       echo "outerloop${N}_stage2" > .current_stage
 591       echo "outerloop${N}_stage2" > .last_stage
 592
 593 #---------------------------------
 594       if ( $JOB == 'LSF' ) then
 595          if ( -e submit_lsf.csh ) rm -f submit_lsf.csh
 596 cat > submit_lsf.csh <<EOF
 597 #!/bin/csh
 598 #
 599 ### LSF batch script to run an MPI application
 600 #
 601 #BSUB -P $PROJID             # project code
 602 #BSUB -W $minimizeclocktime  # wall-clock time (hrs:mins)
 603 #BSUB -n $NPROCS[$N]         # number of tasks in job
 604 #BSUB -R "span[ptile=16]"    # run 16 MPI tasks per node
 605 #BSUB -J o${N}s2           # job name
 606 #BSUB -o o${N}s2.%J.out    # output file name in which %J is replaced by the job ID
 607 #BSUB -e o${N}s2.%J.err    # error file name in which %J is replaced by the job ID
 608 #BSUB -q  $QUEUE             # queue
 609
 610 #run the executable
 611 mpirun.lsf ./da_wrfvar.exe
 612 EOF
 613          bsub -K < submit_lsf.csh
 614
 615         else #if ( $JOB == 'PBS' )
 616            if ( -e submit_pbs.csh ) rm -f submit_pbs.csh
 617 cat > submit_pbs.csh <<EOF
 618 #!/bin/csh
 619 #
 620 ### PBS batch script to run an MPI application
 621 #
 622 #PBS -A $PROJID
 623 #PBS -l walltime=$minimizeclocktime
 624 #PBS -l select=16:ncpus=36:mpiprocs=36:mem=110G
 625 #PBS -N o${N}s2
 626 #PBS -j oe
 627 #PBS -q $QUEUE
 628
 629 #run the executable
 630 mpirun ./da_wrfvar.exe
 631 EOF
 632          set jobid = `qsub  submit_pbs.csh`
 633
 634       #else
 635       #   $RUN_CMD $NPROCS[$N] ${PWD}/da_wrfvar.exe < /dev/null
 636       endif
 637 #------------------------------------
 638
 639     set i = 1
 640     while ( $i <= 1000 )
 641       sleep 30
 642       echo "sleep 30" >> log
 643      if ( "`qstat | grep -o -m 1 $jobid`" != $jobid ) then # if job finish
 644       if ( "`tail  rsl.out.0000 | grep -o -m 1 successfully`" != "successfully" ) then
 645         echo "da_wrfvar stage2_$N failed " > FAIL
 646         exit -1
 647       else
 648         break
 649       endif
 650      else
 651       @ i++
 652      endif
 653     end
 654     echo "done" >> log
 655
 656       echo "------------------------------"
 657       echo "| 2.5 Save and clean output |"
 658       echo "------------------------------"
 659
 660       #if ( $use_vp == true ) ncl nc_vpglobal.ncl
 661       #ncl nc_increment.ncl
 662
 663       set out_storage=o${N}s2
 664       if ( ! -d $out_storage ) mkdir $out_storage; mkdir $out_storage/RSL
 665       rm unpert* gts_omb.* gts_omb_oma_*.* radar_omb_oma_*.* filtered_obs.* rej_obs*
 666       mv buddy_check check_max_iv qcstat_conv_01 $out_storage
 667       mv rsl.*  $out_storage/RSL
 668       mv cost_fn grad_fn jo fort.* statistics $out_storage
 669       mv wrfvar_output namelist.output* $out_storage
 670       mv gts_omb_oma_*  radar_omb_oma_* filtered_obs_* submit_pbs.csh $out_storage
 671       mv analysis_increments*  $out_storage
 672       if ( $use_vp == true || $use_cvt == true ) then
 673         if ( $N > 1 ) rm vp_output.global_hires
 674         mv vp_input.* vv_input.* vv_after* vp_output.global* $out_storage
 675       endif
 676
 677   endif # end if RUN_STAGE2
 678
 679 #-----------------------------------
 680   if ( $RUN_STAGE3 == "true" ) then
 681
 682       echo "--------------------------------------"
 683       echo "| Run outerloop${N} : Stage3-Regrid |"
 684       echo "--------------------------------------"
 685
 686     @ NP1 = $N + 1
 687
 688     if ( "$VAR4D_LBC" == "t" || "$VAR4D_LBC" == "T" ) then
 689       # this is not well tested yet. turned it off
 690       echo "------------------------------------------------------------"
 691       echo "| 3.1 regridding in model space for 2nd time level (fg02) |"
 692       echo "------------------------------------------------------------"
 693       #  only if var4d_lbc=true.
 694       #  NOTE: interpolate to original resolution in model space
 695
 696         ./da_bilin.exe  -fg_lores o${N}s2_fg02 \
 697                         -fg_hires o${N}s1_fg02 \
 698                         -an_lores ana02       \
 699                         -ns       $THIN_FACTOR[$N]   \
 700                         -o        ana02_hires >& o${N}s2_bilin_fg02.log
 701         if ( ! -e ana02_hires ) then
 702           echo "regridding increment failed" > FAIL
 703           exit -1
 704         endif
 705         mv ana02  ana02_lores
 706     else
 707       echo "# Skipped, VAR4D_LBC=$VAR4D_LBC \n"
 708     endif
 709
 710       echo "------------------------------------------------------------"
 711       echo "| 3.1 regridding in model space for 1st time level (fg) |"
 712       echo "------------------------------------------------------------"
 713     #  wrfvar_output_hires = fg_stage1 + S (wrfvar_output_lores - fg_stage2)
 714     #----------------------------------------------------------------------------
 715     if ( $THIN_FACTOR[$N] > 1 ) then # regrid to model resolution
 716       ./da_bilin.exe  -fg_lores o${N}s2_fg          \
 717                       -fg_hires o${N}s1_fg          \
 718                       -an_lores ./o${N}s2/wrfvar_output    \
 719                       -ns       $THIN_FACTOR[$N] \
 720                       -o        wrfvar_output_hires >& o${N}s3_da_bilin.log
 721       if ( ! -e wrfvar_output_hires ) then
 722         echo "regridding increment failed" > FAIL
 723         exit -1
 724       endif
 725      else    # if DA res. is same as model res., no need for interpolation
 726       cp ./o${N}s2/wrfvar_output wrfvar_output_hires
 727      endif
 728
 729     if ( $N < $MAX_OUTERLOOP ) then # no need to do this for the last loop
 730     #------------------------------------------------------
 731     # interpolate vp to next outer loop's resolution
 732     #-------------------------------------------------------------
 733      if ( $use_vp == true || $use_cvt == true ) then
 734         if ( $THIN_FACTOR[$N] != $THIN_FACTOR[$NP1] ) then # only do this if res diff for two loops
 735            echo "---------------------------------------------"
 736            echo "| 3.2 regridding in control variable space  |"
 737            echo "---------------------------------------------"
 738            if ( $N < $MAX_OUTERLOOP ) then
 739                @ ratio = $THIN_FACTOR[$N] / $THIN_FACTOR[$NP1]
 740            else
 741                @ ratio = $THIN_FACTOR[$N] # / 1
 742            endif
 743
 744            if ( -e vp_output.global )  rm -f vp_output.global
 745            ln -sf ./o${N}s2/vp_output.global .
 746            ./da_vp_bilin.exe -ratio $ratio  \
 747                              -cloud_cv_options 3  \
 748                              -use_cv_w 1  >& o${N}s3_vp_bilin.log
 749            if ( ! -e vp_output.global_hires ) then
 750               echo "vp_output.global_hires is not generated" > FAIL
 751               exit -1
 752            endif
 753         else # if resolution same, no interpolation needed.
 754            cp ./o${N}s2/vp_output.global vp_output.global_hires
 755         endif
 756         #ncl nc_vphires.ncl
 757      endif
 758
 759       echo "---------------------------------"
 760       echo "| 3.3 Save and Clean results  |"
 761       echo "---------------------------------"
 762
 763       set out_storage=o${N}s3
 764       if ( ! -d $out_storage ) mkdir $out_storage
 765
 766       mv  wrfvar_output_hires $out_storage
 767       rm  namelist.input
 768
 769       if ( $use_vp == true || $use_cvt == true ) then
 770         mv vp_output.global_hires* $out_storage
 771         rm vp_output.global
 772       endif
 773
 774    endif # $N < $MAX_OUTERLOOP
 775
 776   endif ## end if RUN_STAGE3=true
 777
 778     @ N++
 779
 780   end # End of outerloop
 781
 782   rm fg fg02 wrfinput_d01 wrfbdy_d01
 783
 784 else  ## if NOT multi-resolution incremental 3D/4DVAR, no stop outer loop
 785 ###########################################################################
 786
 787         ln -sf ${BE3}     be.dat
 788         ln -sf orig_fg    fg
 789         if ( $VAR4D == true ) then
 790           ln -sf     orig_wrfinput_d01  wrfinput_d01
 791           ln -sf     orig_wrfbdy_d01    wrfbdy_d01
 792         endif
 793
 794     if ( $OS == "Darwin" ) then
 795        sed -e "/[u][s][e]_[i][n][v][e][r][s][e]_[s][q][u][a][r][e][r][o][o][t][b][ \t]*=/ c\ \
 796            use_inverse_squarerootb=$use_vp," \
 797            -e "/[u][s][e]_[i][n][t][e][r][p][o][l][a][t][e]_[c][v][t][ \t]*=/ c\ \
 798            use_interpolate_cvt=$use_cvt," \
 799            orig_namelist.input > namelist.input
 800     else
 801        sed -e "/[u][s][e]_[i][n][v][e][r][s][e]_[s][q][u][a][r][e][r][o][o][t][b][ \t]*=/ c\use_inverse_squarerootb=$use_vp," \
 802            -e "/[u][s][e]_[i][n][t][e][r][p][o][l][a][t][e]_[c][v][t][ \t]*=/ c\use_interpolate_cvt=$use_cvt," \
 803            orig_namelist.input > namelist.input
 804     endif
 805
 806 #---------------------------------
 807     if ( $JOB == 'LSF' ) then
 808        if ( -e submit_lsf.csh ) rm -f submit_lsf.csh
 809 cat > submit_lsf.csh <<EOF
 810 #!/bin/csh
 811 #
 812 ### LSF batch script to run an MPI application
 813 #
 814 #BSUB -P $PROJID             # project code
 815 #BSUB -W 5:00 # ban  00:20                 # wall-clock time (hrs:mins)
 816 #BSUB -n 1024          # number of tasks in job
 817 #BSUB -R "span[ptile=16]"    # run 16 MPI tasks per node
 818 #BSUB -J wrfvar               # job name
 819 #BSUB -o wrfvar.%J.out        # output file name in which %J is replaced by the job ID
 820 #BSUB -e wrfvar.%J.err        # error file name in which %J is replaced by the job ID
 821 #BSUB -q premium  # regular               # queue
 822
 823 #run the executable
 824 mpirun.lsf ./da_wrfvar.exe
 825 EOF
 826        bsub -K < submit_lsf.csh
 827
 828     else #if ( $JOB == 'PBS' )
 829         if ( -e submit_pbs.csh ) rm -f submit_pbs.csh
 830 cat > submit_pbs.csh <<EOF
 831 #!/bin/csh
 832 #
 833 ### PBS batch script to run an MPI application
 834 #
 835 #PBS -A $PROJID
 836 #PBS -l walltime=5:00:00
 837 #PBS -l select=16:ncpus=36:mpiprocs=36:mem=110G
 838 #PBS -N wrf4dvar
 839 #PBS -j oe
 840 #PBS -q $QUEUE
 841
 842 #run the executable
 843 mpirun ./da_wrfvar.exe
 844 EOF
 845            qsub  submit_pbs.csh
 846
 847     #else
 848     #   set N=1
 849     #   $RUN_CMD $NPROCS[$N] ${PWD}/da_wrfvar.exe < /dev/null
 850     endif
 851 #-----------------------------------
 852
 853       ln -sf $MULTI_INC_TOOLS/nc_increment.ncl .
 854       ncl nc_increment.ncl
 855
 856       set out_storage=$WORK_DIR
 857       if ( ! -d $out_storage ) mkdir $out_storage; mkdir $out_storage/RSL
 858       rm unpert* gts_omb_oma_*.* filtered_obs.*
 859       mv rsl.* rej_obs* qcstat_conv_01 $out_storage/RSL
 860
 861 endif
 862