bin/scripttest

   1 #!/bin/bash
   2 ############################################################
   3 # source: scripttest
   4 # author: CottonCandyOwner(CottonCandyOwner@126.com)
   5 # date: 2021-09-30
   6 ############################################################
   7 # Copyright (C) 2022- Free Software Foundation, Inc.
   8 # This configure script is free software; the Free Software
   9 # Foundation gives unlimited permission to copy, distribute
  10 # and modify it.
  11 ############################################################
  12 # note:
  13 #   scripttest is a program used to running test script one
  14 # by one in testing dir. it's used to test script, or do some
  15 # integerated testing work for binary program.
  16 #   the hi-light of this program is that, every test item
  17 # orgnized by catalog id, tester can specify id or count number
  18 # or id range to running test script.
  19 #   comparing with other test-suite manage test-item by suite,
  20 # this program provide dir structure to orgnize large number of
  21 # test-items. dir structure is easy for comprehenssion in our
  22 # brain, rather then large number of test-item in linear.
  23 #   another point is that, it manage on failed items. tester can
  24 # test the failed item only. it's usefull in developer stage.
  25 #   it's usefull for script module/shlib unit test, and program
  26 # integerated testing, and soft-pkg test before install, and
  27 # loop-back test after some feature append or bug fixed.
  28 #
  29 ############################################################
  30
  31 # Testing:
  32 # @ nest testing, and check if the corresponding testing tmp dir is correct.
  33 # @ test unit for scripttest will be updated.
  34 # @ copy test unit from shlib-lite for paramter -bes.
  35 # @ some place will generate misc files should not be become.
  36 #
  37 #
  38 asdfas='
  39  -O , --output-dir=<output_dir> 指定测试时的临时文件目录。默认值为~/用户目录下的
  40                               .testing目录。
  41  -f , --force                  指定测试信息的目录。默认值为当前目录下的testing
  42                               目录。
  43  -y , --sync                   根据测试信息描述文件，生成测试目录及测试脚本等文
  44      =[sync_to_dir]           件，用于用户编写各种测试用例。
  45         # todo：输出文件目录异常。暂时disable该功能。
  46         # tbd：使用同步输出为-g参数，-y用于sync-back
  47 '
  48
  49
  50
  51
  52 source codegen --load
  53
  54 . shlibinc
  55
  56 include stdio.shlib
  57 include args.shlib
  58 include envar.shlib
  59
  60 include cataid.shlib
  61 include dirgen.shlib
  62
  63
  64 #include dbgout.shlib
  65 #include term.shlib
  66 #include gplib.shlib
  67 #include catalog.shlib
  68 #dsource imifile.shlib
  69
  70 #include param/param.shlib
  71
  72 ##############################
  73 # section: public comment info
  74 ##############################
  75
  76
  77
  78 ##############################
  79 # section: variable define
  80 ##############################
  81
  82
  83 #
  84 # time cost of a standard testing example, the time unit is ms.
  85 # the basic testing example is running on cpu that have performence
  86 # between 100MHz and 5GHz. and the shortest test time is 10ms, the max
  87 # test time is 5s.
  88 #   if it is running on a best cpu at 5GHz, it must longer then 0.01s.
  89 # and it also should be shorter then 5s with a 100MHz cpu.
  90 #   cpu performence is not only decided by frequent. so a 5GHz cpu maybe
  91 # more then 5GHz instruction per-second. so, the best cpu maybe 5*50
  92 # times then lowest cpu.
  93 #   let the example testing code cost 0.02s with best cpu, and it will
  94 # cost about 5s with lowest cpu.
  95 #   if one test script cost 0.2s on the best cpu, and save as 10 unit
  96 # times. when this script is tested on a middle cpu, UNIT_TEST_MS is
  97 # 500ms, it will cost 5s, and it will be displayed on testing info.
  98 # if the cpu performence is better then it, it would not display on
  99 # screen.
 100 #
 101
 102 #
 103 # codegen about
 104 #
 105 scrt_codegen_param_load ()
 106 {
 107         include param/catanode-testdir.shlib
 108 }
 109 vonParamLoad=scrt_codegen_param_load
 110
 111 # set this again for scripttest
 112 #DEF_INPUT_CATALOG="testing/testing.catalog"
 113 DEF_INPUT_CATALOG="doc/designdoc/testing.catalog"
 114
 115
 116 ALIGN=' '
 117
 118 UNIT_TEST_MS=
 119
 120 TESTING_ROOT_DIR=
 121 TESTING_TMP_DIR=~/.testing/$(basename $0)/
 122
 123 test_cnt=0
 124 test_cnt_num=99999
 125
 126 err_continue_cnt=0
 127 err_cnt=0
 128
 129 TEST_CMD=
 130 TEST_CMD_OPT=
 131 TEST_CMDLINE_FMT='${TEST_CMD} ${TEST_CMD_OPT}'
 132 TEST_CMDLINE=
 133
 134 #
 135 # copy code from term.shlib in shlib, do not use whole lib.
 136 #
 137         readonly CNORMAL="\033[0m"    # restore to normal mode
 138         readonly FCGREEN="\033[32m"   # font green
 139         readonly FCMAGENTA="\033[35m" # font magenta
 140         readonly FCRED="\033[31m"     # font red
 141         readonly FCCYAN="\033[36m"    # font cyan
 142         readonly CREV="\033[7m"       # reverse
 143
 144 FCCOLOR=$FCGREEN
 145 IFCCOLOR=$FCMAGENTA
 146 EFCCOLOR=$FCRED
 147 INFOCOLOR=$FCGREEN
 148
 149 readonly FCR2L="${FCCYAN}"
 150 readonly FCL2R="${FCCYAN}${CREV}"
 151
 152 verbos_pfx=" ${INFOCOLOR}+$CNORMAL "
 153
 154 #
 155 # do not delete this comment, it is used for var define code intert.
 156 #
 157 # args-var-define-begin
 158 # args-var-define-end
 159 #
 160
 161 #
 162 # ATTENTION:
 163 # @ the description string can not contain char of "'".
 164 #
 165 #
 166
 167 #
 168 # desc-str for scripttest
 169 # @ every paramter desc-str-line start with 'param'.
 170 # @ every colum start with '|', and seperated by blanks.
 171 # @ '|blank' means a blank line dispalyed in helper.
 172 # @ follow with a '' quoated string is the option category string.
 173 # @ comment '#' in desc-str is also supported, and can be used to
 174 #   disable some option have not been implemented.
 175 # @ desc-str can be in multiple var. and some of them maybe re-used
 176 #   for other program by 'source xxx --loadshlib'.
 177 #
 178 scripttest_desc_hdr_str="
 179 |prog $0 '应用测试程序。用于对指定目录下的一系列测试脚本运行测试，并输出测试结果。如未指定测试目录参数，以当前目录下的testing目录作为默认测试信息目录，并进行测试。'
 180 "
 181
 182 scripttest_desc_str="
 183
 184 |blank 'General Testing Paramters:'
 185 |param |-n |--num          |---    |=<test_cnt_num>   |%<test_cnt_num>      |&                      |'running N test items continously.'
 186 |param |-L |--failed-begin |---    |=                 |%<failed_begin>      |&                      |'从failed列表中的第一项开始测试。'
 187 |param |-F |--failed       |---    |=                 |%<failed_test>       |&<args_failed_test>    |'test the failed items listed by -a option. it\x27s equal to -L -r -q option.'
 188 |param |-r |--ignor-err    |---    |=                 |%<test_ignor_err>    |&                      |'ignor testing error, and running continously.'
 189 |param |-a |--all          |---all |=                 |%<test_all>          |&                      |'test for all.'
 190 |param |-A |--             |---    |=                 |%<continouse_all>    |&<args_all>            |'test for all with -r -q option.'
 191 |blank
 192
 193 |blank 'Testing Range Paramters:'
 194 |param |-t |--test         |---    |=<test_id>        |%<test_id>           |&<args_test_id>        |'test a specified item by test item id.'
 195                                                                                                     |'the id can be a dir id, that means test the items under the dir. if no id specified, it will test from the beginning id of normal testing.'
 196 |param |-b |--begin        |---    |=<begin_test_id>  |%<begin_test_id>     |&                      |'begin id of testing range.'
 197 |param |-e |--end          |---    |=<end_test_id>    |%<end_test_id>       |&                      |'end id of the testing range.'
 198 |param |-x |--exclude      |---    |=<exclude_id>     |%<exclude_id>        |&<args_exclude>        |'exclude id that should not be tested.'
 199 |param |-R |--rollup       |---    |=                 |%<test_id_rollup>    |&                      |'rollup begin id, if the corresponding item is ok.'
 200 |param |-B |--rollback     |---    |=                 |%<test_id_rollback>  |&                      |'rollback begin id to the previous value. it is not simplly decrease the count of id, it will check the test item file first.'
 201 |param |-s |--set          |---    |=                 |%<set_test_info>     |&                      |'save the range setted by -b and -e and -x option.'
 202 |param |-c |--clean        |---    |=                 |%<clean>             |&<args_clean>          |'clean testing temp dir.'
 203 |blank
 204
 205 |blank 'Testing Output Info Paramters:'
 206 |param |-u |--update       |---    |=                 |%<update_test_file>  |&                      |'if the current test item is ok, save stdout output to test item as the standard output content.'
 207 |param |-v |--verbose      |---    |=                 |%<verbose>           |&<args_verbose>        |'output the test script outputed with \'+ \' prefix. it is the detail output info for testing.'
 208 |param |-q |--quiet        |---    |=                 |%<quiet>             |&                      |'disable comparation string output. but it is not conflect with -v option. use -v -q means output detail string info instead of comparation string.'
 209 |blank
 210
 211 "
 212
 213 scripttest_other_desc_str="
 214 |blank 'Misc Paramters:'
 215 |param |-f |--force        |---    |=                 |%<test_force>        |&                      |'force operation option. it\'s a general option combine with other options.'
 216 |param |-j |--multi-task   |---    |=<multi_task>     |%<multi_task>        |&                      |'multi-task running. set the parall num of tasks.'
 217
 218 |blank 'Other Paramters:'
 219 |param |-d |--dir          |---    |=<test_dir>       |%<test_dir>          |&                      |'指定测试信息的目录。默认值为当前目录下的testing目录。'
 220 |param |-O |--output-dir   |---    |=<output_dir>     |%<output_dir>        |&                      |'指定测试时的临时文件目录。默认值为~/用户目录下的.testing目录。'
 221 |param |-p |--print-vars   |---    |=                 |%<print_vars>        |&<args_print_vars>     |'输出参数定义的变量信息。'
 222 |param |-m |--mono         |---    |=                 |%<mono>              |&<args_mono>           |'输出非彩色的字符串信息。'
 223 |param |-g |--logfile      |---    |=                 |%<test_logfile>      |&<test_logfile>        |'测试failed时，不输出差异信息。'
 224 |blank
 225
 226 |blank 'Version & Helper & Debug:'
 227 |param |-V |--version      |---    |=                 |%<version_info>      |&<args_version>        |'output version info of the program.'
 228 |param |-h |--             |---    |=                 |%<h_info>            |&<args_h_info>         |'simplly helper doc only for option.'
 229 |param |-  |--help         |---    |=                 |%<help_info>         |&<args_help_info>      |'this helper doc.'
 230 |param |-  |--debug        |---    |=                 |%<test_debug>        |&<args_test_debug>     |'debug info for arguments dispatch. add the option follow with cmd.'
 231 "
 232
 233
 234 readonly SCRT_PROG=`basename $0`
 235
 236 #
 237 # this comment is used for version id auto update.
 238 #
 239 # @{prog-ver-begin}
 240 readonly SCRT_V1=0
 241 readonly SCRT_V2=1
 242 readonly SCRT_V3=0
 243 readonly SCRT_VEXT=
 244 readonly SCRT_VER_DATE=20220101
 245 # @{prog-ver-end}
 246
 247 readonly eval_SCRT_PROG_VERSION="v${SCRT_V1}.${SCRT_V2}\${SCRT_V3:+\".$SCRT_V3\"}\${SCRT_VEXT:+\"$SCRT_VEXT\"}\${SCRT_VER_DATE:+\"-$SCRT_VER_DATE\"}"
 248 PROG_VERSION="`eval echo $eval_SCRT_PROG_VERSION`
 249 Copyright (C) 2022- Free Software Foundation, Inc.
 250 This configure script is free software; the Free Software
 251 Foundation gives unlimited permission to copy, distribute
 252 and modify it.
 253
 254 Writen by CottenCandyOwner(CottenCandyOwner@126.com).
 255 "
 256
 257 SCRT_PROG_BANNER="$SCRT_PROG $PROG_VERSION"
 258 SCRT_PROG_SYNTAX='Usage: $SCRT_PROG -[${p_shortparam//\|/}]'
 259 SCRT_PROG_DESC="    this is a test-suite program to orgnize a lot of test item script. as a direct structure on file system, and execute them. when testing error,it show differece betwin current output content and the correct output content, which saved in the item files.
 260     this prog can be used as a unit testing tool, and also can be used as a functional testing tool, to running the program (set) testing before installization, and check if it can be running in current enviroument.
 261     i use this util to execute loop-back testing that i tested before. it will save time to test some repeated problem in detail. if i write a new feature for a program, i will check if the new code would make original code error.
 262     some of the option description is writen by chinese char, it is used to test utf-8 char dispaly when it works in words bundary truncate feature."
 263 SCRT_PROG_OTHER_DESC="
 264 EXAMPLES:
 265
 266     @ test soft-pkg before installation:
 267 scripttest -a -r -q
 268     this cmd run a full test continously without faild hint string. it is the same as a loop-back testing for soft-pkg.
 269
 270     @ for program loop-back testing:
 271 scripttest -a -r -q
 272     this cmd run a full test continously without faild hint string.
 273 scripttest -F -v -q
 274     test the failed items listed in full testing. fix them one by one.
 275
 276     @ for function feature unit testing:
 277 scripttest -b 1.1 -e 1.5 -x 1.3 -s
 278     this cmd set the test range between 1.1 and 1.5, and exclude 1.3. save the info to testing temp dir. it will effect on next test work.
 279 scripttest -n 1 -v -q
 280     for unit testing, it test one item, dispaly full output string, disable comparation string output. it is equal to run test script with corresponding feature.
 281 scripttest -n 1 -u
 282     if the testing is ok, save test script output string as a standard file. it will effactive on next testing.
 283 "
 284
 285 SCRT_USAGE_FMT='$SCRT_PROG_BANNER
 286 ${SCRT_PROG_SYNTAX}
 287 $SCRT_PROG_DESC
 288 ${helper}
 289 $SCRT_PROG_OTHER_DESC'
 290
 291
 292 ##############################
 293 # section: private function
 294 ##############################
 295
 296
 297
 298 ############################################################################
 299 # testing running function
 300 ############################################################################
 301
 302 #
 303 # testcase目录下测试程序的调用，及输出比较。
 304 #
 305
 306 #
 307 # exported envar, it can be used in test program.
 308 #
 309 export TPREFIX TEST_DIR_DEPTH
 310 export NCATAID NTYPE NNAME NDESC
 311
 312 #
 313 # TBD: load these format on init.
 314 #
 315 G_D_TEST_FMT='$TESTING_TMP_DIR/$NCATAID.$NNAME/'
 316 G_F_STDOUT_FMT='$NCATAID.$NNAME.sh.${NDESC}.txt'
 317 G_F_STDIN_FMT='$NCATAID.$NNAME.stdin'
 318 G_F_TIMECOST_FMT='$NCATAID.$NNAME.time.txt'
 319 G_F_TESTSCRIPT_FMT='$NCATAID.$NNAME.sh'
 320 G_F_EXE_FMT='$NNAME'
 321
 322 G_F_DEF_EXE=":"
 323
 324 ITEMHEADSTR="#@ ITEM testitem HEAD ab70bf092ecda2e0ac9348c8564bc739"
 325
 326 #
 327 # fsyntax: test_item <id> <script_name> <desc_str>
 328 # fdesc: one item testing.
 329 #
 330 test_item ()
 331 {
 332         local ret=
 333         local tmp=
 334         local exetime=
 335         local tmp_vch_output=
 336
 337         local unitflag=
 338         local d_test=
 339         local f_stdoutfull=
 340         local f_stdout=
 341         local f_stdin=
 342         local f_timecost=
 343         local f_testscript=
 344         local f_exe=
 345
 346         local NCATAID="$1"
 347         local NNAME="$2"
 348         local NDESC="$3"
 349         # local NTYPE NFTYPE NFEXT DIR_TYPE
 350         [[ -z "NCATAID" || -z "NNAME" || -z "NDESC" ]] && err "err: one of NCATAID/NNAME/NDESC is NULL." && return 1
 351
 352         eval d_test="\"$G_D_TEST_FMT\""
 353         eval f_stdoutfull="\"${G_F_STDOUT_FMT}\""
 354         [[ ! -f $f_stdoutfull ]] && err "err: file '$f_stdoutfull' is not exist.\n" && return 1
 355         eval f_stdin="\"${G_F_STDIN_FMT}\""
 356         eval f_timecost="\"${G_F_TIMECOST_FMT}\""
 357         eval f_testscript="\"${G_F_TESTSCRIPT_FMT}\""
 358         eval f_exe="\"${G_F_EXE_FMT}\""
 359
 360 #       declare -p G_F_STDOUT_FMT G_F_TIMECOST_FMT >&2
 361 #       declare -p d_test f_stdin f_stdoutfull f_timecost f_testscript f_exe >&2
 362
 363         #
 364         # validation
 365         # script => exe => test-exe
 366         # in-curr-dir => cmd/func
 367         #
 368         [[ -z $f_timecost ]] && err "err: timecost file name is NULL." && return 1
 369         [[ ! -f $f_stdin ]] && f_stdin=""
 370         if [[ -n "$f_exe" && -e "$f_exe" ]]; then
 371                 f_exe="./$f_exe"
 372         else
 373                 # which just dispaly cmd, type include functions & inner cmd.
 374                 [[ -z "$(type $f_exe 2>/dev/null)" ]] && f_exe=""
 375         fi
 376         if [[ -z "$f_exe" ]]; then
 377                 f_exe="${f_exe}_test"           # append '_test' to run unit test
 378                 TEST_OPT="-a -L exeitemfull.log"
 379                 unitflag=1
 380
 381                 if [[ -n "$f_exe" && -e "$f_exe" ]]; then
 382                         f_exe="./$f_exe"
 383                 else
 384                         # which just dispaly cmd, type include functions & inner cmd.
 385                         [[ -z "$(type $f_exe 2>/dev/null)" ]] && f_exe="" && TEST_OPT="" && unitflag=""
 386                 fi
 387         fi
 388         if [[ -n "$f_testscript" && -e "$f_testscript" ]]; then
 389                 f_testscript="./$f_testscript"
 390         else
 391                 # which just dispaly cmd, type include functions & inner cmd.
 392                 [[ -z "$(type $f_testscript 2>/dev/null)" ]] && f_testscript=""
 393         fi
 394         [[ -n "$f_testscript" ]] && f_exe="$f_testscript"
 395         [[ -z "$f_exe" ]] && err "err: executable '$f_exe' or '$f_testscript' are not valid." && return 1
 396
 397         #
 398         # cmd
 399         #
 400         TEST_CMD="$f_exe"
 401 #       TEST_OPT=
 402         TEST_CMDLINE="$f_exe $TEST_OPT"
 403         [[ -n "$f_stdin" ]] && TEST_CMDLINE+=" < $f_stdin "
 404
 405         # item testing string output.
 406 #       info "$CHIGHL$IFCCOLOR$TPREFIX[item] $NCATAID $NNAME \"$NDESC\" ... "
 407         info "$CHIGHL$IFCCOLOR$TPREFIX[item] $NCATAID.\"$NDESC\" ... "
 408
 409         #
 410         # copy test item file to temp dir, it keeps every testing
 411         # file can not be modified by test script.
 412         #
 413         rm "$d_test" -rf
 414         mkdir -p "$d_test"{,scripts}
 415         cp "$NCATAID".* "$d_test" -rf
 416         cp -rf scripts/"$NCATAID".* "$d_test"/scripts/ 2> /dev/null
 417         # XXX: here should be modified, if the scripts dir is very large,
 418         # it will copy every times for every test script.
 419         cp -rf scripts/* "$d_test"/scripts/ 2> /dev/null
 420         cp -rf scripts/testing/ "$d_test"/scripts/ 2> /dev/null
 421         cd "$d_test"
 422 #       dbgout_cmd ls -l
 423         chmod +x *.sh scripts/* 2> /dev/null
 424
 425         #
 426         # $f_stdoutfull => testitem.stdout + exeitemfull.stdout
 427         # this code must be put after cd "$d_test", or it do operation in src file.
 428         #
 429         cat "$f_stdoutfull" | grep -A1000000 -e "${ITEMHEADSTR}" > testitem.stdout
 430         cat "$f_stdoutfull" | grep -B1000000 -e "${ITEMHEADSTR}" > exeitemfull.stdout
 431         f_stdout=testitem.stdout
 432         [[ ! -s testitem.stdout ]] && cp "$f_stdoutfull" testitem.stdout
 433
 434         #
 435         # calc & dispaly testing time cost if this item would cost more then 5 seconds.
 436         #
 437         if [[ -f "$f_timecost" ]]; then
 438                 exetime=`head -n 2 "$f_timecost"`
 439                 tmp=${exetime##$'\nreal '}
 440                 tmp=${tmp%%m*}
 441                 [[ -z $tmp ]] && err "err: the syntax of test time info is not correct.\n"
 442                 exetime=${exetime##*m}
 443                 exetime=${exetime%.*}
 444                 exetime=$(( tmp * 60 + exetime ))
 445                 # dispaly only longer then 5 seconds.
 446                 [[ -n $exetime && $exetime -gt 5 ]] && info "(time:${exetime}s) "
 447         fi
 448
 449         #
 450         # here, re-direct output of test script, include stdout and stderr.
 451         # using ttyout to force string output when debug this code, event if
 452         # the stdout and stderr is re-directed.
 453         #
 454 #       ttyout dbgout_ttydev=$dbgout_ttydev
 455
 456         # set output stream pipe.
 457         tmp_vch_output=$vch_progstdout
 458         [[ $output_disable_list =~ "info" ]] && tmp_vch_output=$vch_null
 459
 460         [[ -s /tmp/onscreen.log ]] && truncate -s 0 /tmp/onscreen.log 2>/dev/null
 461         exec 5<>/tmp/onscreen.log
 462
 463         #
 464         # testing with running timecost output
 465         #
 466         info "$CNORMAL"
 467         if [[ $verbose == "enable" ]]; then
 468                 #
 469                 # time语句的code-block，stdout调试信息输出使用stderr，time语句输出到stderr，设置stderr输出为time.txt
 470                 # time语句内的code-block，stderr输出到stdout
 471                 # 这样，time的stderr输出到time.txt文件，time之内的code-block输出到vch_dbgout(stderr)
 472                 #
 473                 {
 474                         time ( :; {
 475                                 # statement below shows how to output debug info at here.
 476 #                               info "\n${INFOCOLOR} + ###############################################${CNORMAL}\n" ;
 477                                 { { $TEST_CMDLINE; echo; } 2>&1 | tee testitem.log; } | { IFS=$'\n'; while read -r line; do echo -ne "$verbos_pfx$line\n"; done } ;
 478 #                               info "${INFOCOLOR} + ###############################################${CNORMAL}\n" ;
 479                                 } 2>&1 )
 480                 } >&$tmp_vch_output 2>time.txt
 481
 482 #               ( time (
 483 #               info "abc\n";
 484 #               ttyout "testing\n";
 485 #                                       info "\n###############################################\n" ;
 486 #                                       ttyout "testing\n";
 487 #                                       # 输出测试脚本的stdout和stderr到管道，管道的调试信息使用tee在控制台stdout输出，并保存到日志文件
 488 #                                       ( ./$script_name 2>&1 ) | tee log.txt;
 489 #                                       info "###############################################\n" ;
 490 #                                       ttyout "testing\n";
 491 #                               ) 2>&1 # | { IFS=$'\n'; while read -r line; do echo "$verbos_pfx$line"; done } >&$vch_dbgout #2> /dev/null
 492 #               ) 2> time.txt #2>&1
 493         else
 494                 # an echo cmd append at the end of test script.
 495                 # script output string though pipe, but there must be a '\n' at the end
 496                 # of string, or it will not be displayed on screen.
 497
 498                 # there is some thing errors. it doesn't need to add ':;' before $TEST_CMDLINE
 499                 # running with redirection.
 500                 ( time ( :; ( $TEST_CMDLINE; echo; ) > testitem.log 2>&1 ); ) 2> time.txt #2>&1
 501         fi
 502         # close onscreen.log, save cached content in auto.
 503         exec 5<>-
 504
 505         #
 506         # append to log file.
 507         #
 508         if [[ ! $logout_disable_list =~ "info" ]]; then
 509 #               echo -ne "$INFO_PFX$@$NEWLINE_SFX" | tee -a $f_logout >&$vch_progstdout
 510                 if [[ -s /tmp/onscreen.log ]]; then
 511                         while read tmp; do
 512                                 echo "$tmp" >> $f_logout
 513                                 log_size_limit
 514                         done < <(cat /tmp/onscreen.log)
 515                 fi
 516
 517                 while read tmp; do
 518                         echo "$tmp" >> $f_logout
 519                         log_size_limit
 520                 done < <(cat testitem.log)
 521         fi
 522
 523         #
 524         # diff with file of .stdout.
 525         #
 526         # TBD:
 527 #       cp testitem.log $PWD/log/
 528         diff testitem.log "$f_stdout" > testitem.diff
 529         if [[ -z "`head -n 5 testitem.diff`" ]]; then
 530                 info "$CHIGHL$INFOCOLOR[  ok  ]$CNORMAL\n"
 531                 [[ -s /tmp/onscreen.log ]] && cat /tmp/onscreen.log > $tmp_vch_output
 532                 ret=0
 533
 534                 # 如果有failed record，进行记录。
 535                 sed -i $TESTING_TMP_DIR/testing_failed_list.txt -e "/^$(cataid_get TEST)$/d" 2>/dev/null
 536         else
 537                 info "$CHIGHL$EFCCOLOR[failed]$CNORMAL\n"
 538                 [[ -s /tmp/onscreen.log ]] && cat /tmp/onscreen.log > $tmp_vch_output
 539                 if [[ $quiet != "enable" ]]; then
 540                         info "testing result difference from correct output.\n"
 541                         info "stdout > testout\n"
 542                         info "###############################################\n"
 543                         cat testitem.diff |
 544                         while read data; do
 545                                 case ${data:0:1} in
 546                                         '>' )
 547                                                 info -ne "${FCL2R}>${CNORMAL}${data:1}\n"
 548                                                 ;;
 549                                         '<' )
 550                                                 info -ne "${FCR2L}<${CNORMAL}${data:1}\n"
 551                                                 ;;
 552                                         * )
 553                                                 info -ne "${data}\n"
 554                                                 ;;
 555                                 esac
 556                         done
 557 #                       dbgout_cmd cat testitem.diff # >&2
 558                         # 将测试failed项的日志保存到log目录，便于测试后的查看。
 559                         info "###############################################\n"
 560                 fi
 561                 ret=1
 562
 563                 # faled record
 564                 sed -i $TESTING_TMP_DIR/testing_failed_list.txt -e "/^$(cataid_get TEST)$/d" 2>/dev/null
 565                 cataid_get TEST >> $TESTING_TMP_DIR/testing_failed_list.txt
 566         fi
 567
 568         #
 569         # update .timecost & .stdout if needed.
 570         #
 571         cd - > /dev/null
 572         if [[ $update_test_file == "enable" ]]; then
 573                 # update .stdout
 574                 # testitem.log + $ITEMHEADSTR + exeitemfull.log => $f_stdoutfull
 575                 cp "$d_test"/testitem.log "$f_stdoutfull" -f
 576                 echo "$ITEMHEADSTR" >> $f_stdoutfull
 577                 [[ -s exeitemfull.log ]] && cat exeitemfull.log >> $f_stdoutfull
 578
 579                 # update .timecost
 580                 exetime=`head -n 2 $d_test/time.txt`
 581                 tmp=${exetime%%m*}
 582                 tmp=${tmp##`echo -ne "\nreal    "`} #   }
 583                 exetime=${exetime##*m}
 584                 exetime=${exetime%.*}
 585                 exetime=$(( tmp * 60 + exetime ))
 586                 [[ -n $exetime && $exetime -gt 5 ]] && cp "$d_test"/time.txt "$f_timecost"
 587         fi
 588
 589         # clean testing info
 590         rm "$d_test" -rf
 591
 592         return $ret
 593 }
 594
 595 #
 596 # test_unit <test_dir>
 597 test_unit ()
 598 {
 599         local tmp=
 600         local test_dir=$1
 601         local ret=0
 602         local chkret=0
 603         local saved_test_id=
 604
 605         cd "$1"
 606         test_cnt=`test_list_curr_content "$test_dirs" |
 607         ( while read line; do
 608 #               echo == line=$line >&2
 609                 eval tmp=( $line )
 610                 cataid_step_to TEST "${tmp[1]}"
 611
 612                 # id范围判断
 613                 cataid_compare TEST "${tmp[1]}"
 614                 chk_ret=$?
 615 #               dbgoutd "chk_ret=$chk_ret\n"
 616                 case $chk_ret in
 617                 0 | 3 )
 618                         #
 619                         ;;
 620                 1 )
 621                         continue
 622                         ;;
 623                 2 )
 624                         break
 625                         ;;
 626                 esac
 627
 628                 case ${tmp[0]} in
 629                 "diritem" )
 630                         #
 631                         cd "${tmp[4]}"
 632
 633                         TPREFIX="$(printf '%*s' $(( $TEST_DIR_DEPTH * 4 + 4)) ' ')"
 634                         #dbgoutd "TEST_DIR_DEPTH=$TEST_DIR_DEPTH\n"
 635
 636                         test_cnt=$(( test_cnt + 1 ))
 637
 638                         test_item "${tmp[1]}" "${tmp[3]}" "${tmp[2]}"
 639
 640                         if [[ $? != 0 ]]; then
 641                                 err_cnt=$(( err_cnt + 1 ))
 642                                 err_continue_cnt=$(( err_continue_cnt + 1 ))
 643                                 if [[ $test_ignor_err != "enable" ]]; then
 644                                         err "err interrupt ...\n"
 645                                         ret=1
 646                                 else
 647                                         ret=0
 648                                         [[ $err_continue_cnt -ge 10 ]] && ret=1
 649                                 fi
 650                         else
 651                                 if [[ $err_continue_cnt != 0 ]]; then
 652                                         err_continue_cnt=0
 653                                 fi
 654                                 ret=0
 655                         fi
 656                         if [[ -n $test_cnt_num && $test_cnt -ge $test_cnt_num ]]; then
 657                                 ret=1
 658                         fi
 659
 660 #                       curr_test_id[$saved_test_id]="${tmp[1]}"
 661 #                       TEST_DIR_DEPTH=$(( TEST_DIR_DEPTH - 1 ))
 662 #                       TPREFIX=$(printf "%*s" $(( $TEST_DIR_DEPTH * 4)) " ")
 663
 664                         cd ..
 665
 666                         [[ $ret == 1 ]] && break
 667                         ;;
 668                 "item" )
 669                         TPREFIX="$(printf '%*s' $(( $TEST_DIR_DEPTH * 4 + 4)) ' ')"
 670                         #dbgoutd "TEST_DIR_DEPTH=$TEST_DIR_DEPTH\n"
 671
 672                         test_cnt=$(( test_cnt + 1 ))
 673
 674                         test_item "${tmp[1]}" "${tmp[3]}" "${tmp[2]}"
 675
 676                         if [[ $? != 0 ]]; then
 677                                 err_cnt=$(( err_cnt + 1 ))
 678                                 err_continue_cnt=$(( err_continue_cnt + 1 ))
 679                                 if [[ $test_ignor_err != "enable" ]]; then
 680                                         err "err interrupt ...\n"
 681                                         ret=1
 682                                 else
 683                                         ret=0
 684                                         [[ $err_continue_cnt -ge 10 ]] && ret=1
 685                                 fi
 686                         else
 687                                 if [[ $err_continue_cnt != 0 ]]; then
 688                                         err_continue_cnt=0
 689                                 fi
 690                                 ret=0
 691                         fi
 692                         if [[ -n $test_cnt_num && $test_cnt -ge $test_cnt_num ]]; then
 693                                 ret=1
 694                         fi
 695
 696 #                       curr_test_id[$saved_test_id]="${tmp[1]}"
 697 #                       TEST_DIR_DEPTH=$(( TEST_DIR_DEPTH - 1 ))
 698 #                       TPREFIX=$(printf "%*s" $(( $TEST_DIR_DEPTH * 4)) " ")
 699
 700                         [[ $ret == 1 ]] && break
 701                         ;;
 702                 "module" | "unit" | "dir" )
 703                         if [[ -n $test_cnt_num && $test_cnt -gt $test_cnt_num ]]; then
 704                                 ret=1
 705                                 break
 706                         fi
 707
 708                         cataid_step_into TEST "${tmp[1]}"
 709
 710                         TPREFIX="$(printf '%*s' $(( $TEST_DIR_DEPTH * 4)) ' ')"
 711
 712                         info "$CHIGHL$FCCOLOR$TPREFIX[${tmp[0]}] ${tmp[1]}.\"${tmp[2]}\" begin testing ...$CNORMAL\n"
 713
 714                         test_unit "${tmp[1]}.${tmp[3]}.${tmp[2]}.${tmp[0]}"
 715                         ret=$?
 716
 717                         cataid_step_out TEST "${tmp[1]}"
 718
 719                         TPREFIX=$(printf "%*s" $(( $TEST_DIR_DEPTH * 4)) " ")
 720                         [[ $ret != 0 ]] && break
 721                         ;;
 722                 * )
 723                         break
 724                         ;;
 725                 esac
 726                 [[ $chk_ret == 3 ]] && break
 727         done; echo $test_cnt $err_cnt $err_continue_cnt $TEST_CID_STATE $TEST_DIR_DEPTH ; return $ret ); return $?`
 728
 729         # while语句由于|pipe操作符，以一个sub-process运行，所以使用return和exit都只是从sub-process中退出。
 730         # sub-proc中的环境变量test_cnt在return之前echo，在循环结束时echo，以stdout赋值给外部的test_cnt。
 731         ret=$?
 732
 733         test_cnt=( $test_cnt )
 734         err_cnt=${test_cnt[1]}
 735         err_continue_cnt=${test_cnt[2]}
 736         cataid_set_init_state TEST "${test_cnt[3]}"
 737
 738         TEST_DIR_DEPTH=${test_cnt[4]}
 739         unset test_cnt[1]
 740         unset test_cnt[2]
 741
 742         [[ $err_continue_cnt -ge 10 ]] && info "连续10个测试项failed，暂停测试。\n" && ret=1
 743
 744         cd ..
 745         return $ret
 746 }
 747
 748 skip_file_pfx="="
 749
 750 # 列出指定目录下的测试内容，包括模块、单元、目录、测试项
 751 # test_list_content <path>
 752 test_list_curr_content ()
 753 {
 754         local tmp=
 755         local script_name=
 756         local test_desc=
 757
 758         test_list_depth=$((test_list_depth++))
 759
 760         if [[ -z $1 ]]; then
 761                 tmp=
 762         else
 763                 tmp="\"$1\""
 764         fi
 765
 766         while read file; do
 767             # 忽略.txt文件，在处理.sh文件时再进行处理
 768             [[ $file =~ ".txt" ]] && continue
 769
 770                 # 以小数点分隔，包含3个字符串的为测试脚本
 771                 # 包含3个字符串的为测试模块/单元/目录
 772                 # 包含5个字符串的为测试项信息描述文件
 773                 OLD_IFS=$IFS
 774                 IFS="."
 775                 tmp=( $file )
 776                 IFS=$OLD_IFS
 777
 778         # todo：添加非数字idx过滤
 779 #               [[ ${tmp[1]} =~ "^[:digital:]" ]] && continue
 780
 781                 if [[ ${tmp[0]:0:1} == $skip_file_pfx ]]; then
 782                         # ignor this file with "=" pfx.
 783                         dbgoutd "'$file' is ignored.\n"
 784                 elif [[ ${#tmp[@]} == 5 ]]; then
 785                         test_desc=`ls -d -1 "$file.*.txt"`
 786                         [[ -z "$test_desc" ]] && echo "err: 123aaa txt description file not found for $file." >&2 && return 1
 787                         test_desc="${test_desc#$file\.}"
 788                         test_desc="${test_desc%\.txt}"
 789                         script_name=${tmp[1]}
 790                 elif [[ ${#tmp[@]} == 4 ]]; then
 791                         case ${tmp[3]} in
 792                                 "module" )
 793                                 echo "\"${tmp[3]}\" \"${tmp[0]}\" \"${tmp[2]}\" \"${tmp[1]}\""
 794                                 ;;
 795                                 "unit" )
 796                                 echo "\"${tmp[3]}\" \"${tmp[0]}\" \"${tmp[2]}\" \"${tmp[1]}\""
 797                                 ;;
 798                                 "dir" )
 799                                 echo "\"${tmp[3]}\" \"${tmp[0]}\" \"${tmp[2]}\" \"${tmp[1]}\""
 800                                 ;;
 801                         esac
 802                 elif [[ ${#tmp[@]} == 3 ]]; then
 803                         if [[ "${tmp[2]}" == "sh" ]]; then
 804                                 test_desc=`eval ls -d -1 "$file.*.txt"`
 805
 806                                 [[ -z $test_desc ]] && echo "err: txt descript file not found for $file." >&2 && return 1
 807                                 test_desc=${test_desc#$file\.}
 808                                 test_desc=${test_desc%\.txt}
 809
 810                                 echo "\"item\" \"${tmp[0]}\" \"$test_desc\" \"${tmp[1]}\""
 811                         fi
 812                 fi
 813         done < <(eval ls -1 "$tmp" | sort -n)
 814
 815         test_list_depth=$((test_list_depth--))
 816 }
 817
 818
 819
 820
 821
 822
 823
 824
 825
 826
 827
 828
 829
 830
 831 ############################################################################
 832 #
 833 ############################################################################
 834
 835
 836 FILE_EXT_LIST=
 837 DIR_TYPE_LIST=
 838
 839 vonThisDoc=onThisDoc
 840 vonContProc=onContProc
 841 vonDirFileProc=onDirFileProc
 842 vonItemFileProc=onItemFileProc
 843 vonDirSubjectDirEntering=onDirSubjectDirEntering
 844 vonItemEntering=onItemEntering
 845 vonItemSubjectDirEntering=onItemSubjectDirEntering
 846 vonItemExiting=onItemExiting
 847
 848 #include ../shlib/test.shlib
 849
 850 #
 851 # fsyntax: testing_paths_init [<base-tmp-dir>]
 852 # fdesc:
 853 testing_paths_init ()
 854 {
 855         #
 856         # 测试目录使用-d参数对应的test_dir环境变量
 857         # 指定的目录作为测试信息目录。
 858         # 默认为当前路径下的testing目录。
 859         #
 860         if [[ -n $test_dir ]]; then
 861                 if [[ -d $test_dir ]]; then
 862                         test_path=$test_dir
 863                 else
 864                         err "err: parameter -d specified a invalid path $test_dir\n"
 865                         exit
 866                 fi
 867         fi
 868
 869         echo $test_dir
 870         SRCPKG_DIR="$(pwd)"
 871         TEST_DIR_STR_FMT="$(envar_get catalog::testdir::STR_FMT)"
 872         TEST_SUBJECT_DIR="$SRCPKG_DIR/$(envar_get catalog::testdir::SUBJECTDIR)"
 873
 874         if [[ ! -d $test_path ]]; then
 875         test_path=testing/
 876         fi
 877
 878         [[ ! -d $test_path ]] && err "err: testcase dir is not exist.\n" && exit
 879
 880         mkdir -p "$test_path"
 881         cd "$test_path"
 882         test_path=$PWD
 883         cd - > /dev/null
 884
 885         #
 886         # 测试的tmp目录设置
 887         # 目录名称添加testcase路径的cksum，避免同一个软件包不同目录时，
 888         # 使用同一个tmp目录。
 889         #
 890         TESTING_TMP_DIR=$(basename $0)-$TESTING_TMP_DIR
 891         if [[ -n $1 ]]; then
 892                 TESTING_TMP_DIR="$1/"
 893                 [[ -d $1 ]] && TESTING_TMP_DIR=~/.testing/ && warn "directoy specified \"$1\" is not existing, use default directrory \"$TESTING_TMP_DIR\" instead.\n"
 894         else
 895                 TESTING_TMP_DIR=~/.testing/
 896         fi
 897
 898         mkdir -p "$TESTING_TMP_DIR"
 899         cd "$TESTING_TMP_DIR"
 900         TESTING_TMP_DIR=$PWD
 901         cd - > /dev/null
 902
 903         TESTING_TMP_DIR=$TESTING_TMP_DIR/$(basename $0)-$(echo $test_path | cksum - | cut -d ' ' -f1)
 904
 905         mkdir -p "$TESTING_TMP_DIR"
 906 }
 907
 908 #
 909 # fsyntax: testing_cataid_init
 910 # fdesc: range id init is a general feature programs. this function can be used in
 911 #      other programs.
 912 #
 913 testing_cataid_init ()
 914 {
 915         local tmp
 916
 917         cataid_create TEST
 918
 919         # load保存的id参数
 920         local tmp_exclude_id="$exclude_id"
 921         local tmp_begin_test_id="$begin_test_id"
 922         local tmp_end_test_id="$end_test_id"
 923
 924         [[ -e $TESTING_TMP_DIR/test_id.txt ]] && source $TESTING_TMP_DIR/test_id.txt
 925         [[ -n $tmp_exclude_id && $tmp_exclude_id =~ [^\ ] ]] && exclude_id="$tmp_exclude_id"
 926         [[ -n $tmp_begin_test_id ]] && begin_test_id="$tmp_begin_test_id"
 927         [[ -n $tmp_end_test_id ]] && end_test_id="$tmp_end_test_id"
 928         unset tmp_exclude_id
 929         unset tmp_begin_test_id
 930         unset tmp_end_test_id
 931
 932         #
 933         # 测试id起止范围和状态的初始化
 934         # 以testing目录中保存的起止id、参数设置、failed测试项的
 935         # 顺序设置id值。
 936         #
 937         if [[ $failed_begin == "enable" && -f $TESTING_TMP_DIR/testing_failed_list.txt ]]; then
 938                 tmp=$(head -n 1 $TESTING_TMP_DIR/testing_failed_list.txt)
 939                 [[ -n $tmp ]] && begin_test_id=$tmp
 940         fi
 941
 942         if [[ $test_all == "enable" ]]; then
 943                 exclude_id=""
 944                 test_cnt_num=
 945 #               test_ignor_err="enable"
 946         else
 947                 exclude_id=$(echo "${exclude_id}" | tr ' ' '\n' | sort - | uniq)
 948                 exclude_id=" $(echo "${exclude_id}" | tr "$'\n'" ' ') "
 949         fi
 950
 951         # inner
 952         cataid_set_init_state TEST "inner"
 953         if [[ -n $end_test_id ]]; then
 954                 cataid_set_end_id TEST end_test_id
 955         fi
 956
 957         if [[ -n $begin_test_id ]]; then
 958                 cataid_set_begin_id TEST begin_test_id
 959
 960                 # 如果有begin-id，默认起始状态为 lower
 961                 cataid_set_init_state TEST "lower"
 962         fi
 963
 964 #       dbgoutd "TEST_CID_STATE=${TEST_CID_STATE[@]}\n"
 965 #       dbgoutd "TEST_CID_BEGIN=${TEST_CID_BEGIN[@]}\n"
 966 #       dbgoutd "TEST_CID_END=${TEST_CID_END[@]}\n"
 967
 968         if [[ $test_all == "enable" ]]; then
 969                 begin_test_id=""
 970                 end_test_id=""
 971
 972                 cataid_set_begin_id TEST begin_test_id
 973                 cataid_set_end_id TEST end_test_id
 974
 975                 # 如果有begin-id，默认起始状态为 lower
 976                 cataid_set_init_state TEST "inner"
 977         fi
 978
 979         # id起止范围有效性判断
 980         cataid_begin_end_chk TEST
 981
 982         # 测试起始范围递增1
 983         if [[ $test_id_rollup == "enable" ]]; then
 984                 [[ -z $begin_test_id ]] && echo "请使用-b参数指定起始测试编号" && exit
 985
 986                 cataid_rollup_in_dir TEST begin_test_id $test_path
 987                 ret=$?
 988                 [[ $ret != 0 ]] && exit
 989
 990                 begin_test_id=${TEST_CID_BEGIN[@]}
 991                 begin_test_id=${begin_test_id// /.}
 992                 set_test_info="enable"
 993     fi
 994
 995         # 测试起始范围回退1
 996         if [[ $test_id_rollback == "enable" ]]; then
 997                 [[ -z $begin_test_id ]] && echo "请使用-b参数指定起始测试编号" && exit
 998
 999                 cataid_rollback_in_dir TEST begin_test_id $test_path
1000                 [[ $? != 0 ]] && exit
1001
1002                 begin_test_id=${TEST_CID_BEGIN[@]}
1003                 begin_test_id=${begin_test_id// /.}
1004                 set_test_info="enable"
1005         fi
1006
1007         # 保存-b -e参数指定的id
1008         if [[ $set_test_info == "enable" ]]; then
1009                 [[ -z $begin_test_id && -z $end_test_id && -z $exclude_id && ! $exclude_id =~ [^\ ] ]] && echo "请使用-x-b-e参数指定起止测试编号和exclude-id" && exit
1010
1011                 info "begin test_id: ${begin_test_id[@]}\n"
1012                 info "end test_id: ${end_test_id[@]}\n"
1013                 info "exclude test_id: '$exclude_id'\n"
1014
1015                 # 保存参数
1016                 mkdir -p "$TESTING_TMP_DIR"
1017                 echo "# [paramters]" > $TESTING_TMP_DIR/test_id.txt
1018                 [[ -n $begin_test_id ]] && echo "begin_test_id=${begin_test_id}" >> $TESTING_TMP_DIR/test_id.txt
1019                 [[ -n $end_test_id ]] && echo "end_test_id=${end_test_id}" >> $TESTING_TMP_DIR/test_id.txt
1020                 [[ -n $exclude_id && $exclude_id =~ [^\ ] ]] && echo "exclude_id='`echo ${exclude_id}`'" >> $TESTING_TMP_DIR/test_id.txt
1021
1022                 info "test id saved!\n"
1023
1024                 # 测试id保存时，不进行测试，便于testcase中测试起止id编号
1025                 exit
1026         fi
1027
1028         # dispaly test id range.
1029         if [[ -n ${begin_test_id[@]} || -n ${end_test_id[@]} ]]; then
1030                 info "Test from <${begin_test_id[@]}> to <${end_test_id[@]}>.\n"
1031                 [[ -n $exclude_id ]] && info "Except: ${exclude_id}\n"
1032         else
1033                 info "Full item testing.\n"
1034                 [[ $exclude_id =~ [^\ ] ]] && info "Except: ${exclude_id}\n"
1035         fi
1036         [[ -n ${test_cnt_num} ]] && info "Test item count: ${test_cnt_num}\n"
1037 }
1038
1039 test_result_info ()
1040 {
1041         if [[ $err_cnt != 0 ]]; then
1042                 info "$EFCCOLOR($FCGREEN$test_cnt$EFCCOLOR) items is tested，Error count is ($FCGREEN$err_cnt$EFCCOLOR)$CNORMAL\n"
1043                 return -1
1044         else
1045                 info "$FCGREEN($test_cnt) items is tested，Error count is ($err_cnt)$CNORMAL\n"
1046                 return 0
1047         fi
1048 }
1049
1050 onItemNodeCataNodeDisplay ()
1051 {
1052         NCATAID=$2
1053         NTYPE=$1
1054         NNAME=$4
1055         NDESC=$3
1056         strfmt @CATANODE_STR_FMT
1057         echo "$TPREFIX$CATANODE_STR"
1058 #       echo -ne "$TPREFIX[item] ${2} ${tmp4} \"${3}\"\n" # >&2
1059 }
1060
1061 onDirNodeCataNodeDisplay ()
1062 {
1063         NCATAID=$2
1064         NTYPE=$1
1065         NNAME=$4
1066         NDESC=$3
1067 #       declare -p NCATAID NTYPE NNAME NDESC
1068         strfmt @CATANODE_STR_FMT
1069         echo "$TPREFIX$CATANODE_STR"
1070 #       echo -ne "$TPREFIX[${1}] ${2} ${4} \"${3}\"\n" # >&2
1071 }
1072
1073
1074 ############################################################################
1075 # arg proc func
1076 ############################################################################
1077
1078 #
1079 # option action function list.
1080 #
1081 #
1082
1083 #
1084 #
1085 args_list_item ()
1086 {
1087         include param/param.shlib
1088
1089         testing_paths_init
1090
1091         # 使用测试目录中的begin-id和end-id
1092         cataid_init TEST
1093         testing_cataid_init
1094
1095         # Init Testing Path
1096         SRCPKG_DIR="$(pwd)"
1097         TEST_DIR_STR_FMT="$(envar_get catalog::testdir::STR_FMT)"
1098         vonItemNode=onItemNodeCataNodeDisplay
1099         vonDirNode=onDirNodeCataNodeDisplay
1100         CATANODE_STR_FMT="$(envar_get catalog::catanode::STR_FMT)"
1101
1102         if [[ $test_force == "enable" ]]; then
1103                 list_full_content $test_path
1104         else
1105                 list_full_content $test_path | more
1106         fi
1107
1108         exit
1109 }
1110
1111 #
1112 #
1113 args_save_catalog ()
1114 {
1115         info "generate catalog from testing dir!\n"
1116
1117         [[ -z $1 ]] && return
1118
1119         test_force="enable"
1120         list_fullrange=1
1121         args_list_item > $1
1122
1123         exit
1124 }
1125
1126 #
1127 # 描述文件中第一个有效行的空格数或tab数为缩进字符串的单位，用于计算目录层数。
1128 # tabh空格不能混用。
1129 #
1130 args_sync_to_dir ()
1131 {
1132         local tmp="$1"
1133         local path
1134
1135         info "generate testing dir framework from catalog!\n"
1136
1137         [[ -z $tmp || ! -f $tmp ]] && tmp=$3
1138
1139         # 初始化测试目录
1140         testing_paths_init
1141
1142         [[ ! -f $tmp ]] && tmp="$test_path/${DEF_TESTING_CATALOG}"
1143         [[ ! -f $tmp ]] && err "err: file ($tmp) does not exist.\n" && exit
1144
1145         path=$(basename $tmp)
1146         tmp=$(dirname $tmp)
1147         cd "$tmp"
1148         path=$(pwd)/$path
1149         cd - 2>/dev/null
1150
1151         #
1152         # todo:
1153         # 比较已有的testcase目录与func_list，对差异部分进行更新。
1154         #
1155 #       catalog_content_cmp $path
1156 #       output_dir_cmp $path
1157
1158         cata_proc_init
1159         catalog_2_dirfiles $path
1160 ##      catalog_2_dir $path
1161
1162         exit
1163 }
1164
1165 #
1166 # functions below are for testing.
1167 #
1168
1169 #
1170 # todo：-F参数的-n未添加，以及err_cnt信息
1171 args_failed_test ()
1172 {
1173         local test_id
1174         local id
1175         local i
1176         local cnt
1177         local tmp
1178         local curr_path=$PWD
1179         local ret
1180
1181         cataid_set_begin_id TEST ""
1182         cataid_set_end_id TEST ""
1183 #       begin_test_id=""
1184 #       end_test_id=""
1185
1186         # 初始化测试目录
1187         testing_paths_init
1188         cataid_init TEST
1189
1190         info "failed item testing!\n"
1191
1192 #       dbgout_cmd cat $TESTING_TMP_DIR/testing_failed_list.txt
1193
1194         [[ ! -f $TESTING_TMP_DIR/testing_failed_list.txt ]] && warn "testing failed recode is not exist.\n" && exit
1195
1196         while read test_id; do
1197                 [[ -z $test_id ]] && continue
1198                 args_test_id $test_id "continue"
1199                 ret=$?
1200                 if [[ $test_ignor_err != "enable" ]]; then
1201                         err "err interrupt ...\n"
1202                         break
1203                 fi
1204                 continue
1205         done < $TESTING_TMP_DIR/testing_failed_list.txt
1206
1207         if [[ -e $TESTING_TMP_DIR/testing_failed_list.txt ]]; then
1208                 cat $TESTING_TMP_DIR/testing_failed_list.txt | sort -n | uniq > ~/testing_failed_list.txt
1209                 mv ~/testing_failed_list.txt $TESTING_TMP_DIR/testing_failed_list.txt
1210         fi
1211
1212         exit
1213 }
1214
1215 #
1216 #
1217 #
1218 args_all ()
1219 {
1220         declare -g -x test_all="enable"
1221         declare -g -x test_ignor_err="enable"
1222         declare -g -x quiet="enable"
1223 }
1224
1225 #
1226 #
1227 args_test_id ()
1228 {
1229         local test_id
1230         local id
1231         local i
1232         local cnt
1233         local tmp
1234         local curr_path=$PWD
1235         local ret
1236
1237         cataid_set_begin_id TEST ""
1238         cataid_set_end_id TEST ""
1239 #       begin_test_id=""
1240 #       end_test_id=""
1241
1242         # 初始化测试目录
1243         testing_paths_init
1244         cataid_init TEST
1245
1246         test_id=$1
1247         [[ ! ( -n $2 && "$2" == "continue" ) ]] && info "test specified item($test_id).!\n"
1248
1249         [[ "$1" =~ [^0-9.] ]] && test_id=$3
1250         [[ "$test_id" =~ [^0-9.] ]] && err "test id($test_id) is not correct.\n" && exit
1251
1252         OLD_IFS=$IFS
1253         IFS="."
1254         test_id=( $test_id )
1255         cnt=${#test_id[@]}
1256         IFS=$OLD_IFS
1257
1258         cd "$test_path"
1259         IFS=$'\n'
1260         for (( i=0; i<cnt; i++ )); do
1261                 [[ -z ${test_id[$i]} ]] && err "err: test_id[$i] does not invalid.\n" && exit
1262                 id=${test_id[$i]}
1263                 tmp="`find $id.* -maxdepth 1 -type d 2>/dev/null`"
1264                 tmp=( $tmp )
1265
1266                 if [[ -d $tmp ]]; then
1267                         cd "$tmp"
1268                         tmp=""
1269                         cataid_step_into TEST "$id"
1270                         continue
1271                 fi
1272                 tmp=( `ls -1 "$id".*.sh 2>/dev/null` )
1273
1274                 [[ -e ${tmp[0]} ]] && break
1275                 warn "test id '$1' is not a valid id.\n"
1276                 exit
1277         done
1278         IFS=$OLD_IFS
1279
1280         # use the script code to retest for loop back testing.
1281         if [[ $i == $cnt ]]; then
1282                 test_unit .
1283                 ret=$?
1284                 test_result_info
1285
1286                 [[ -z $2 || "$2" == "continue" ]] && exit
1287
1288                 return $ret
1289         fi
1290
1291         # todo：这里可以改为测试一个unit
1292         [[ -z $tmp ]]  && err "err: test id ($1) does not specify a valide test item file.\n" && exit
1293
1294         tmp=`ls -d "$id".*.txt`
1295         [[ -z $tmp ]] && err "err: test id ($i) does not contain a corresponding txt description file.\n" && exit
1296
1297         OLD_IFS=$IFS
1298         IFS="."
1299         tmp=( $tmp )
1300         IFS=$OLD_IFS
1301
1302         cataid_step_to TEST "${tmp[0]}"
1303
1304         test_item "${tmp[0]}" "${tmp[1]}" "${tmp[3]}"
1305         ret=$?
1306
1307         [[ ! ( -n $2 && "$2" == "continue" ) ]] && exit
1308
1309         return $ret
1310 }
1311
1312 args_clean ()
1313 {
1314         info "clean testing temp dir!\n"
1315 }
1316
1317 #
1318 #
1319 args_exclude ()
1320 {
1321         exclude_id+=" $1"
1322 }
1323
1324 #
1325 # 测试运脚本输出信息的显示。
1326 args_verbose ()
1327 {
1328         IFCCOLOR="$FCBLUE"
1329 }
1330
1331 #
1332 # 输出程序中的变量信息。
1333 #
1334 args_print_vars ()
1335 {
1336         info "printf vars in scripttest!\n"
1337
1338         OptDescParamPrint
1339
1340         exit
1341 }
1342
1343 #
1344 # 输出信息单色显示。
1345 # args_mono
1346 args_mono ()
1347 {
1348         FCCOLOR=""
1349         IFCCOLOR=""
1350         EFCCOLOR=""
1351 }
1352
1353 #
1354 # 显示版本信息。
1355 # args_version
1356 args_version ()
1357 {
1358         echo "$PROG_VERSION"
1359
1360         exit
1361 }
1362
1363 #
1364 #
1365 args_h_info ()
1366 {
1367         echo -ne "Options:\n"
1368         opt_helper
1369         echo -ne "\nuse '$SCRT_PROG --help' for more details.\n"
1370         exit
1371 }
1372
1373 #
1374 # args_help_info
1375 args_help_info ()
1376 {
1377         [[ -z $term_width ]] && term_width=80
1378
1379         helper="$(opt_helper)"
1380
1381         eval SCRT_PROG_SYNTAX="\"$SCRT_PROG_SYNTAX\""
1382         eval echo -ne "\"$SCRT_USAGE_FMT\""
1383
1384         exit
1385 }
1386
1387 #
1388 # 显示调试信息，这里显示desc-str解析和参数解析后的环境变量信息。
1389 #
1390 args_test_debug ()
1391 {
1392         :
1393 }
1394
1395 ############################################################################
1396 #
1397 ############################################################################
1398
1399 #
1400 # main function
1401 #
1402
1403 main ()
1404 {
1405         #
1406         # program arguments resolve.
1407         # it invoke opt_desc_str_dispatch and prog_opt_proc seperatly,
1408         # instead of prog_opt_dispatch, for multiple desc-str.
1409         # 测试起止范围id的初始化
1410         # 初始化测试的tmp目录路径，设置testcase的目录
1411         #
1412         # prog_opt_dispatch "$@"
1413         opt_desc_str_dispatch scripttest_desc_hdr_str
1414         opt_desc_str_dispatch codegen_desc_str
1415         opt_desc_str_dispatch scripttest_desc_str
1416         opt_desc_str_dispatch scripttest_other_desc_str
1417
1418 #       OptDescParamPrint
1419         prog_opt_proc "$@"
1420
1421         # todo:
1422         # ini文件加载环境变量
1423
1424         init_dbglogout 2 testing 20000
1425 #       set_output_prefix info ""
1426         set_auto_newline ""
1427
1428         #
1429         # running action list function for options after init.
1430         # the paramter for init will be effact, then execute the
1431         # action function.
1432         #
1433         action_list_exec
1434
1435         testing_paths_init $output_dir
1436         if [[ $clean == "enable" ]]; then
1437                 info "re-init tmp dir.\n"
1438                 rm "$TESTING_TMP_DIR" -rf
1439                 exit
1440         fi
1441
1442         # 使用测试目录中的begin-id和end-id
1443         cataid_init TEST
1444         testing_cataid_init
1445
1446         #
1447         # test the specified dir.
1448         #
1449         info "begin testing ... \n"
1450         test_unit "$test_path"
1451         if [[ -e $TESTING_TMP_DIR/testing_failed_list.txt ]]; then
1452                 cat $TESTING_TMP_DIR/testing_failed_list.txt | sort -n | uniq > ~/testing_failed_list.txt
1453                 mv ~/testing_failed_list.txt $TESTING_TMP_DIR/testing_failed_list.txt
1454         fi
1455         info "end testing ...\n"
1456
1457         #
1458         # testing result info
1459         #
1460         test_result_info
1461         exit $?
1462 }
1463
1464 ##############################
1465 # section: public function
1466 ##############################
1467
1468 #
1469 #
1470 # fdesc: the wrap of main(), and it append init code of some feature.
1471 #
1472 ExeMain ()
1473 {
1474         if [[ "$@" =~ "--debug" ]]; then
1475                 declare -g DBGOUTD_OUTPUT=1
1476
1477                 # does not need to shift args if the --debug option is not the
1478                 # first option. it will be ignore in process.
1479 #               shift
1480         fi
1481
1482 if [[ $1 != "--loadshlib" ]]; then
1483         main "$@"
1484 else
1485         shift
1486 fi
1487 }
1488
1489 ExeMain "$@"
1490 unset DBGOUTD_OUTPUT
1491
1492 ##############################
1493 # section: file tail
1494 ##############################
1495
1496
1497 #
1498 #|blank 'Design File Auto Gen Paramters:'
1499 #|param |-l |--list         |---    |=                 |%<list_test_list>    |&<args_list_item>      |'list items and dirs under testing dir.'
1500 #|param |-S |--save         |---    |=<save_desc_file> |%<save_desc_file>    |&<args_save_catalog>   |'generate catalog from testing dir. if the catalog file is exist, just display desc info only. append -f option to recover an original catalog file. '
1501 ## -y参数存在bug，不添加到程序功能中。
1502 #|param |-y |--sync         |---    |=[sync_to_dir]     |%<sync_to_dir>       |&<args_sync_to_dir>    |'generate 根据测试信息描述文件，生成测试目录及测试脚本等文件，用于用户编写各种测试用例。'
1503 #|blank
1504 #
1505 #|param |-M |--matching     |---    |=<matching>       |%<matching>          |&<args_matching>       |'string matching test, and generate chksum.'
1506 #|blank
1507
1508 asdfjlkj="
1509 ##
1510 |blank 'Design File Auto Gen Paramters:'
1511 |param |-l |--list         |---    |=                 |%<list_test_list>    |&<args_list_item>      |'list items and dirs under testing dir.'
1512 |param |-S |--save         |---    |=<save_desc_file> |%<save_desc_file>    |&<args_save_catalog>   |'generate catalog from testing dir. if the catalog file is exist, just display desc info only. append -f option to recover an original catalog file. '
1513 # -y参数存在bug，不添加到程序功能中。
1514 |param |-y |--sync        |---    |=[sync_to_dir]    |%<sync_to_dir>       |&<args_sync_to_dir>    |'generate 根据测试信息描述文件，生成测试目录及测试脚本等文件，用于用户编写各种测试用例。'
1515 ##
1516 "
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535 #
1536 # fsyntax: test_item <id> <script_name> <desc_str>
1537 # fdesc: one item testing.
1538 #
1539 test_item_abc ()
1540 {
1541         local ret=
1542         local tmp=
1543         local script_name=
1544         local test_desc=
1545         local exetime=
1546         local tmp_vch_output=
1547
1548         NCATAID="$1"
1549         NNAME="$2"
1550         NDESC="$3"
1551         # NTYPE NFTYPE NFEXT DIR_TYPE
1552
1553         script_name="$1.$2.sh"
1554         test_desc="$3"
1555
1556         # paramter validation judgement.
1557         if [[ ! -f $script_name ]]; then
1558                 err "err: file $script_name is not exist.\n"
1559                 return 1
1560         fi
1561
1562         if [[ -z "$test_desc" ]]; then
1563                 err "err: descript string is missing.\n"
1564                 return 1
1565         fi
1566
1567         if [[ ! -f $script_name."$test_desc".txt ]]; then
1568                 err "err: file $script_name.$test_desc.txt is not exist.\n"
1569                 return 1
1570         fi
1571
1572         # item testing string output.
1573 #       info "$CHIGHL$IFCCOLOR$TPREFIX[item] $1 $2 \"$3\" ... "
1574         info "$CHIGHL$IFCCOLOR$TPREFIX[item] $1.\"$3\" ... "
1575
1576         #
1577         # copy test item file to temp dir, it keeps every testing
1578         # file can not be modified by test script.
1579         #
1580         rm "$TESTING_TMP_DIR/$1.$2/" -rf
1581         mkdir -p "$TESTING_TMP_DIR/$1.$2/"{,scripts}
1582         cp "$1".* "$TESTING_TMP_DIR/$1.$2/" -rf
1583         cp -rf scripts/"$1".* "$TESTING_TMP_DIR/$1.$2"/scripts/ 2> /dev/null
1584         # XXX: here should be modified, if the scripts dir is very large,
1585         # it will copy every times for every test script.
1586         cp -rf scripts/* "$TESTING_TMP_DIR/$1.$2"/scripts/ 2> /dev/null
1587         cp -rf scripts/testing/ "$TESTING_TMP_DIR/$1.$2"/scripts/ 2> /dev/null
1588         cd "$TESTING_TMP_DIR/$1.$2"
1589 #       dbgout_cmd ls -l
1590         chmod +x *.sh scripts/* 2> /dev/null
1591
1592         # dispaly testing time cost if this item would cost more then 5 seconds.
1593         if [[ -f $1.$2.time.txt ]]; then
1594                 exetime=`head -n 2 $1.$2.time.txt`
1595                 tmp=${exetime##$'\nreal '}
1596                 tmp=${tmp%%m*}
1597                 [[ -z $tmp ]] && err "err: the syntax of test time info is not correct.\n"
1598                 exetime=${exetime##*m}
1599                 exetime=${exetime%.*}
1600                 exetime=$(( tmp * 60 + exetime ))
1601                 # dispaly only longer then 5 seconds.
1602                 [[ -n $exetime && $exetime -gt 5 ]] && info "(time:${exetime}s) "
1603         fi
1604
1605         #
1606         # here, re-direct output of test script, include stdout and stderr.
1607         # using ttyout to force string output when debug this code, event if
1608         # the stdout and stderr is re-directed.
1609         #
1610 #       ttyout dbgout_ttydev=$dbgout_ttydev
1611
1612         # set output stream pipe.
1613         tmp_vch_output=$vch_progstdout
1614         [[ $output_disable_list =~ "info" ]] && tmp_vch_output=$vch_null
1615
1616         TEST_CMD="./$script_name"
1617         TEST_OPT=
1618         eval TEST_CMDLINE="\"${TEST_CMDLINE_FMT}\""
1619
1620         info "$CNORMAL"
1621         # 带运行时间信息输出的测试项测试
1622         if [[ $verbose == "enable" ]]; then
1623                 #
1624                 # time语句的code-block，stdout调试信息输出使用stderr，time语句输出到stderr，设置stderr输出为time.txt
1625                 # time语句内的code-block，stderr输出到stdout
1626                 # 这样，time的stderr输出到time.txt文件，time之内的code-block输出到vch_dbgout(stderr)
1627                 #
1628                 {
1629                         time ( {
1630                                 info "\n${INFOCOLOR} + ###############################################${CNORMAL}\n" ;
1631                                 { { $TEST_CMDLINE; echo; } 2>&1 | tee log.txt; } | { IFS=$'\n'; while read -r line; do echo -ne "$verbos_pfx$line\n"; done } ;
1632                                 info "${INFOCOLOR} + ###############################################${CNORMAL}\n" ;
1633                                 } 2>&1 )
1634                 } 5>&1 >&$tmp_vch_output 2>time.txt
1635
1636 #               ( time (
1637 #               info "abc\n";
1638 #               ttyout "testing\n";
1639 #                                       info "\n###############################################\n" ;
1640 #                                       ttyout "testing\n";
1641 #                                       # 输出测试脚本的stdout和stderr到管道，管道的调试信息使用tee在控制台stdout输出，并保存到日志文件
1642 #                                       ( ./$script_name 2>&1 ) | tee log.txt;
1643 #                                       info "###############################################\n" ;
1644 #                                       ttyout "testing\n";
1645 #                               ) 2>&1 # | { IFS=$'\n'; while read -r line; do echo "$verbos_pfx$line"; done } >&$vch_dbgout #2> /dev/null
1646 #               ) 2> time.txt #2>&1
1647         else
1648                 # an echo cmd append at the end of test script.
1649                 # script output string though pipe, but there must be a '\n' at the end
1650                 # of string, or it will not be displayed on screen.
1651                 ( time ( ( $TEST_CMDLINE; echo; ) > log.txt 2>&1 ) ) 5>&1 2> time.txt #2>&1
1652         fi
1653
1654         # append to log file.
1655         if [[ ! $logout_disable_list =~ "info" ]]; then
1656 #               echo -ne "$INFO_PFX$@$NEWLINE_SFX" | tee -a $f_logout >&$vch_progstdout
1657                 while read tmp; do
1658                         echo "$tmp" >> $f_logout
1659                         log_size_limit
1660                 done < <(cat log.txt)
1661         fi
1662
1663         # TBD:
1664 #       cp log.txt $PWD/log/
1665         diff log.txt "$script_name.$test_desc".txt > diff.txt
1666         if [[ -z "`head -n 5 diff.txt`" ]]; then
1667                 info "$CHIGHL$INFOCOLOR[  ok  ]$CNORMAL\n"
1668                 ret=0
1669
1670                 # 如果有failed record，进行记录。
1671                 sed -i $TESTING_TMP_DIR/testing_failed_list.txt -e "/^$(cataid_get TEST)$/d" 2>/dev/null
1672         else
1673                 info "$CHIGHL$EFCCOLOR[failed]$CNORMAL\n"
1674                 if [[ $quiet != "enable" ]]; then
1675                         info "testing result difference from correct output.\n"
1676                         info "stdout > testout\n"
1677                         info "###############################################\n"
1678                         cat diff.txt |
1679                         while read data; do
1680                                 case ${data:0:1} in
1681                                         '>' )
1682                                                 info -ne "${FCL2R}>${CNORMAL}${data:1}\n"
1683                                                 ;;
1684                                         '<' )
1685                                                 info -ne "${FCR2L}<${CNORMAL}${data:1}\n"
1686                                                 ;;
1687                                         * )
1688                                                 info -ne "${data}\n"
1689                                                 ;;
1690                                 esac
1691                         done
1692 #                       dbgout_cmd cat diff.txt # >&2
1693                         # 将测试failed项的日志保存到log目录，便于测试后的查看。
1694                         info "###############################################\n"
1695                 fi
1696                 ret=1
1697
1698                 # faled record
1699                 sed -i $TESTING_TMP_DIR/testing_failed_list.txt -e "/^$(cataid_get TEST)$/d" 2>/dev/null
1700                 cataid_get TEST >> $TESTING_TMP_DIR/testing_failed_list.txt
1701         fi
1702
1703         cd - > /dev/null
1704         if [[ $update_test_file == "enable" ]]; then
1705                 cp "$TESTING_TMP_DIR/$1.$2"/log.txt "$script_name.$test_desc".txt
1706                 exetime=`head -n 2 $TESTING_TMP_DIR/$1.$2/time.txt`
1707                 tmp=${exetime%%m*}
1708                 tmp=${tmp##`echo -ne "\nreal    "`} #   }
1709                 exetime=${exetime##*m}
1710                 exetime=${exetime%.*}
1711                 exetime=$(( tmp * 60 + exetime ))
1712                 [[ -n $exetime && $exetime -gt 5 ]] && cp "$TESTING_TMP_DIR/$1.$2"/time.txt "$1.$2".time.txt
1713         fi
1714
1715         # clean testing info
1716         rm "$TESTING_TMP_DIR/$1.$2/" -rf
1717
1718         return $ret
1719 }
1720
1721