tools/testing/selftests/ntb/ntb_test.sh

   1 #!/bin/bash
   2 # SPDX-License-Identifier: GPL-2.0-or-later
   3 # Copyright (c) 2016 Microsemi. All Rights Reserved.
   4 #
   5 # Author: Logan Gunthorpe <logang@deltatee.com>
   6
   7 REMOTE_HOST=
   8 LIST_DEVS=FALSE
   9
  10 DEBUGFS=${DEBUGFS-/sys/kernel/debug}
  11
  12 PERF_RUN_ORDER=32
  13 MAX_MW_SIZE=0
  14 RUN_DMA_TESTS=
  15 DONT_CLEANUP=
  16 MW_SIZE=65536
  17
  18 function show_help()
  19 {
  20         echo "Usage: $0 [OPTIONS] LOCAL_DEV REMOTE_DEV"
  21         echo "Run tests on a pair of NTB endpoints."
  22         echo
  23         echo "If the NTB device loops back to the same host then,"
  24         echo "just specifying the two PCI ids on the command line is"
  25         echo "sufficient. Otherwise, if the NTB link spans two hosts"
  26         echo "use the -r option to specify the hostname for the remote"
  27         echo "device. SSH will then be used to test the remote side."
  28         echo "An SSH key between the root users of the host would then"
  29         echo "be highly recommended."
  30         echo
  31         echo "Options:"
  32         echo "  -C              don't cleanup ntb modules on exit"
  33         echo "  -h              show this help message"
  34         echo "  -l              list available local and remote PCI ids"
  35         echo "  -r REMOTE_HOST  specify the remote's hostname to connect"
  36         echo "                  to for the test (using ssh)"
  37         echo "  -m MW_SIZE      memory window size for ntb_tool"
  38         echo "                  (default: $MW_SIZE)"
  39         echo "  -d              run dma tests for ntb_perf"
  40         echo "  -p ORDER        total data order for ntb_perf"
  41         echo "                  (default: $PERF_RUN_ORDER)"
  42         echo "  -w MAX_MW_SIZE  maxmium memory window size for ntb_perf"
  43         echo
  44 }
  45
  46 function parse_args()
  47 {
  48         OPTIND=0
  49         while getopts "b:Cdhlm:r:p:w:" opt; do
  50                 case "$opt" in
  51                 C)  DONT_CLEANUP=1 ;;
  52                 d)  RUN_DMA_TESTS=1 ;;
  53                 h)  show_help; exit 0 ;;
  54                 l)  LIST_DEVS=TRUE ;;
  55                 m)  MW_SIZE=${OPTARG} ;;
  56                 r)  REMOTE_HOST=${OPTARG} ;;
  57                 p)  PERF_RUN_ORDER=${OPTARG} ;;
  58                 w)  MAX_MW_SIZE=${OPTARG} ;;
  59                 \?)
  60                     echo "Invalid option: -$OPTARG" >&2
  61                     exit 1
  62                     ;;
  63                 esac
  64         done
  65 }
  66
  67 parse_args "$@"
  68 shift $((OPTIND-1))
  69 LOCAL_DEV=$1
  70 shift
  71 parse_args "$@"
  72 shift $((OPTIND-1))
  73 REMOTE_DEV=$1
  74 shift
  75 parse_args "$@"
  76
  77 set -e
  78
  79 function _modprobe()
  80 {
  81         modprobe "$@" || return 1
  82
  83         if [[ "$REMOTE_HOST" != "" ]]; then
  84                 ssh "$REMOTE_HOST" modprobe "$@" || return 1
  85         fi
  86 }
  87
  88 function split_remote()
  89 {
  90         VPATH=$1
  91         REMOTE=
  92
  93         if [[ "$VPATH" == *":/"* ]]; then
  94                 REMOTE=${VPATH%%:*}
  95                 VPATH=${VPATH#*:}
  96         fi
  97 }
  98
  99 function read_file()
 100 {
 101         split_remote $1
 102         if [[ "$REMOTE" != "" ]]; then
 103                 ssh "$REMOTE" cat "$VPATH"
 104         else
 105                 cat "$VPATH"
 106         fi
 107 }
 108
 109 function write_file()
 110 {
 111         split_remote $2
 112         VALUE=$1
 113
 114         if [[ "$REMOTE" != "" ]]; then
 115                 ssh "$REMOTE" "echo \"$VALUE\" > \"$VPATH\""
 116         else
 117                 echo "$VALUE" > "$VPATH"
 118         fi
 119 }
 120
 121 function check_file()
 122 {
 123         split_remote $1
 124
 125         if [[ "$REMOTE" != "" ]]; then
 126                 ssh "$REMOTE" "[[ -e ${VPATH} ]]"
 127         else
 128                 [[ -e ${VPATH} ]]
 129         fi
 130 }
 131
 132 function subdirname()
 133 {
 134         echo $(basename $(dirname $1)) 2> /dev/null
 135 }
 136
 137 function find_pidx()
 138 {
 139         PORT=$1
 140         PPATH=$2
 141
 142         for ((i = 0; i < 64; i++)); do
 143                 PEER_DIR="$PPATH/peer$i"
 144
 145                 check_file ${PEER_DIR} || break
 146
 147                 PEER_PORT=$(read_file "${PEER_DIR}/port")
 148                 if [[ ${PORT} -eq $PEER_PORT ]]; then
 149                         echo $i
 150                         return 0
 151                 fi
 152         done
 153
 154         return 1
 155 }
 156
 157 function port_test()
 158 {
 159         LOC=$1
 160         REM=$2
 161
 162         echo "Running port tests on: $(basename $LOC) / $(basename $REM)"
 163
 164         LOCAL_PORT=$(read_file "$LOC/port")
 165         REMOTE_PORT=$(read_file "$REM/port")
 166
 167         LOCAL_PIDX=$(find_pidx ${REMOTE_PORT} "$LOC")
 168         REMOTE_PIDX=$(find_pidx ${LOCAL_PORT} "$REM")
 169
 170         echo "Local port ${LOCAL_PORT} with index ${REMOTE_PIDX} on remote host"
 171         echo "Peer port ${REMOTE_PORT} with index ${LOCAL_PIDX} on local host"
 172
 173         echo "  Passed"
 174 }
 175
 176 function link_test()
 177 {
 178         LOC=$1
 179         REM=$2
 180         EXP=0
 181
 182         echo "Running link tests on: $(subdirname $LOC) / $(subdirname $REM)"
 183
 184         if ! write_file "N" "$LOC/../link" 2> /dev/null; then
 185                 echo "  Unsupported"
 186                 return
 187         fi
 188
 189         write_file "N" "$LOC/link_event"
 190
 191         if [[ $(read_file "$REM/link") != "N" ]]; then
 192                 echo "Expected link to be down in $REM/link" >&2
 193                 exit -1
 194         fi
 195
 196         write_file "Y" "$LOC/../link"
 197
 198         echo "  Passed"
 199 }
 200
 201 function doorbell_test()
 202 {
 203         LOC=$1
 204         REM=$2
 205         EXP=0
 206
 207         echo "Running db tests on: $(basename $LOC) / $(basename $REM)"
 208
 209         DB_VALID_MASK=$(read_file "$LOC/db_valid_mask")
 210
 211         write_file "c $DB_VALID_MASK" "$REM/db"
 212
 213         for ((i = 0; i < 64; i++)); do
 214                 DB=$(read_file "$REM/db")
 215                 if [[ "$DB" -ne "$EXP" ]]; then
 216                         echo "Doorbell doesn't match expected value $EXP " \
 217                              "in $REM/db" >&2
 218                         exit -1
 219                 fi
 220
 221                 let "MASK = (1 << $i) & $DB_VALID_MASK" || true
 222                 let "EXP = $EXP | $MASK" || true
 223
 224                 write_file "s $MASK" "$LOC/peer_db"
 225         done
 226
 227         write_file "c $DB_VALID_MASK" "$REM/db_mask"
 228         write_file $DB_VALID_MASK "$REM/db_event"
 229         write_file "s $DB_VALID_MASK" "$REM/db_mask"
 230
 231         write_file "c $DB_VALID_MASK" "$REM/db"
 232
 233         echo "  Passed"
 234 }
 235
 236 function get_files_count()
 237 {
 238         NAME=$1
 239         LOC=$2
 240
 241         split_remote $LOC
 242
 243         if [[ "$REMOTE" == "" ]]; then
 244                 echo $(ls -1 "$LOC"/${NAME}* 2>/dev/null | wc -l)
 245         else
 246                 echo $(ssh "$REMOTE" "ls -1 \"$VPATH\"/${NAME}* | \
 247                        wc -l" 2> /dev/null)
 248         fi
 249 }
 250
 251 function scratchpad_test()
 252 {
 253         LOC=$1
 254         REM=$2
 255
 256         echo "Running spad tests on: $(subdirname $LOC) / $(subdirname $REM)"
 257
 258         CNT=$(get_files_count "spad" "$LOC")
 259
 260         if [[ $CNT -eq 0 ]]; then
 261                 echo "  Unsupported"
 262                 return
 263         fi
 264
 265         for ((i = 0; i < $CNT; i++)); do
 266                 VAL=$RANDOM
 267                 write_file "$VAL" "$LOC/spad$i"
 268                 RVAL=$(read_file "$REM/../spad$i")
 269
 270                 if [[ "$VAL" -ne "$RVAL" ]]; then
 271                         echo "Scratchpad $i value $RVAL doesn't match $VAL" >&2
 272                         exit -1
 273                 fi
 274         done
 275
 276         echo "  Passed"
 277 }
 278
 279 function message_test()
 280 {
 281         LOC=$1
 282         REM=$2
 283
 284         echo "Running msg tests on: $(subdirname $LOC) / $(subdirname $REM)"
 285
 286         CNT=$(get_files_count "msg" "$LOC")
 287
 288         if [[ $CNT -eq 0 ]]; then
 289                 echo "  Unsupported"
 290                 return
 291         fi
 292
 293         MSG_OUTBITS_MASK=$(read_file "$LOC/../msg_inbits")
 294         MSG_INBITS_MASK=$(read_file "$REM/../msg_inbits")
 295
 296         write_file "c $MSG_OUTBITS_MASK" "$LOC/../msg_sts"
 297         write_file "c $MSG_INBITS_MASK" "$REM/../msg_sts"
 298
 299         for ((i = 0; i < $CNT; i++)); do
 300                 VAL=$RANDOM
 301                 write_file "$VAL" "$LOC/msg$i"
 302                 RVAL=$(read_file "$REM/../msg$i")
 303
 304                 if [[ "$VAL" -ne "${RVAL%%<-*}" ]]; then
 305                         echo "Message $i value $RVAL doesn't match $VAL" >&2
 306                         exit -1
 307                 fi
 308         done
 309
 310         echo "  Passed"
 311 }
 312
 313 function get_number()
 314 {
 315         KEY=$1
 316
 317         sed -n "s/^\(${KEY}\)[ \t]*\(0x[0-9a-fA-F]*\)\(\[p\]\)\?$/\2/p"
 318 }
 319
 320 function mw_alloc()
 321 {
 322         IDX=$1
 323         LOC=$2
 324         REM=$3
 325
 326         write_file $MW_SIZE "$LOC/mw_trans$IDX"
 327
 328         INB_MW=$(read_file "$LOC/mw_trans$IDX")
 329         MW_ALIGNED_SIZE=$(echo "$INB_MW" | get_number "Window Size")
 330         MW_DMA_ADDR=$(echo "$INB_MW" | get_number "DMA Address")
 331
 332         write_file "$MW_DMA_ADDR:$(($MW_ALIGNED_SIZE))" "$REM/peer_mw_trans$IDX"
 333
 334         if [[ $MW_SIZE -ne $MW_ALIGNED_SIZE ]]; then
 335                 echo "MW $IDX size aligned to $MW_ALIGNED_SIZE"
 336         fi
 337 }
 338
 339 function write_mw()
 340 {
 341         split_remote $2
 342
 343         if [[ "$REMOTE" != "" ]]; then
 344                 ssh "$REMOTE" \
 345                         dd if=/dev/urandom "of=$VPATH" 2> /dev/null || true
 346         else
 347                 dd if=/dev/urandom "of=$VPATH" 2> /dev/null || true
 348         fi
 349 }
 350
 351 function mw_check()
 352 {
 353         IDX=$1
 354         LOC=$2
 355         REM=$3
 356
 357         write_mw "$LOC/mw$IDX"
 358
 359         split_remote "$LOC/mw$IDX"
 360         if [[ "$REMOTE" == "" ]]; then
 361                 A=$VPATH
 362         else
 363                 A=/tmp/ntb_test.$$.A
 364                 ssh "$REMOTE" cat "$VPATH" > "$A"
 365         fi
 366
 367         split_remote "$REM/peer_mw$IDX"
 368         if [[ "$REMOTE" == "" ]]; then
 369                 B=$VPATH
 370         else
 371                 B=/tmp/ntb_test.$$.B
 372                 ssh "$REMOTE" cat "$VPATH" > "$B"
 373         fi
 374
 375         cmp -n $MW_ALIGNED_SIZE "$A" "$B"
 376         if [[ $? != 0 ]]; then
 377                 echo "Memory window $MW did not match!" >&2
 378         fi
 379
 380         if [[ "$A" == "/tmp/*" ]]; then
 381                 rm "$A"
 382         fi
 383
 384         if [[ "$B" == "/tmp/*" ]]; then
 385                 rm "$B"
 386         fi
 387 }
 388
 389 function mw_free()
 390 {
 391         IDX=$1
 392         LOC=$2
 393         REM=$3
 394
 395         write_file "$MW_DMA_ADDR:0" "$REM/peer_mw_trans$IDX"
 396
 397         write_file 0 "$LOC/mw_trans$IDX"
 398 }
 399
 400 function mw_test()
 401 {
 402         LOC=$1
 403         REM=$2
 404
 405         CNT=$(get_files_count "mw_trans" "$LOC")
 406
 407         for ((i = 0; i < $CNT; i++)); do
 408                 echo "Running mw$i tests on: $(subdirname $LOC) / " \
 409                      "$(subdirname $REM)"
 410
 411                 mw_alloc $i $LOC $REM
 412
 413                 mw_check $i $LOC $REM
 414
 415                 mw_free $i $LOC  $REM
 416
 417                 echo "  Passed"
 418         done
 419
 420 }
 421
 422 function pingpong_test()
 423 {
 424         LOC=$1
 425         REM=$2
 426
 427         echo "Running ping pong tests on: $(basename $LOC) / $(basename $REM)"
 428
 429         LOC_START=$(read_file "$LOC/count")
 430         REM_START=$(read_file "$REM/count")
 431
 432         sleep 7
 433
 434         LOC_END=$(read_file "$LOC/count")
 435         REM_END=$(read_file "$REM/count")
 436
 437         if [[ $LOC_START == $LOC_END ]] || [[ $REM_START == $REM_END ]]; then
 438                 echo "Ping pong counter not incrementing!" >&2
 439                 exit 1
 440         fi
 441
 442         echo "  Passed"
 443 }
 444
 445 function msi_test()
 446 {
 447         LOC=$1
 448         REM=$2
 449
 450         write_file 1 $LOC/ready
 451
 452         echo "Running MSI interrupt tests on: $(subdirname $LOC) / $(subdirname $REM)"
 453
 454         CNT=$(read_file "$LOC/count")
 455         for ((i = 0; i < $CNT; i++)); do
 456                 START=$(read_file $REM/../irq${i}_occurrences)
 457                 write_file $i $LOC/trigger
 458                 END=$(read_file $REM/../irq${i}_occurrences)
 459
 460                 if [[ $(($END - $START)) != 1 ]]; then
 461                         echo "MSI did not trigger the interrupt on the remote side!" >&2
 462                         exit 1
 463                 fi
 464         done
 465
 466         echo "  Passed"
 467 }
 468
 469 function perf_test()
 470 {
 471         USE_DMA=$1
 472
 473         if [[ $USE_DMA == "1" ]]; then
 474                 WITH="with"
 475         else
 476                 WITH="without"
 477         fi
 478
 479         _modprobe ntb_perf total_order=$PERF_RUN_ORDER \
 480                 max_mw_size=$MAX_MW_SIZE use_dma=$USE_DMA
 481
 482         echo "Running local perf test $WITH DMA"
 483         write_file "$LOCAL_PIDX" "$LOCAL_PERF/run"
 484         echo -n "  "
 485         read_file "$LOCAL_PERF/run"
 486         echo "  Passed"
 487
 488         echo "Running remote perf test $WITH DMA"
 489         write_file "$REMOTE_PIDX" "$REMOTE_PERF/run"
 490         echo -n "  "
 491         read_file "$REMOTE_PERF/run"
 492         echo "  Passed"
 493
 494         _modprobe -r ntb_perf
 495 }
 496
 497 function ntb_tool_tests()
 498 {
 499         LOCAL_TOOL="$DEBUGFS/ntb_tool/$LOCAL_DEV"
 500         REMOTE_TOOL="$REMOTE_HOST:$DEBUGFS/ntb_tool/$REMOTE_DEV"
 501
 502         echo "Starting ntb_tool tests..."
 503
 504         _modprobe ntb_tool
 505
 506         port_test "$LOCAL_TOOL" "$REMOTE_TOOL"
 507
 508         LOCAL_PEER_TOOL="$LOCAL_TOOL/peer$LOCAL_PIDX"
 509         REMOTE_PEER_TOOL="$REMOTE_TOOL/peer$REMOTE_PIDX"
 510
 511         link_test "$LOCAL_PEER_TOOL" "$REMOTE_PEER_TOOL"
 512         link_test "$REMOTE_PEER_TOOL" "$LOCAL_PEER_TOOL"
 513
 514         #Ensure the link is up on both sides before continuing
 515         write_file "Y" "$LOCAL_PEER_TOOL/link_event"
 516         write_file "Y" "$REMOTE_PEER_TOOL/link_event"
 517
 518         doorbell_test "$LOCAL_TOOL" "$REMOTE_TOOL"
 519         doorbell_test "$REMOTE_TOOL" "$LOCAL_TOOL"
 520
 521         scratchpad_test "$LOCAL_PEER_TOOL" "$REMOTE_PEER_TOOL"
 522         scratchpad_test "$REMOTE_PEER_TOOL" "$LOCAL_PEER_TOOL"
 523
 524         message_test "$LOCAL_PEER_TOOL" "$REMOTE_PEER_TOOL"
 525         message_test "$REMOTE_PEER_TOOL" "$LOCAL_PEER_TOOL"
 526
 527         mw_test "$LOCAL_PEER_TOOL" "$REMOTE_PEER_TOOL"
 528         mw_test "$REMOTE_PEER_TOOL" "$LOCAL_PEER_TOOL"
 529
 530         _modprobe -r ntb_tool
 531 }
 532
 533 function ntb_pingpong_tests()
 534 {
 535         LOCAL_PP="$DEBUGFS/ntb_pingpong/$LOCAL_DEV"
 536         REMOTE_PP="$REMOTE_HOST:$DEBUGFS/ntb_pingpong/$REMOTE_DEV"
 537
 538         echo "Starting ntb_pingpong tests..."
 539
 540         _modprobe ntb_pingpong
 541
 542         pingpong_test $LOCAL_PP $REMOTE_PP
 543
 544         _modprobe -r ntb_pingpong
 545 }
 546
 547 function ntb_msi_tests()
 548 {
 549         LOCAL_MSI="$DEBUGFS/ntb_msi_test/$LOCAL_DEV"
 550         REMOTE_MSI="$REMOTE_HOST:$DEBUGFS/ntb_msi_test/$REMOTE_DEV"
 551
 552         echo "Starting ntb_msi_test tests..."
 553
 554         if ! _modprobe ntb_msi_test 2> /dev/null; then
 555                 echo "  Not doing MSI tests seeing the module is not available."
 556                 return
 557         fi
 558
 559         port_test $LOCAL_MSI $REMOTE_MSI
 560
 561         LOCAL_PEER="$LOCAL_MSI/peer$LOCAL_PIDX"
 562         REMOTE_PEER="$REMOTE_MSI/peer$REMOTE_PIDX"
 563
 564         msi_test $LOCAL_PEER $REMOTE_PEER
 565         msi_test $REMOTE_PEER $LOCAL_PEER
 566
 567         _modprobe -r ntb_msi_test
 568 }
 569
 570 function ntb_perf_tests()
 571 {
 572         LOCAL_PERF="$DEBUGFS/ntb_perf/$LOCAL_DEV"
 573         REMOTE_PERF="$REMOTE_HOST:$DEBUGFS/ntb_perf/$REMOTE_DEV"
 574
 575         echo "Starting ntb_perf tests..."
 576
 577         perf_test 0
 578
 579         if [[ $RUN_DMA_TESTS ]]; then
 580                 perf_test 1
 581         fi
 582 }
 583
 584 function cleanup()
 585 {
 586         set +e
 587         _modprobe -r ntb_tool 2> /dev/null
 588         _modprobe -r ntb_perf 2> /dev/null
 589         _modprobe -r ntb_pingpong 2> /dev/null
 590         _modprobe -r ntb_transport 2> /dev/null
 591         _modprobe -r ntb_msi_test 2> /dev/null
 592         set -e
 593 }
 594
 595 cleanup
 596
 597 if ! [[ $$DONT_CLEANUP ]]; then
 598         trap cleanup EXIT
 599 fi
 600
 601 if [ "$(id -u)" != "0" ]; then
 602         echo "This script must be run as root" 1>&2
 603         exit 1
 604 fi
 605
 606 if [[ "$LIST_DEVS" == TRUE ]]; then
 607         echo "Local Devices:"
 608         ls -1 /sys/bus/ntb/devices
 609         echo
 610
 611         if [[ "$REMOTE_HOST" != "" ]]; then
 612                 echo "Remote Devices:"
 613                 ssh $REMOTE_HOST ls -1 /sys/bus/ntb/devices
 614         fi
 615
 616         exit 0
 617 fi
 618
 619 if [[ "$LOCAL_DEV" == $"" ]] || [[ "$REMOTE_DEV" == $"" ]]; then
 620         show_help
 621         exit 1
 622 fi
 623
 624 ntb_tool_tests
 625 echo
 626 ntb_pingpong_tests
 627 echo
 628 ntb_msi_tests
 629 echo
 630 ntb_perf_tests
 631 echo