dev-support/hbase_nightly_pseudo-distributed-test.sh

   1 #!/usr/bin/env bash
   2 # Licensed to the Apache Software Foundation (ASF) under one
   3 # or more contributor license agreements.  See the NOTICE file
   4 # distributed with this work for additional information
   5 # regarding copyright ownership.  The ASF licenses this file
   6 # to you under the Apache License, Version 2.0 (the
   7 # "License"); you may not use this file except in compliance
   8 # with the License.  You may obtain a copy of the License at
   9 #
  10 #   http://www.apache.org/licenses/LICENSE-2.0
  11 #
  12 # Unless required by applicable law or agreed to in writing,
  13 # software distributed under the License is distributed on an
  14 # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  15 # KIND, either express or implied.  See the License for the
  16 # specific language governing permissions and limitations
  17 # under the License.
  18
  19 set -e
  20 function usage {
  21   echo "Usage: ${0} [options] /path/to/component/bin-install /path/to/hadoop/executable /path/to/hadoop/hadoop-yarn-server-tests-tests.jar /path/to/hadoop/hadoop-mapreduce-client-jobclient-tests.jar"
  22   echo ""
  23   echo "    --zookeeper-data /path/to/use                                     Where the embedded zookeeper instance should write its data."
  24   echo "                                                                      defaults to 'zk-data' in the working-dir."
  25   echo "    --working-dir /path/to/use                                        Path for writing configs and logs. must exist."
  26   echo "                                                                      defaults to making a directory via mktemp."
  27   echo "    --hadoop-client-classpath /path/to/some.jar:/path/to/another.jar  classpath for hadoop jars."
  28   echo "                                                                      defaults to 'hadoop classpath'"
  29   echo "    --hbase-client-install /path/to/unpacked/client/tarball           if given we'll look here for hbase client jars instead of the bin-install"
  30   echo "    --force-data-clean                                                Delete all data in HDFS and ZK prior to starting up hbase"
  31   echo "    --single-process                                                  Run as single process instead of pseudo-distributed"
  32   echo ""
  33   exit 1
  34 }
  35 # if no args specified, show usage
  36 if [ $# -lt 4 ]; then
  37   usage
  38 fi
  39
  40 # Get arguments
  41 declare component_install
  42 declare hadoop_exec
  43 declare working_dir
  44 declare zk_data_dir
  45 declare clean
  46 declare distributed="true"
  47 declare hadoop_jars
  48 declare hbase_client
  49 while [ $# -gt 0 ]
  50 do
  51   case "$1" in
  52     --working-dir) shift; working_dir=$1; shift;;
  53     --force-data-clean) shift; clean="true";;
  54     --zookeeper-data) shift; zk_data_dir=$1; shift;;
  55     --single-process) shift; distributed="false";;
  56     --hadoop-client-classpath) shift; hadoop_jars="$1"; shift;;
  57     --hbase-client-install) shift; hbase_client="$1"; shift;;
  58     --) shift; break;;
  59     -*) usage ;;
  60     *)  break;;  # terminate while loop
  61   esac
  62 done
  63
  64 # should still have where component checkout is.
  65 if [ $# -lt 4 ]; then
  66   usage
  67 fi
  68 component_install="$(cd "$(dirname "$1")"; pwd)/$(basename "$1")"
  69 hadoop_exec="$(cd "$(dirname "$2")"; pwd)/$(basename "$2")"
  70 yarn_server_tests_test_jar="$(cd "$(dirname "$3")"; pwd)/$(basename "$3")"
  71 mapred_jobclient_test_jar="$(cd "$(dirname "$4")"; pwd)/$(basename "$4")"
  72
  73 if [ ! -x "${hadoop_exec}" ]; then
  74   echo "hadoop cli does not appear to be executable." >&2
  75   exit 1
  76 fi
  77
  78 if [ ! -d "${component_install}" ]; then
  79   echo "Path to HBase binary install should be a directory." >&2
  80   exit 1
  81 fi
  82
  83 if [ ! -f "${yarn_server_tests_test_jar}" ]; then
  84   echo "Specified YARN server tests test jar is not a file." >&2
  85   exit 1
  86 fi
  87
  88 if [ ! -f "${mapred_jobclient_test_jar}" ]; then
  89   echo "Specified MapReduce jobclient test jar is not a file." >&2
  90   exit 1
  91 fi
  92
  93 if [ -z "${working_dir}" ]; then
  94   if ! working_dir="$(mktemp -d -t hbase-pseudo-dist-test)" ; then
  95     echo "Failed to create temporary working directory. Please specify via --working-dir" >&2
  96     exit 1
  97   fi
  98 else
  99   # absolutes please
 100   working_dir="$(cd "$(dirname "${working_dir}")"; pwd)/$(basename "${working_dir}")"
 101   if [ ! -d "${working_dir}" ]; then
 102     echo "passed working directory '${working_dir}' must already exist." >&2
 103     exit 1
 104   fi
 105 fi
 106
 107 if [ -z "${zk_data_dir}" ]; then
 108   zk_data_dir="${working_dir}/zk-data"
 109   mkdir "${zk_data_dir}"
 110 else
 111   # absolutes please
 112   zk_data_dir="$(cd "$(dirname "${zk_data_dir}")"; pwd)/$(basename "${zk_data_dir}")"
 113   if [ ! -d "${zk_data_dir}" ]; then
 114     echo "passed directory for unpacking the source tarball '${zk_data_dir}' must already exist."
 115     exit 1
 116   fi
 117 fi
 118
 119 if [ -z "${hbase_client}" ]; then
 120   hbase_client="${component_install}"
 121 else
 122   echo "Using HBase client-side artifact"
 123   # absolutes please
 124   hbase_client="$(cd "$(dirname "${hbase_client}")"; pwd)/$(basename "${hbase_client}")"
 125   if [ ! -d "${hbase_client}" ]; then
 126     echo "If given hbase client install should be a directory with contents of the client tarball." >&2
 127     exit 1
 128   fi
 129 fi
 130
 131 if [ -n "${hadoop_jars}" ]; then
 132   declare -a tmp_jars
 133   for entry in $(echo "${hadoop_jars}" | tr ':' '\n'); do
 134     tmp_jars=("${tmp_jars[@]}" "$(cd "$(dirname "${entry}")"; pwd)/$(basename "${entry}")")
 135   done
 136   hadoop_jars="$(IFS=:; echo "${tmp_jars[*]}")"
 137 fi
 138
 139
 140 echo "You'll find logs and temp files in ${working_dir}"
 141
 142 function redirect_and_run {
 143   log_base=$1
 144   shift
 145   echo "$*" >"${log_base}.err"
 146   "$@" >"${log_base}.out" 2>>"${log_base}.err"
 147 }
 148
 149 (cd "${working_dir}"
 150
 151 echo "Hadoop version information:"
 152 "${hadoop_exec}" version
 153 hadoop_version=$("${hadoop_exec}" version | head -n 1)
 154 hadoop_version="${hadoop_version#Hadoop }"
 155 if [ "${hadoop_version%.*.*}" -gt 2 ]; then
 156   "${hadoop_exec}" envvars
 157 else
 158   echo "JAVA_HOME: ${JAVA_HOME}"
 159 fi
 160
 161 # Ensure that if some other Hadoop install happens to be present in the environment we ignore it.
 162 HBASE_DISABLE_HADOOP_CLASSPATH_LOOKUP="true"
 163 export HBASE_DISABLE_HADOOP_CLASSPATH_LOOKUP
 164
 165 if [ -n "${clean}" ]; then
 166   echo "Cleaning out ZooKeeper..."
 167   rm -rf "${zk_data_dir:?}/*"
 168 fi
 169
 170 echo "HBase version information:"
 171 "${component_install}/bin/hbase" version 2>/dev/null
 172 hbase_version=$("${component_install}/bin/hbase" version | head -n 1 2>/dev/null)
 173 hbase_version="${hbase_version#HBase }"
 174
 175 if [ ! -s "${hbase_client}/lib/shaded-clients/hbase-shaded-mapreduce-${hbase_version}.jar" ]; then
 176   echo "HBase binary install doesn't appear to include a shaded mapreduce artifact." >&2
 177   exit 1
 178 fi
 179
 180 if [ ! -s "${hbase_client}/lib/shaded-clients/hbase-shaded-client-${hbase_version}.jar" ]; then
 181   echo "HBase binary install doesn't appear to include a shaded client artifact." >&2
 182   exit 1
 183 fi
 184
 185 if [ ! -s "${hbase_client}/lib/shaded-clients/hbase-shaded-client-byo-hadoop-${hbase_version}.jar" ]; then
 186   echo "HBase binary install doesn't appear to include a shaded client artifact." >&2
 187   exit 1
 188 fi
 189
 190 echo "Writing out configuration for HBase."
 191 rm -rf "${working_dir}/hbase-conf"
 192 mkdir "${working_dir}/hbase-conf"
 193
 194 if [ -f "${component_install}/conf/log4j.properties" ]; then
 195   cp "${component_install}/conf/log4j.properties" "${working_dir}/hbase-conf/log4j.properties"
 196 else
 197   cat >"${working_dir}/hbase-conf/log4j.properties" <<EOF
 198 # Define some default values that can be overridden by system properties
 199 hbase.root.logger=INFO,console
 200
 201 # Define the root logger to the system property "hbase.root.logger".
 202 log4j.rootLogger=${hbase.root.logger}
 203
 204 # Logging Threshold
 205 log4j.threshold=ALL
 206 # console
 207 log4j.appender.console=org.apache.log4j.ConsoleAppender
 208 log4j.appender.console.target=System.err
 209 log4j.appender.console.layout=org.apache.log4j.PatternLayout
 210 log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2}: %.1000m%n
 211 EOF
 212 fi
 213
 214 cat >"${working_dir}/hbase-conf/hbase-site.xml" <<EOF
 215 <?xml version="1.0"?>
 216 <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
 217 <!--
 218 /**
 219  *
 220  * Licensed to the Apache Software Foundation (ASF) under one
 221  * or more contributor license agreements.  See the NOTICE file
 222  * distributed with this work for additional information
 223  * regarding copyright ownership.  The ASF licenses this file
 224  * to you under the Apache License, Version 2.0 (the
 225  * "License"); you may not use this file except in compliance
 226  * with the License.  You may obtain a copy of the License at
 227  *
 228  *     http://www.apache.org/licenses/LICENSE-2.0
 229  *
 230  * Unless required by applicable law or agreed to in writing, software
 231  * distributed under the License is distributed on an "AS IS" BASIS,
 232  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 233  * See the License for the specific language governing permissions and
 234  * limitations under the License.
 235  */
 236 -->
 237 <configuration>
 238   <property>
 239     <name>hbase.rootdir</name>
 240     <!-- We rely on the defaultFS being set in our hadoop confs -->
 241     <value>/hbase</value>
 242   </property>
 243   <property>
 244     <name>hbase.zookeeper.property.dataDir</name>
 245     <value>${zk_data_dir}</value>
 246   </property>
 247   <property>
 248     <name>hbase.cluster.distributed</name>
 249     <value>${distributed}</value>
 250   </property>
 251 </configuration>
 252 EOF
 253
 254 if [ "true" = "${distributed}" ]; then
 255   cat >"${working_dir}/hbase-conf/regionservers" <<EOF
 256 localhost
 257 EOF
 258 fi
 259
 260 function cleanup {
 261
 262   echo "Shutting down HBase"
 263   HBASE_CONF_DIR="${working_dir}/hbase-conf/" "${component_install}/bin/stop-hbase.sh"
 264
 265   if [ -f "${working_dir}/hadoop.pid" ]; then
 266     echo "Shutdown: listing HDFS contents"
 267     redirect_and_run "${working_dir}/hadoop_listing_at_end" \
 268     "${hadoop_exec}" --config "${working_dir}/hbase-conf/" fs -ls -R /
 269
 270     echo "Shutting down Hadoop"
 271     kill -6 "$(cat "${working_dir}/hadoop.pid")"
 272   fi
 273 }
 274
 275 trap cleanup EXIT SIGQUIT
 276
 277 echo "Starting up Hadoop"
 278
 279 HADOOP_CLASSPATH="${yarn_server_tests_test_jar}" "${hadoop_exec}" jar "${mapred_jobclient_test_jar}" minicluster -writeConfig "${working_dir}/hbase-conf/core-site.xml" -writeDetails "${working_dir}/hadoop_cluster_info.json" >"${working_dir}/hadoop_cluster_command.out" 2>"${working_dir}/hadoop_cluster_command.err" &
 280 echo "$!" > "${working_dir}/hadoop.pid"
 281
 282 sleep_time=2
 283 until [ -s "${working_dir}/hbase-conf/core-site.xml" ]; do
 284   printf '\twaiting for Hadoop to finish starting up.\n'
 285   sleep "${sleep_time}"
 286   sleep_time="$((sleep_time*2))"
 287 done
 288
 289 if [ "${hadoop_version%.*.*}" -gt 2 ]; then
 290   echo "Verifying configs"
 291   "${hadoop_exec}" --config "${working_dir}/hbase-conf/" conftest
 292 fi
 293
 294 if [ -n "${clean}" ]; then
 295   echo "Cleaning out HDFS..."
 296   "${hadoop_exec}" --config "${working_dir}/hbase-conf/" fs -rm -r /hbase
 297   "${hadoop_exec}" --config "${working_dir}/hbase-conf/" fs -rm -r example/
 298   "${hadoop_exec}" --config "${working_dir}/hbase-conf/" fs -rm -r example-region-listing.data
 299 fi
 300
 301 echo "Listing HDFS contents"
 302 redirect_and_run "${working_dir}/hadoop_cluster_smoke" \
 303     "${hadoop_exec}" --config "${working_dir}/hbase-conf/" fs -ls -R /
 304
 305 echo "Starting up HBase"
 306 HBASE_CONF_DIR="${working_dir}/hbase-conf/" "${component_install}/bin/start-hbase.sh"
 307
 308 sleep_time=2
 309 until "${component_install}/bin/hbase" --config "${working_dir}/hbase-conf/" shell --noninteractive >"${working_dir}/waiting_hbase_startup.log" 2>&1 <<EOF
 310   count 'hbase:meta'
 311 EOF
 312 do
 313   printf '\tretry waiting for hbase to come up.\n'
 314   sleep "${sleep_time}"
 315   sleep_time="$((sleep_time*2))"
 316 done
 317
 318 echo "Setting up table 'test:example' with 1,000 regions"
 319 "${hbase_client}/bin/hbase" --config "${working_dir}/hbase-conf/" shell --noninteractive >"${working_dir}/table_create.log" 2>&1 <<EOF
 320   create_namespace 'test'
 321   create 'test:example', 'family1', 'family2', {NUMREGIONS => 1000, SPLITALGO => 'UniformSplit'}
 322 EOF
 323
 324 echo "writing out example TSV to example.tsv"
 325 cat >"${working_dir}/example.tsv" <<EOF
 326 row1    value8  value8
 327 row3                    value2
 328 row2    value9
 329 row10           value1
 330 pow1    value8          value8
 331 pow3            value2
 332 pow2                    value9
 333 pow10   value1
 334 paw1            value8  value8
 335 paw3    value2
 336 paw2            value9
 337 paw10                   value1
 338 raw1    value8  value8
 339 raw3                    value2
 340 raw2    value9
 341 raw10           value1
 342 aow1    value8          value8
 343 aow3            value2
 344 aow2                    value9
 345 aow10   value1
 346 aaw1            value8  value8
 347 aaw3    value2
 348 aaw2            value9
 349 aaw10                   value1
 350 how1    value8  value8
 351 how3                    value2
 352 how2    value9
 353 how10           value1
 354 zow1    value8          value8
 355 zow3            value2
 356 zow2                    value9
 357 zow10   value1
 358 zaw1            value8  value8
 359 zaw3    value2
 360 zaw2            value9
 361 zaw10                   value1
 362 haw1    value8  value8
 363 haw3                    value2
 364 haw2    value9
 365 haw10           value1
 366 low1    value8          value8
 367 low3            value2
 368 low2                    value9
 369 low10   value1
 370 law1            value8  value8
 371 law3    value2
 372 law2            value9
 373 law10                   value1
 374 EOF
 375
 376 echo "uploading example.tsv to HDFS"
 377 "${hadoop_exec}" --config "${working_dir}/hbase-conf/" fs -mkdir example
 378 "${hadoop_exec}" --config "${working_dir}/hbase-conf/" fs -copyFromLocal "${working_dir}/example.tsv" "example/"
 379
 380 echo "Importing TSV via shaded client artifact for HBase - MapReduce integration."
 381 # hbase_thirdparty_jars=("${component_install}"/lib/htrace-core4*.jar \
 382 #     "${component_install}"/lib/slf4j-api-*.jar \
 383 #     "${component_install}"/lib/commons-logging-*.jar \
 384 #     "${component_install}"/lib/slf4j-log4j12-*.jar \
 385 #     "${component_install}"/lib/log4j-1.2.*.jar \
 386 #     "${working_dir}/hbase-conf/log4j.properties")
 387 # hbase_dep_classpath=$(IFS=:; echo "${hbase_thirdparty_jars[*]}")
 388 hbase_dep_classpath="$("${hbase_client}/bin/hbase" --config "${working_dir}/hbase-conf/" mapredcp)"
 389 HADOOP_CLASSPATH="${hbase_dep_classpath}" redirect_and_run "${working_dir}/mr-importtsv" \
 390     "${hadoop_exec}" --config "${working_dir}/hbase-conf/" jar "${hbase_client}/lib/shaded-clients/hbase-shaded-mapreduce-${hbase_version}.jar" importtsv -Dimporttsv.columns=HBASE_ROW_KEY,family1:column1,family1:column4,family1:column3 test:example example/ -libjars "${hbase_dep_classpath}"
 391 "${hbase_client}/bin/hbase" --config "${working_dir}/hbase-conf/" shell --noninteractive >"${working_dir}/scan_import.out" 2>"${working_dir}/scan_import.err" <<EOF
 392   scan 'test:example'
 393 EOF
 394
 395 echo "Verifying row count from import."
 396 import_rowcount=$(echo 'count "test:example"' | "${hbase_client}/bin/hbase" --config "${working_dir}/hbase-conf/" shell --noninteractive 2>/dev/null | tail -n 1)
 397 if [ ! "${import_rowcount}" -eq 48 ]; then
 398   echo "ERROR: Instead of finding 48 rows, we found ${import_rowcount}."
 399   exit 2
 400 fi
 401
 402 if [ -z "${hadoop_jars}" ]; then
 403   echo "Hadoop client jars not given; getting them from 'hadoop classpath' for the example."
 404   hadoop_jars=$("${hadoop_exec}" --config "${working_dir}/hbase-conf/" classpath)
 405 fi
 406
 407 echo "Building shaded client example."
 408 cat >"${working_dir}/HBaseClientReadWriteExample.java" <<EOF
 409 import org.apache.hadoop.conf.Configuration;
 410 import org.apache.hadoop.fs.FileSystem;
 411 import org.apache.hadoop.fs.FSDataInputStream;
 412 import org.apache.hadoop.fs.FSDataOutputStream;
 413 import org.apache.hadoop.fs.Path;
 414 import org.apache.hadoop.hbase.Cell;
 415 import org.apache.hadoop.hbase.CellBuilder;
 416 import org.apache.hadoop.hbase.CellBuilderFactory;
 417 import org.apache.hadoop.hbase.CellBuilderType;
 418 import org.apache.hadoop.hbase.ClusterMetrics;
 419 import org.apache.hadoop.hbase.HBaseConfiguration;
 420 import org.apache.hadoop.hbase.RegionMetrics;
 421 import org.apache.hadoop.hbase.ServerMetrics;
 422 import org.apache.hadoop.hbase.TableName;
 423 import org.apache.hadoop.hbase.client.Admin;
 424 import org.apache.hadoop.hbase.client.Connection;
 425 import org.apache.hadoop.hbase.client.ConnectionFactory;
 426 import org.apache.hadoop.hbase.client.Put;
 427 import org.apache.hadoop.hbase.client.Table;
 428 import org.apache.hadoop.hbase.util.Bytes;
 429
 430 import java.util.LinkedList;
 431 import java.util.List;
 432
 433
 434 public class HBaseClientReadWriteExample {
 435   private static final byte[] FAMILY_BYTES = Bytes.toBytes("family2");
 436
 437   public static void main(String[] args) throws Exception {
 438     Configuration hbase = HBaseConfiguration.create();
 439     Configuration hadoop = new Configuration();
 440     try (Connection connection = ConnectionFactory.createConnection(hbase)) {
 441       System.out.println("Generating list of regions");
 442       final List<String> regions = new LinkedList<>();
 443       try (Admin admin = connection.getAdmin()) {
 444         final ClusterMetrics cluster = admin.getClusterMetrics();
 445         System.out.println(String.format("\tCluster reports version %s, ave load %f, region count %d", cluster.getHBaseVersion(), cluster.getAverageLoad(), cluster.getRegionCount()));
 446         for (ServerMetrics server : cluster.getLiveServerMetrics().values()) {
 447           for (RegionMetrics region : server.getRegionMetrics().values()) {
 448             regions.add(region.getNameAsString());
 449           }
 450         }
 451       }
 452       final Path listing = new Path("example-region-listing.data");
 453       System.out.println("Writing list to HDFS");
 454       try (FileSystem fs = FileSystem.newInstance(hadoop)) {
 455         final Path path = fs.makeQualified(listing);
 456         try (FSDataOutputStream out = fs.create(path)) {
 457           out.writeInt(regions.size());
 458           for (String region : regions) {
 459             out.writeUTF(region);
 460           }
 461           out.hsync();
 462         }
 463       }
 464       final List<Put> puts = new LinkedList<>();
 465       final Put marker = new Put(new byte[] { (byte)0 });
 466       System.out.println("Reading list from HDFS");
 467       try (FileSystem fs = FileSystem.newInstance(hadoop)) {
 468         final Path path = fs.makeQualified(listing);
 469         final CellBuilder builder = CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY);
 470         try (FSDataInputStream in = fs.open(path)) {
 471           final int count = in.readInt();
 472           marker.addColumn(FAMILY_BYTES, Bytes.toBytes("count"), Bytes.toBytes(count));
 473           for(int i = 0; i < count; i++) {
 474             builder.clear();
 475             final byte[] row = Bytes.toBytes(in.readUTF());
 476             final Put put = new Put(row);
 477             builder.setRow(row);
 478             builder.setFamily(FAMILY_BYTES);
 479             builder.setType(Cell.Type.Put);
 480             put.add(builder.build());
 481             puts.add(put);
 482           }
 483         }
 484       }
 485       System.out.println("Writing list into HBase table");
 486       try (Table table = connection.getTable(TableName.valueOf("test:example"))) {
 487         table.put(marker);
 488         table.put(puts);
 489       }
 490     }
 491   }
 492 }
 493 EOF
 494 redirect_and_run "${working_dir}/hbase-shaded-client-compile" \
 495     javac -cp "${hbase_client}/lib/shaded-clients/hbase-shaded-client-byo-hadoop-${hbase_version}.jar:${hadoop_jars}" "${working_dir}/HBaseClientReadWriteExample.java"
 496 echo "Running shaded client example. It'll fetch the set of regions, round-trip them to a file in HDFS, then write them one-per-row into the test table."
 497 # The order of classpath entries here is important. if we're using non-shaded Hadoop 3 / 2.9.0 jars, we have to work around YARN-2190.
 498 redirect_and_run "${working_dir}/hbase-shaded-client-example" \
 499     java -cp "${working_dir}/hbase-conf/:${hbase_client}/lib/shaded-clients/hbase-shaded-client-byo-hadoop-${hbase_version}.jar:${hbase_dep_classpath}:${working_dir}:${hadoop_jars}" HBaseClientReadWriteExample
 500
 501 echo "Checking on results of example program."
 502 "${hadoop_exec}" --config "${working_dir}/hbase-conf/" fs -copyToLocal "example-region-listing.data" "${working_dir}/example-region-listing.data"
 503
 504 "${hbase_client}/bin/hbase" --config "${working_dir}/hbase-conf/" shell --noninteractive >"${working_dir}/scan_example.out" 2>"${working_dir}/scan_example.err" <<EOF
 505   scan 'test:example'
 506 EOF
 507
 508 echo "Verifying row count from example."
 509 example_rowcount=$(echo 'count "test:example"' | "${hbase_client}/bin/hbase" --config "${working_dir}/hbase-conf/" shell --noninteractive 2>/dev/null | tail -n 1)
 510 if [ "${example_rowcount}" -gt "1050" ]; then
 511   echo "Found ${example_rowcount} rows, which is enough to cover 48 for import, 1000 example's use of user table regions, 2 for example's use of meta/root regions, and 1 for example's count record"
 512 else
 513   echo "ERROR: Only found ${example_rowcount} rows."
 514 fi
 515
 516 )