hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestCase.java

   1 /**
   2  *
   3  * Licensed to the Apache Software Foundation (ASF) under one
   4  * or more contributor license agreements.  See the NOTICE file
   5  * distributed with this work for additional information
   6  * regarding copyright ownership.  The ASF licenses this file
   7  * to you under the Apache License, Version 2.0 (the
   8  * "License"); you may not use this file except in compliance
   9  * with the License.  You may obtain a copy of the License at
  10  *
  11  *     http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  */
  19 package org.apache.hadoop.hbase;
  20
  21 import java.io.IOException;
  22 import java.nio.charset.StandardCharsets;
  23 import java.util.NavigableMap;
  24
  25 import org.apache.hadoop.conf.Configuration;
  26 import org.apache.hadoop.fs.FileSystem;
  27 import org.apache.hadoop.fs.Path;
  28 import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
  29 import org.apache.hadoop.hbase.client.Durability;
  30 import org.apache.hadoop.hbase.client.Get;
  31 import org.apache.hadoop.hbase.client.Put;
  32 import org.apache.hadoop.hbase.client.Result;
  33 import org.apache.hadoop.hbase.client.Table;
  34 import org.apache.hadoop.hbase.client.TableDescriptor;
  35 import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
  36 import org.apache.hadoop.hbase.log.HBaseMarkers;
  37 import org.apache.hadoop.hbase.regionserver.HRegion;
  38 import org.apache.hadoop.hbase.regionserver.Region;
  39 import org.apache.hadoop.hbase.regionserver.RegionAsTable;
  40 import org.apache.hadoop.hbase.util.Bytes;
  41 import org.apache.hadoop.hbase.util.FSTableDescriptors;
  42 import org.apache.hadoop.hbase.util.FSUtils;
  43 import org.apache.hadoop.hdfs.MiniDFSCluster;
  44 import org.slf4j.Logger;
  45 import org.slf4j.LoggerFactory;
  46
  47 import junit.framework.AssertionFailedError;
  48 import junit.framework.TestCase;
  49
  50 /**
  51  * Abstract HBase test class.  Initializes a few things that can come in handly
  52  * like an HBaseConfiguration and filesystem.
  53  * @deprecated since 2.0.0 and will be removed in 3.0.0. Write junit4 unit tests using
  54  *   {@link HBaseTestingUtility}.
  55  * @see HBaseTestingUtility
  56  * @see <a href="https://issues.apache.org/jira/browse/HBASE-11912">HBASE-11912</a>
  57  */
  58 @Deprecated
  59 public abstract class HBaseTestCase extends TestCase {
  60   private static final Logger LOG = LoggerFactory.getLogger(HBaseTestCase.class);
  61
  62   protected final static byte [] fam1 = Bytes.toBytes("colfamily11");
  63   protected final static byte [] fam2 = Bytes.toBytes("colfamily21");
  64   protected final static byte [] fam3 = Bytes.toBytes("colfamily31");
  65
  66   protected static final byte [][] COLUMNS = {fam1, fam2, fam3};
  67
  68   private boolean localfs = false;
  69   protected static Path testDir = null;
  70   protected FileSystem fs = null;
  71   protected HRegion meta = null;
  72   protected static final char FIRST_CHAR = 'a';
  73   protected static final char LAST_CHAR = 'z';
  74   protected static final String PUNCTUATION = "~`@#$%^&*()-_+=:;',.<>/?[]{}|";
  75   protected static final byte [] START_KEY_BYTES = {FIRST_CHAR, FIRST_CHAR, FIRST_CHAR};
  76   protected String START_KEY = new String(START_KEY_BYTES, HConstants.UTF8_CHARSET);
  77   protected static final int MAXVERSIONS = 3;
  78
  79   protected final HBaseTestingUtility testUtil = new HBaseTestingUtility();
  80
  81   public volatile Configuration conf = testUtil.getConfiguration();
  82   public final FSTableDescriptors fsTableDescriptors;
  83   {
  84     try {
  85       fsTableDescriptors = new FSTableDescriptors(conf);
  86     } catch (IOException e) {
  87       throw new RuntimeException("Failed to init descriptors", e);
  88     }
  89   }
  90
  91   /** constructor */
  92   public HBaseTestCase() {
  93     super();
  94   }
  95
  96   /**
  97    * @param name
  98    */
  99   public HBaseTestCase(String name) {
 100     super(name);
 101   }
 102
 103   /**
 104    * Note that this method must be called after the mini hdfs cluster has
 105    * started or we end up with a local file system.
 106    */
 107   @Override
 108   protected void setUp() throws Exception {
 109     super.setUp();
 110     localfs =
 111       (conf.get("fs.defaultFS", "file:///").compareTo("file:///") == 0);
 112
 113     if (fs == null) {
 114       this.fs = FileSystem.get(conf);
 115     }
 116     try {
 117       if (localfs) {
 118         testDir = getUnitTestdir(getName());
 119         if (fs.exists(testDir)) {
 120           fs.delete(testDir, true);
 121         }
 122       } else {
 123         testDir = FSUtils.getRootDir(conf);
 124       }
 125     } catch (Exception e) {
 126       LOG.error(HBaseMarkers.FATAL, "error during setup", e);
 127       throw e;
 128     }
 129   }
 130
 131   @Override
 132   protected void tearDown() throws Exception {
 133     try {
 134       if (localfs) {
 135         if (this.fs.exists(testDir)) {
 136           this.fs.delete(testDir, true);
 137         }
 138       }
 139     } catch (Exception e) {
 140       LOG.error(HBaseMarkers.FATAL, "error during tear down", e);
 141     }
 142     super.tearDown();
 143   }
 144
 145   /**
 146    * @see HBaseTestingUtility#getBaseTestDir
 147    * @param testName
 148    * @return directory to use for this test
 149    */
 150     protected Path getUnitTestdir(String testName) {
 151       return testUtil.getDataTestDir(testName);
 152     }
 153
 154   /**
 155    * You must call close on the returned region and then close on the log file it created. Do
 156    * {@link HBaseTestingUtility#closeRegionAndWAL(HRegion)} to close both the region and the WAL.
 157    * @param tableDescriptor TableDescriptor
 158    * @param startKey Start Key
 159    * @param endKey End Key
 160    * @return An {@link HRegion}
 161    * @throws IOException If thrown by
 162    *   {@link #createNewHRegion(TableDescriptor, byte[], byte[], Configuration)}
 163    */
 164   public HRegion createNewHRegion(TableDescriptor tableDescriptor, byte [] startKey,
 165       byte [] endKey)
 166   throws IOException {
 167     return createNewHRegion(tableDescriptor, startKey, endKey, this.conf);
 168   }
 169
 170   public HRegion createNewHRegion(TableDescriptor tableDescriptor, byte [] startKey,
 171       byte [] endKey, Configuration conf)
 172   throws IOException {
 173     HRegionInfo hri = new HRegionInfo(tableDescriptor.getTableName(), startKey, endKey);
 174     return HBaseTestingUtility.createRegionAndWAL(hri, testDir, conf, tableDescriptor);
 175   }
 176
 177   protected HRegion openClosedRegion(final HRegion closedRegion)
 178   throws IOException {
 179     return HRegion.openHRegion(closedRegion, null);
 180   }
 181
 182   /**
 183    * Create a table of name {@code name} with {@link #COLUMNS} for
 184    * families.
 185    * @param name Name to give table.
 186    * @return Column descriptor.
 187    */
 188   protected TableDescriptor createTableDescriptor(final String name) {
 189     return createTableDescriptor(name, MAXVERSIONS);
 190   }
 191
 192   /**
 193    * Create a table of name {@code name} with {@link #COLUMNS} for
 194    * families.
 195    * @param name Name to give table.
 196    * @param versions How many versions to allow per column.
 197    * @return Column descriptor.
 198    */
 199   protected TableDescriptor createTableDescriptor(final String name,
 200       final int versions) {
 201     return createTableDescriptor(name, HColumnDescriptor.DEFAULT_MIN_VERSIONS,
 202         versions, HConstants.FOREVER, HColumnDescriptor.DEFAULT_KEEP_DELETED);
 203   }
 204
 205   /**
 206    * Create a table of name {@code name} with {@link #COLUMNS} for
 207    * families.
 208    * @param name Name to give table.
 209    * @param versions How many versions to allow per column.
 210    * @return Column descriptor.
 211    */
 212   protected TableDescriptor createTableDescriptor(final String name,
 213       final int minVersions, final int versions, final int ttl, KeepDeletedCells keepDeleted) {
 214     TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
 215       new TableDescriptorBuilder.ModifyableTableDescriptor(TableName.valueOf(name));
 216     for (byte[] cfName : new byte[][]{ fam1, fam2, fam3 }) {
 217       tableDescriptor.setColumnFamily(
 218         new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(cfName)
 219           .setMinVersions(minVersions)
 220           .setMaxVersions(versions)
 221           .setKeepDeletedCells(keepDeleted)
 222           .setBlockCacheEnabled(false)
 223           .setTimeToLive(ttl)
 224       );
 225     }
 226     return tableDescriptor;
 227   }
 228
 229   /**
 230    * Add content to region <code>r</code> on the passed column
 231    * <code>column</code>.
 232    * Adds data of the from 'aaa', 'aab', etc where key and value are the same.
 233    * @param r
 234    * @param columnFamily
 235    * @param column
 236    * @throws IOException
 237    * @return count of what we added.
 238    */
 239   public static long addContent(final Region r, final byte [] columnFamily, final byte[] column)
 240   throws IOException {
 241     byte [] startKey = r.getRegionInfo().getStartKey();
 242     byte [] endKey = r.getRegionInfo().getEndKey();
 243     byte [] startKeyBytes = startKey;
 244     if (startKeyBytes == null || startKeyBytes.length == 0) {
 245       startKeyBytes = START_KEY_BYTES;
 246     }
 247     return addContent(new RegionAsTable(r), Bytes.toString(columnFamily), Bytes.toString(column),
 248       startKeyBytes, endKey, -1);
 249   }
 250
 251   public static long addContent(final Region r, final byte [] columnFamily) throws IOException {
 252     return addContent(r, columnFamily, null);
 253   }
 254
 255   /**
 256    * Add content to region <code>r</code> on the passed column
 257    * <code>column</code>.
 258    * Adds data of the from 'aaa', 'aab', etc where key and value are the same.
 259    * @throws IOException
 260    * @return count of what we added.
 261    */
 262   public static long addContent(final Table updater,
 263       final String columnFamily) throws IOException {
 264     return addContent(updater, columnFamily, START_KEY_BYTES, null);
 265   }
 266
 267   public static long addContent(final Table updater, final String family,
 268       final String column) throws IOException {
 269     return addContent(updater, family, column, START_KEY_BYTES, null);
 270   }
 271
 272   /**
 273    * Add content to region <code>r</code> on the passed column
 274    * <code>column</code>.
 275    * Adds data of the from 'aaa', 'aab', etc where key and value are the same.
 276    * @return count of what we added.
 277    * @throws IOException
 278    */
 279   public static long addContent(final Table updater, final String columnFamily,
 280       final byte [] startKeyBytes, final byte [] endKey)
 281   throws IOException {
 282     return addContent(updater, columnFamily, null, startKeyBytes, endKey, -1);
 283   }
 284
 285   public static long addContent(final Table updater, final String family, String column,
 286       final byte [] startKeyBytes, final byte [] endKey) throws IOException {
 287     return addContent(updater, family, column, startKeyBytes, endKey, -1);
 288   }
 289
 290   /**
 291    * Add content to region <code>r</code> on the passed column
 292    * <code>column</code>.
 293    * Adds data of the from 'aaa', 'aab', etc where key and value are the same.
 294    * @return count of what we added.
 295    * @throws IOException
 296    */
 297   public static long addContent(final Table updater,
 298                                    final String columnFamily,
 299                                    final String column,
 300       final byte [] startKeyBytes, final byte [] endKey, final long ts)
 301   throws IOException {
 302     long count = 0;
 303     // Add rows of three characters.  The first character starts with the
 304     // 'a' character and runs up to 'z'.  Per first character, we run the
 305     // second character over same range.  And same for the third so rows
 306     // (and values) look like this: 'aaa', 'aab', 'aac', etc.
 307     char secondCharStart = (char)startKeyBytes[1];
 308     char thirdCharStart = (char)startKeyBytes[2];
 309     EXIT: for (char c = (char)startKeyBytes[0]; c <= LAST_CHAR; c++) {
 310       for (char d = secondCharStart; d <= LAST_CHAR; d++) {
 311         for (char e = thirdCharStart; e <= LAST_CHAR; e++) {
 312           byte [] t = new byte [] {(byte)c, (byte)d, (byte)e};
 313           if (endKey != null && endKey.length > 0
 314               && Bytes.compareTo(endKey, t) <= 0) {
 315             break EXIT;
 316           }
 317           try {
 318             Put put;
 319             if(ts != -1) {
 320               put = new Put(t, ts);
 321             } else {
 322               put = new Put(t);
 323             }
 324             try {
 325               StringBuilder sb = new StringBuilder();
 326               if (column != null && column.contains(":")) {
 327                 sb.append(column);
 328               } else {
 329                 if (columnFamily != null) {
 330                   sb.append(columnFamily);
 331                   if (!columnFamily.endsWith(":")) {
 332                     sb.append(":");
 333                   }
 334                   if (column != null) {
 335                     sb.append(column);
 336                   }
 337                 }
 338               }
 339               byte[][] split =
 340                 CellUtil.parseColumn(Bytes.toBytes(sb.toString()));
 341               if(split.length == 1) {
 342                 byte[] qualifier = new byte[0];
 343                 put.addColumn(split[0], qualifier, t);
 344               } else {
 345                 put.addColumn(split[0], split[1], t);
 346               }
 347               put.setDurability(Durability.SKIP_WAL);
 348               updater.put(put);
 349               count++;
 350             } catch (RuntimeException ex) {
 351               ex.printStackTrace();
 352               throw ex;
 353             } catch (IOException ex) {
 354               ex.printStackTrace();
 355               throw ex;
 356             }
 357           } catch (RuntimeException ex) {
 358             ex.printStackTrace();
 359             throw ex;
 360           } catch (IOException ex) {
 361             ex.printStackTrace();
 362             throw ex;
 363           }
 364         }
 365         // Set start character back to FIRST_CHAR after we've done first loop.
 366         thirdCharStart = FIRST_CHAR;
 367       }
 368       secondCharStart = FIRST_CHAR;
 369     }
 370     return count;
 371   }
 372
 373   protected void assertResultEquals(final HRegion region, final byte [] row,
 374       final byte [] family, final byte [] qualifier, final long timestamp,
 375       final byte [] value) throws IOException {
 376     Get get = new Get(row);
 377     get.setTimestamp(timestamp);
 378     Result res = region.get(get);
 379     NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> map =
 380       res.getMap();
 381     byte [] res_value = map.get(family).get(qualifier).get(timestamp);
 382
 383     if (value == null) {
 384       assertEquals(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
 385           " at timestamp " + timestamp, null, res_value);
 386     } else {
 387       if (res_value == null) {
 388         fail(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
 389             " at timestamp " + timestamp + "\" was expected to be \"" +
 390             Bytes.toStringBinary(value) + " but was null");
 391       }
 392       if (res_value != null) {
 393         assertEquals(Bytes.toString(family) + " " + Bytes.toString(qualifier) +
 394             " at timestamp " +
 395             timestamp, value, new String(res_value, StandardCharsets.UTF_8));
 396       }
 397     }
 398   }
 399
 400   /**
 401    * Common method to close down a MiniDFSCluster and the associated file system
 402    *
 403    * @param cluster
 404    */
 405   public static void shutdownDfs(MiniDFSCluster cluster) {
 406     if (cluster != null) {
 407       LOG.info("Shutting down Mini DFS ");
 408       try {
 409         cluster.shutdown();
 410       } catch (Exception e) {
 411         /// Can get a java.lang.reflect.UndeclaredThrowableException thrown
 412         // here because of an InterruptedException. Don't let exceptions in
 413         // here be cause of test failure.
 414       }
 415       try {
 416         FileSystem fs = cluster.getFileSystem();
 417         if (fs != null) {
 418           LOG.info("Shutting down FileSystem");
 419           fs.close();
 420         }
 421         FileSystem.closeAll();
 422       } catch (IOException e) {
 423         LOG.error("error closing file system", e);
 424       }
 425     }
 426   }
 427
 428   /**
 429    * You must call {@link #closeRootAndMeta()} when done after calling this
 430    * method. It does cleanup.
 431    * @throws IOException
 432    */
 433   protected void createMetaRegion() throws IOException {
 434     FSTableDescriptors fsTableDescriptors = new FSTableDescriptors(conf);
 435     meta = HBaseTestingUtility.createRegionAndWAL(HRegionInfo.FIRST_META_REGIONINFO, testDir,
 436         conf, fsTableDescriptors.get(TableName.META_TABLE_NAME));
 437   }
 438
 439   protected void closeRootAndMeta() throws IOException {
 440     HBaseTestingUtility.closeRegionAndWAL(meta);
 441   }
 442
 443   public static void assertByteEquals(byte[] expected,
 444                                byte[] actual) {
 445     if (Bytes.compareTo(expected, actual) != 0) {
 446       throw new AssertionFailedError("expected:<" +
 447       Bytes.toString(expected) + "> but was:<" +
 448       Bytes.toString(actual) + ">");
 449     }
 450   }
 451
 452   public static void assertEquals(byte[] expected,
 453                                byte[] actual) {
 454     if (Bytes.compareTo(expected, actual) != 0) {
 455       throw new AssertionFailedError("expected:<" +
 456       Bytes.toStringBinary(expected) + "> but was:<" +
 457       Bytes.toStringBinary(actual) + ">");
 458     }
 459   }
 460 }