hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotMetadata.java

   1 /**
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase.client;
  19
  20 import static org.junit.Assert.assertEquals;
  21 import static org.junit.Assert.assertTrue;
  22
  23 import java.util.ArrayList;
  24 import java.util.Collections;
  25 import java.util.List;
  26 import org.apache.hadoop.conf.Configuration;
  27 import org.apache.hadoop.fs.FileSystem;
  28 import org.apache.hadoop.fs.Path;
  29 import org.apache.hadoop.hbase.HBaseClassTestRule;
  30 import org.apache.hadoop.hbase.HBaseTestingUtility;
  31 import org.apache.hadoop.hbase.HConstants;
  32 import org.apache.hadoop.hbase.HTableDescriptor;
  33 import org.apache.hadoop.hbase.TableName;
  34 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
  35 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
  36 import org.apache.hadoop.hbase.regionserver.BloomType;
  37 import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
  38 import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
  39 import org.apache.hadoop.hbase.testclassification.ClientTests;
  40 import org.apache.hadoop.hbase.testclassification.MediumTests;
  41 import org.apache.hadoop.hbase.util.Bytes;
  42 import org.junit.After;
  43 import org.junit.AfterClass;
  44 import org.junit.Before;
  45 import org.junit.BeforeClass;
  46 import org.junit.ClassRule;
  47 import org.junit.Test;
  48 import org.junit.experimental.categories.Category;
  49 import org.slf4j.Logger;
  50 import org.slf4j.LoggerFactory;
  51
  52 /**
  53  * Test class to verify that metadata is consistent before and after a snapshot attempt.
  54  */
  55 @Category({MediumTests.class, ClientTests.class})
  56 public class TestSnapshotMetadata {
  57
  58   @ClassRule
  59   public static final HBaseClassTestRule CLASS_RULE =
  60       HBaseClassTestRule.forClass(TestSnapshotMetadata.class);
  61
  62   private static final Logger LOG = LoggerFactory.getLogger(TestSnapshotMetadata.class);
  63
  64   private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
  65   private static final int NUM_RS = 2;
  66   private static final String STRING_TABLE_NAME = "TestSnapshotMetadata";
  67
  68   private static final String MAX_VERSIONS_FAM_STR = "fam_max_columns";
  69   private static final byte[] MAX_VERSIONS_FAM = Bytes.toBytes(MAX_VERSIONS_FAM_STR);
  70
  71   private static final String COMPRESSED_FAM_STR = "fam_compressed";
  72   private static final byte[] COMPRESSED_FAM = Bytes.toBytes(COMPRESSED_FAM_STR);
  73
  74   private static final String BLOCKSIZE_FAM_STR = "fam_blocksize";
  75   private static final byte[] BLOCKSIZE_FAM = Bytes.toBytes(BLOCKSIZE_FAM_STR);
  76
  77   private static final String BLOOMFILTER_FAM_STR = "fam_bloomfilter";
  78   private static final byte[] BLOOMFILTER_FAM = Bytes.toBytes(BLOOMFILTER_FAM_STR);
  79
  80   private static final String TEST_CONF_CUSTOM_VALUE = "TestCustomConf";
  81   private static final String TEST_CUSTOM_VALUE = "TestCustomValue";
  82
  83   private static final byte[][] families = {
  84     MAX_VERSIONS_FAM, BLOOMFILTER_FAM, COMPRESSED_FAM, BLOCKSIZE_FAM
  85   };
  86
  87   private static final DataBlockEncoding DATA_BLOCK_ENCODING_TYPE = DataBlockEncoding.FAST_DIFF;
  88   private static final BloomType BLOOM_TYPE = BloomType.ROW;
  89   private static final int BLOCK_SIZE = 98;
  90   private static final int MAX_VERSIONS = 8;
  91
  92   private Admin admin;
  93   private String originalTableDescription;
  94   private HTableDescriptor originalTableDescriptor;
  95   TableName originalTableName;
  96
  97   private static FileSystem fs;
  98   private static Path rootDir;
  99
 100   @BeforeClass
 101   public static void setupCluster() throws Exception {
 102     setupConf(UTIL.getConfiguration());
 103     UTIL.startMiniCluster(NUM_RS);
 104
 105     fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
 106     rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
 107   }
 108
 109   @AfterClass
 110   public static void cleanupTest() throws Exception {
 111     try {
 112       UTIL.shutdownMiniCluster();
 113     } catch (Exception e) {
 114       LOG.warn("failure shutting down cluster", e);
 115     }
 116   }
 117
 118   private static void setupConf(Configuration conf) {
 119     // enable snapshot support
 120     conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
 121     // disable the ui
 122     conf.setInt("hbase.regionsever.info.port", -1);
 123     // change the flush size to a small amount, regulating number of store files
 124     conf.setInt("hbase.hregion.memstore.flush.size", 25000);
 125     // so make sure we get a compaction when doing a load, but keep around
 126     // some files in the store
 127     conf.setInt("hbase.hstore.compaction.min", 10);
 128     conf.setInt("hbase.hstore.compactionThreshold", 10);
 129     // block writes if we get to 12 store files
 130     conf.setInt("hbase.hstore.blockingStoreFiles", 12);
 131     conf.setInt("hbase.regionserver.msginterval", 100);
 132     conf.setBoolean("hbase.master.enabletable.roundrobin", true);
 133     // Avoid potentially aggressive splitting which would cause snapshot to fail
 134     conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
 135       ConstantSizeRegionSplitPolicy.class.getName());
 136   }
 137
 138   @Before
 139   public void setup() throws Exception {
 140     admin = UTIL.getAdmin();
 141     createTableWithNonDefaultProperties();
 142   }
 143
 144   @After
 145   public void tearDown() throws Exception {
 146     SnapshotTestingUtils.deleteAllSnapshots(admin);
 147   }
 148
 149   /*
 150    *  Create a table that has non-default properties so we can see if they hold
 151    */
 152   private void createTableWithNonDefaultProperties() throws Exception {
 153     final long startTime = System.currentTimeMillis();
 154     final String sourceTableNameAsString = STRING_TABLE_NAME + startTime;
 155     originalTableName = TableName.valueOf(sourceTableNameAsString);
 156
 157     // enable replication on a column family
 158     ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor maxVersionsColumn =
 159       new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(MAX_VERSIONS_FAM);
 160     ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor bloomFilterColumn =
 161       new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(BLOOMFILTER_FAM);
 162     ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor dataBlockColumn =
 163       new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(COMPRESSED_FAM);
 164     ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor blockSizeColumn =
 165       new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(BLOCKSIZE_FAM);
 166
 167     maxVersionsColumn.setMaxVersions(MAX_VERSIONS);
 168     bloomFilterColumn.setBloomFilterType(BLOOM_TYPE);
 169     dataBlockColumn.setDataBlockEncoding(DATA_BLOCK_ENCODING_TYPE);
 170     blockSizeColumn.setBlocksize(BLOCK_SIZE);
 171
 172     TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
 173       new TableDescriptorBuilder.ModifyableTableDescriptor(
 174         TableName.valueOf(sourceTableNameAsString));
 175     tableDescriptor.setColumnFamily(maxVersionsColumn);
 176     tableDescriptor.setColumnFamily(bloomFilterColumn);
 177     tableDescriptor.setColumnFamily(dataBlockColumn);
 178     tableDescriptor.setColumnFamily(blockSizeColumn);
 179     tableDescriptor.setValue(TEST_CUSTOM_VALUE, TEST_CUSTOM_VALUE);
 180     tableDescriptor.setValue(TEST_CONF_CUSTOM_VALUE, TEST_CONF_CUSTOM_VALUE);
 181     assertTrue(tableDescriptor.getConfiguration().size() > 0);
 182
 183     admin.createTable(tableDescriptor);
 184     Table original = UTIL.getConnection().getTable(originalTableName);
 185     originalTableName = TableName.valueOf(sourceTableNameAsString);
 186     originalTableDescriptor = new HTableDescriptor(admin.getDescriptor(originalTableName));
 187     originalTableDescription = originalTableDescriptor.toStringCustomizedValues();
 188
 189     original.close();
 190   }
 191
 192
 193   /**
 194    * Verify that the describe for a cloned table matches the describe from the original.
 195    */
 196   @Test
 197   public void testDescribeMatchesAfterClone() throws Exception {
 198     // Clone the original table
 199     final String clonedTableNameAsString = "clone" + originalTableName;
 200     final TableName clonedTableName = TableName.valueOf(clonedTableNameAsString);
 201     final String snapshotNameAsString = "snapshot" + originalTableName
 202         + System.currentTimeMillis();
 203     final String snapshotName = snapshotNameAsString;
 204
 205     // restore the snapshot into a cloned table and examine the output
 206     List<byte[]> familiesList = new ArrayList<>();
 207     Collections.addAll(familiesList, families);
 208
 209     // Create a snapshot in which all families are empty
 210     SnapshotTestingUtils.createSnapshotAndValidate(admin, originalTableName, null,
 211       familiesList, snapshotNameAsString, rootDir, fs, /* onlineSnapshot= */ false);
 212
 213     admin.cloneSnapshot(snapshotName, clonedTableName);
 214     Table clonedTable = UTIL.getConnection().getTable(clonedTableName);
 215     HTableDescriptor cloneHtd = new HTableDescriptor(admin.getDescriptor(clonedTableName));
 216     assertEquals(
 217       originalTableDescription.replace(originalTableName.getNameAsString(),clonedTableNameAsString),
 218       cloneHtd.toStringCustomizedValues());
 219
 220     // Verify the custom fields
 221     assertEquals(originalTableDescriptor.getValues().size(),
 222                         cloneHtd.getValues().size());
 223     assertEquals(originalTableDescriptor.getConfiguration().size(),
 224                         cloneHtd.getConfiguration().size());
 225     assertEquals(TEST_CUSTOM_VALUE, cloneHtd.getValue(TEST_CUSTOM_VALUE));
 226     assertEquals(TEST_CONF_CUSTOM_VALUE, cloneHtd.getConfigurationValue(TEST_CONF_CUSTOM_VALUE));
 227     assertEquals(originalTableDescriptor.getValues(), cloneHtd.getValues());
 228     assertEquals(originalTableDescriptor.getConfiguration(), cloneHtd.getConfiguration());
 229
 230     admin.enableTable(originalTableName);
 231     clonedTable.close();
 232   }
 233
 234   /**
 235    * Verify that the describe for a restored table matches the describe for one the original.
 236    */
 237   @Test
 238   public void testDescribeMatchesAfterRestore() throws Exception {
 239     runRestoreWithAdditionalMetadata(false);
 240   }
 241
 242   /**
 243    * Verify that if metadata changed after a snapshot was taken, that the old metadata replaces the
 244    * new metadata during a restore
 245    */
 246   @Test
 247   public void testDescribeMatchesAfterMetadataChangeAndRestore() throws Exception {
 248     runRestoreWithAdditionalMetadata(true);
 249   }
 250
 251   /**
 252    * Verify that when the table is empty, making metadata changes after the restore does not affect
 253    * the restored table's original metadata
 254    * @throws Exception
 255    */
 256   @Test
 257   public void testDescribeOnEmptyTableMatchesAfterMetadataChangeAndRestore() throws Exception {
 258     runRestoreWithAdditionalMetadata(true, false);
 259   }
 260
 261   private void runRestoreWithAdditionalMetadata(boolean changeMetadata) throws Exception {
 262     runRestoreWithAdditionalMetadata(changeMetadata, true);
 263   }
 264
 265   private void runRestoreWithAdditionalMetadata(boolean changeMetadata, boolean addData)
 266       throws Exception {
 267
 268     if (admin.isTableDisabled(originalTableName)) {
 269       admin.enableTable(originalTableName);
 270     }
 271
 272     // populate it with data
 273     final byte[] familyForUpdate = BLOCKSIZE_FAM;
 274
 275     List<byte[]> familiesWithDataList = new ArrayList<>();
 276     List<byte[]> emptyFamiliesList = new ArrayList<>();
 277     if (addData) {
 278       Table original = UTIL.getConnection().getTable(originalTableName);
 279       UTIL.loadTable(original, familyForUpdate); // family arbitrarily chosen
 280       original.close();
 281
 282       for (byte[] family : families) {
 283         if (family != familyForUpdate) {
 284           emptyFamiliesList.add(family);
 285         }
 286       }
 287       familiesWithDataList.add(familyForUpdate);
 288     } else {
 289       Collections.addAll(emptyFamiliesList, families);
 290     }
 291
 292     // take a "disabled" snapshot
 293     final String snapshotNameAsString = "snapshot" + originalTableName
 294         + System.currentTimeMillis();
 295
 296     SnapshotTestingUtils.createSnapshotAndValidate(admin, originalTableName,
 297       familiesWithDataList, emptyFamiliesList, snapshotNameAsString, rootDir, fs,
 298       /* onlineSnapshot= */ false);
 299
 300     admin.enableTable(originalTableName);
 301
 302     if (changeMetadata) {
 303       final String newFamilyNameAsString = "newFamily" + System.currentTimeMillis();
 304       final byte[] newFamilyName = Bytes.toBytes(newFamilyNameAsString);
 305
 306       admin.disableTable(originalTableName);
 307       ColumnFamilyDescriptor familyDescriptor =
 308         new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(newFamilyName);
 309       admin.addColumnFamily(originalTableName, familyDescriptor);
 310       assertTrue("New column family was not added.",
 311         admin.getDescriptor(originalTableName).toString().contains(newFamilyNameAsString));
 312     }
 313
 314     // restore it
 315     if (!admin.isTableDisabled(originalTableName)) {
 316       admin.disableTable(originalTableName);
 317     }
 318
 319     admin.restoreSnapshot(snapshotNameAsString);
 320     admin.enableTable(originalTableName);
 321
 322     // verify that the descrption is reverted
 323     try (Table original = UTIL.getConnection().getTable(originalTableName)) {
 324       assertEquals(originalTableDescriptor,
 325         new HTableDescriptor(admin.getDescriptor(originalTableName)));
 326       assertEquals(originalTableDescriptor, new HTableDescriptor(original.getDescriptor()));
 327     }
 328   }
 329 }