hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotMetadata.java

   1 /**
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase.client;
  19
  20 import static org.junit.Assert.assertEquals;
  21 import static org.junit.Assert.assertTrue;
  22
  23 import java.util.ArrayList;
  24 import java.util.Collections;
  25 import java.util.List;
  26 import org.apache.hadoop.conf.Configuration;
  27 import org.apache.hadoop.fs.FileSystem;
  28 import org.apache.hadoop.fs.Path;
  29 import org.apache.hadoop.hbase.HBaseClassTestRule;
  30 import org.apache.hadoop.hbase.HBaseTestingUtil;
  31 import org.apache.hadoop.hbase.HConstants;
  32 import org.apache.hadoop.hbase.TableName;
  33 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
  34 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
  35 import org.apache.hadoop.hbase.regionserver.BloomType;
  36 import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
  37 import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
  38 import org.apache.hadoop.hbase.testclassification.ClientTests;
  39 import org.apache.hadoop.hbase.testclassification.MediumTests;
  40 import org.apache.hadoop.hbase.util.Bytes;
  41 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
  42 import org.junit.After;
  43 import org.junit.AfterClass;
  44 import org.junit.Before;
  45 import org.junit.BeforeClass;
  46 import org.junit.ClassRule;
  47 import org.junit.Test;
  48 import org.junit.experimental.categories.Category;
  49 import org.slf4j.Logger;
  50 import org.slf4j.LoggerFactory;
  51
  52 /**
  53  * Test class to verify that metadata is consistent before and after a snapshot attempt.
  54  */
  55 @Category({MediumTests.class, ClientTests.class})
  56 public class TestSnapshotMetadata {
  57
  58   @ClassRule
  59   public static final HBaseClassTestRule CLASS_RULE =
  60       HBaseClassTestRule.forClass(TestSnapshotMetadata.class);
  61
  62   private static final Logger LOG = LoggerFactory.getLogger(TestSnapshotMetadata.class);
  63
  64   private static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
  65   private static final int NUM_RS = 2;
  66   private static final String STRING_TABLE_NAME = "TestSnapshotMetadata";
  67
  68   private static final String MAX_VERSIONS_FAM_STR = "fam_max_columns";
  69   private static final byte[] MAX_VERSIONS_FAM = Bytes.toBytes(MAX_VERSIONS_FAM_STR);
  70
  71   private static final String COMPRESSED_FAM_STR = "fam_compressed";
  72   private static final byte[] COMPRESSED_FAM = Bytes.toBytes(COMPRESSED_FAM_STR);
  73
  74   private static final String BLOCKSIZE_FAM_STR = "fam_blocksize";
  75   private static final byte[] BLOCKSIZE_FAM = Bytes.toBytes(BLOCKSIZE_FAM_STR);
  76
  77   private static final String BLOOMFILTER_FAM_STR = "fam_bloomfilter";
  78   private static final byte[] BLOOMFILTER_FAM = Bytes.toBytes(BLOOMFILTER_FAM_STR);
  79
  80   private static final String TEST_CONF_CUSTOM_VALUE = "TestCustomConf";
  81   private static final String TEST_CUSTOM_VALUE = "TestCustomValue";
  82
  83   private static final byte[][] families = {
  84     MAX_VERSIONS_FAM, BLOOMFILTER_FAM, COMPRESSED_FAM, BLOCKSIZE_FAM
  85   };
  86
  87   private static final DataBlockEncoding DATA_BLOCK_ENCODING_TYPE = DataBlockEncoding.FAST_DIFF;
  88   private static final BloomType BLOOM_TYPE = BloomType.ROW;
  89   private static final int BLOCK_SIZE = 98;
  90   private static final int MAX_VERSIONS = 8;
  91
  92   private Admin admin;
  93   private String originalTableDescription;
  94   private TableDescriptor originalTableDescriptor;
  95   TableName originalTableName;
  96
  97   private static FileSystem fs;
  98   private static Path rootDir;
  99
 100   @BeforeClass
 101   public static void setupCluster() throws Exception {
 102     setupConf(UTIL.getConfiguration());
 103     UTIL.startMiniCluster(NUM_RS);
 104
 105     fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
 106     rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
 107   }
 108
 109   @AfterClass
 110   public static void cleanupTest() throws Exception {
 111     try {
 112       UTIL.shutdownMiniCluster();
 113     } catch (Exception e) {
 114       LOG.warn("failure shutting down cluster", e);
 115     }
 116   }
 117
 118   private static void setupConf(Configuration conf) {
 119     // enable snapshot support
 120     conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
 121     // disable the ui
 122     conf.setInt("hbase.regionsever.info.port", -1);
 123     // change the flush size to a small amount, regulating number of store files
 124     conf.setInt("hbase.hregion.memstore.flush.size", 25000);
 125     // so make sure we get a compaction when doing a load, but keep around
 126     // some files in the store
 127     conf.setInt("hbase.hstore.compaction.min", 10);
 128     conf.setInt("hbase.hstore.compactionThreshold", 10);
 129     // block writes if we get to 12 store files
 130     conf.setInt("hbase.hstore.blockingStoreFiles", 12);
 131     conf.setInt("hbase.regionserver.msginterval", 100);
 132     conf.setBoolean("hbase.master.enabletable.roundrobin", true);
 133     // Avoid potentially aggressive splitting which would cause snapshot to fail
 134     conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
 135       ConstantSizeRegionSplitPolicy.class.getName());
 136   }
 137
 138   @Before
 139   public void setup() throws Exception {
 140     admin = UTIL.getAdmin();
 141     createTableWithNonDefaultProperties();
 142   }
 143
 144   @After
 145   public void tearDown() throws Exception {
 146     SnapshotTestingUtils.deleteAllSnapshots(admin);
 147   }
 148
 149   /*
 150    *  Create a table that has non-default properties so we can see if they hold
 151    */
 152   private void createTableWithNonDefaultProperties() throws Exception {
 153     final long startTime = EnvironmentEdgeManager.currentTime();
 154     final String sourceTableNameAsString = STRING_TABLE_NAME + startTime;
 155     originalTableName = TableName.valueOf(sourceTableNameAsString);
 156
 157     // enable replication on a column family
 158     ColumnFamilyDescriptor maxVersionsColumn = ColumnFamilyDescriptorBuilder
 159       .newBuilder(MAX_VERSIONS_FAM).setMaxVersions(MAX_VERSIONS).build();
 160     ColumnFamilyDescriptor bloomFilterColumn = ColumnFamilyDescriptorBuilder
 161       .newBuilder(BLOOMFILTER_FAM).setBloomFilterType(BLOOM_TYPE).build();
 162     ColumnFamilyDescriptor dataBlockColumn = ColumnFamilyDescriptorBuilder
 163       .newBuilder(COMPRESSED_FAM).setDataBlockEncoding(DATA_BLOCK_ENCODING_TYPE).build();
 164     ColumnFamilyDescriptor blockSizeColumn =
 165       ColumnFamilyDescriptorBuilder.newBuilder(BLOCKSIZE_FAM).setBlocksize(BLOCK_SIZE).build();
 166
 167     TableDescriptor tableDescriptor = TableDescriptorBuilder
 168       .newBuilder(TableName.valueOf(sourceTableNameAsString)).setColumnFamily(maxVersionsColumn)
 169       .setColumnFamily(bloomFilterColumn).setColumnFamily(dataBlockColumn)
 170       .setColumnFamily(blockSizeColumn).setValue(TEST_CUSTOM_VALUE, TEST_CUSTOM_VALUE)
 171       .setValue(TEST_CONF_CUSTOM_VALUE, TEST_CONF_CUSTOM_VALUE).build();
 172     assertTrue(tableDescriptor.getValues().size() > 0);
 173
 174     admin.createTable(tableDescriptor);
 175     Table original = UTIL.getConnection().getTable(originalTableName);
 176     originalTableName = TableName.valueOf(sourceTableNameAsString);
 177     originalTableDescriptor = admin.getDescriptor(originalTableName);
 178     originalTableDescription = originalTableDescriptor.toStringCustomizedValues();
 179
 180     original.close();
 181   }
 182
 183
 184   /**
 185    * Verify that the describe for a cloned table matches the describe from the original.
 186    */
 187   @Test
 188   public void testDescribeMatchesAfterClone() throws Exception {
 189     // Clone the original table
 190     final String clonedTableNameAsString = "clone" + originalTableName;
 191     final TableName clonedTableName = TableName.valueOf(clonedTableNameAsString);
 192     final String snapshotNameAsString = "snapshot" + originalTableName
 193         + EnvironmentEdgeManager.currentTime();
 194     final String snapshotName = snapshotNameAsString;
 195
 196     // restore the snapshot into a cloned table and examine the output
 197     List<byte[]> familiesList = new ArrayList<>();
 198     Collections.addAll(familiesList, families);
 199
 200     // Create a snapshot in which all families are empty
 201     SnapshotTestingUtils.createSnapshotAndValidate(admin, originalTableName, null,
 202       familiesList, snapshotNameAsString, rootDir, fs, /* onlineSnapshot= */ false);
 203
 204     admin.cloneSnapshot(snapshotName, clonedTableName);
 205     Table clonedTable = UTIL.getConnection().getTable(clonedTableName);
 206     TableDescriptor cloneHtd = admin.getDescriptor(clonedTableName);
 207     assertEquals(
 208       originalTableDescription.replace(originalTableName.getNameAsString(),clonedTableNameAsString),
 209       cloneHtd.toStringCustomizedValues());
 210
 211     // Verify the custom fields
 212     assertEquals(originalTableDescriptor.getValues().size(),
 213                         cloneHtd.getValues().size());
 214     assertEquals(TEST_CUSTOM_VALUE, cloneHtd.getValue(TEST_CUSTOM_VALUE));
 215     assertEquals(TEST_CONF_CUSTOM_VALUE, cloneHtd.getValue(TEST_CONF_CUSTOM_VALUE));
 216     assertEquals(originalTableDescriptor.getValues(), cloneHtd.getValues());
 217
 218     admin.enableTable(originalTableName);
 219     clonedTable.close();
 220   }
 221
 222   /**
 223    * Verify that the describe for a restored table matches the describe for one the original.
 224    */
 225   @Test
 226   public void testDescribeMatchesAfterRestore() throws Exception {
 227     runRestoreWithAdditionalMetadata(false);
 228   }
 229
 230   /**
 231    * Verify that if metadata changed after a snapshot was taken, that the old metadata replaces the
 232    * new metadata during a restore
 233    */
 234   @Test
 235   public void testDescribeMatchesAfterMetadataChangeAndRestore() throws Exception {
 236     runRestoreWithAdditionalMetadata(true);
 237   }
 238
 239   /**
 240    * Verify that when the table is empty, making metadata changes after the restore does not affect
 241    * the restored table's original metadata
 242    * @throws Exception
 243    */
 244   @Test
 245   public void testDescribeOnEmptyTableMatchesAfterMetadataChangeAndRestore() throws Exception {
 246     runRestoreWithAdditionalMetadata(true, false);
 247   }
 248
 249   private void runRestoreWithAdditionalMetadata(boolean changeMetadata) throws Exception {
 250     runRestoreWithAdditionalMetadata(changeMetadata, true);
 251   }
 252
 253   private void runRestoreWithAdditionalMetadata(boolean changeMetadata, boolean addData)
 254       throws Exception {
 255
 256     if (admin.isTableDisabled(originalTableName)) {
 257       admin.enableTable(originalTableName);
 258     }
 259
 260     // populate it with data
 261     final byte[] familyForUpdate = BLOCKSIZE_FAM;
 262
 263     List<byte[]> familiesWithDataList = new ArrayList<>();
 264     List<byte[]> emptyFamiliesList = new ArrayList<>();
 265     if (addData) {
 266       Table original = UTIL.getConnection().getTable(originalTableName);
 267       UTIL.loadTable(original, familyForUpdate); // family arbitrarily chosen
 268       original.close();
 269
 270       for (byte[] family : families) {
 271         if (family != familyForUpdate) {
 272           emptyFamiliesList.add(family);
 273         }
 274       }
 275       familiesWithDataList.add(familyForUpdate);
 276     } else {
 277       Collections.addAll(emptyFamiliesList, families);
 278     }
 279
 280     // take a "disabled" snapshot
 281     final String snapshotNameAsString = "snapshot" + originalTableName
 282         + EnvironmentEdgeManager.currentTime();
 283
 284     SnapshotTestingUtils.createSnapshotAndValidate(admin, originalTableName,
 285       familiesWithDataList, emptyFamiliesList, snapshotNameAsString, rootDir, fs,
 286       /* onlineSnapshot= */ false);
 287
 288     admin.enableTable(originalTableName);
 289
 290     if (changeMetadata) {
 291       final String newFamilyNameAsString = "newFamily" + EnvironmentEdgeManager.currentTime();
 292       final byte[] newFamilyName = Bytes.toBytes(newFamilyNameAsString);
 293
 294       admin.disableTable(originalTableName);
 295       ColumnFamilyDescriptor familyDescriptor = ColumnFamilyDescriptorBuilder.of(newFamilyName);
 296       admin.addColumnFamily(originalTableName, familyDescriptor);
 297       assertTrue("New column family was not added.",
 298         admin.getDescriptor(originalTableName).toString().contains(newFamilyNameAsString));
 299     }
 300
 301     // restore it
 302     if (!admin.isTableDisabled(originalTableName)) {
 303       admin.disableTable(originalTableName);
 304     }
 305
 306     admin.restoreSnapshot(snapshotNameAsString);
 307     admin.enableTable(originalTableName);
 308
 309     // verify that the descrption is reverted
 310     try (Table original = UTIL.getConnection().getTable(originalTableName)) {
 311       assertEquals(originalTableDescriptor, admin.getDescriptor(originalTableName));
 312       assertEquals(originalTableDescriptor, original.getDescriptor());
 313     }
 314   }
 315 }