hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestSnapshotFromClient.java

   1 /**
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase.client;
  19
  20 import static org.junit.Assert.assertEquals;
  21 import static org.junit.Assert.assertFalse;
  22 import static org.junit.Assert.assertTrue;
  23 import static org.junit.Assert.fail;
  24
  25 import java.util.ArrayList;
  26 import java.util.Arrays;
  27 import java.util.List;
  28 import java.util.regex.Pattern;
  29 import org.apache.hadoop.conf.Configuration;
  30 import org.apache.hadoop.fs.FileSystem;
  31 import org.apache.hadoop.fs.Path;
  32 import org.apache.hadoop.hbase.HBaseClassTestRule;
  33 import org.apache.hadoop.hbase.HBaseTestingUtil;
  34 import org.apache.hadoop.hbase.HConstants;
  35 import org.apache.hadoop.hbase.TableName;
  36 import org.apache.hadoop.hbase.TableNameTestRule;
  37 import org.apache.hadoop.hbase.TableNotFoundException;
  38 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
  39 import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
  40 import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
  41 import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
  42 import org.apache.hadoop.hbase.snapshot.SnapshotDoesNotExistException;
  43 import org.apache.hadoop.hbase.snapshot.SnapshotManifestV1;
  44 import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
  45 import org.apache.hadoop.hbase.testclassification.ClientTests;
  46 import org.apache.hadoop.hbase.testclassification.LargeTests;
  47 import org.apache.hadoop.hbase.util.Bytes;
  48 import org.apache.hadoop.hbase.util.CommonFSUtils;
  49 import org.junit.After;
  50 import org.junit.AfterClass;
  51 import org.junit.Before;
  52 import org.junit.BeforeClass;
  53 import org.junit.ClassRule;
  54 import org.junit.Rule;
  55 import org.junit.Test;
  56 import org.junit.experimental.categories.Category;
  57 import org.junit.runner.RunWith;
  58 import org.junit.runners.Parameterized;
  59 import org.junit.runners.Parameterized.Parameter;
  60 import org.junit.runners.Parameterized.Parameters;
  61 import org.slf4j.Logger;
  62 import org.slf4j.LoggerFactory;
  63
  64 import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
  65
  66 import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
  67
  68 /**
  69  * Test create/using/deleting snapshots from the client
  70  * <p>
  71  * This is an end-to-end test for the snapshot utility
  72  */
  73 @RunWith(Parameterized.class)
  74 @Category({ LargeTests.class, ClientTests.class })
  75 public class TestSnapshotFromClient {
  76
  77   @ClassRule
  78   public static final HBaseClassTestRule CLASS_RULE =
  79       HBaseClassTestRule.forClass(TestSnapshotFromClient.class);
  80
  81   private static final Logger LOG = LoggerFactory.getLogger(TestSnapshotFromClient.class);
  82
  83   protected static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
  84   protected static final int NUM_RS = 2;
  85   protected static final String STRING_TABLE_NAME = "test";
  86   protected static final byte[] TEST_FAM = Bytes.toBytes("fam");
  87   protected static final TableName TABLE_NAME =
  88       TableName.valueOf(STRING_TABLE_NAME);
  89   private static final Pattern MATCH_ALL = Pattern.compile(".*");
  90
  91   @Rule
  92   public TableNameTestRule name = new TableNameTestRule();
  93
  94   @Parameter
  95   public StoreFileTrackerFactory.Trackers trackerImpl;
  96
  97   @Parameters(name = "{index}: tracker={0}")
  98   public static List<Object[]> params() {
  99     return Arrays.asList(new Object[] { StoreFileTrackerFactory.Trackers.DEFAULT },
 100       new Object[] { StoreFileTrackerFactory.Trackers.FILE });
 101   }
 102
 103   /**
 104    * Setup the config for the cluster
 105    * @throws Exception on failure
 106    */
 107   @BeforeClass
 108   public static void setupCluster() throws Exception {
 109     setupConf(UTIL.getConfiguration());
 110     UTIL.startMiniCluster(NUM_RS);
 111   }
 112
 113   protected static void setupConf(Configuration conf) {
 114     // disable the ui
 115     conf.setInt("hbase.regionsever.info.port", -1);
 116     // change the flush size to a small amount, regulating number of store files
 117     conf.setInt("hbase.hregion.memstore.flush.size", 25000);
 118     // so make sure we get a compaction when doing a load, but keep around some
 119     // files in the store
 120     conf.setInt("hbase.hstore.compaction.min", 10);
 121     conf.setInt("hbase.hstore.compactionThreshold", 10);
 122     // block writes if we get to 12 store files
 123     conf.setInt("hbase.hstore.blockingStoreFiles", 12);
 124     // Enable snapshot
 125     conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
 126     conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
 127         ConstantSizeRegionSplitPolicy.class.getName());
 128   }
 129
 130   @Before
 131   public void setup() throws Exception {
 132     createTable();
 133   }
 134
 135   protected void createTable() throws Exception {
 136     TableDescriptor htd =
 137       TableDescriptorBuilder.newBuilder(TABLE_NAME).setRegionReplication(getNumReplicas())
 138         .setValue(StoreFileTrackerFactory.TRACKER_IMPL, trackerImpl.name()).build();
 139     UTIL.createTable(htd, new byte[][] { TEST_FAM }, null);
 140   }
 141
 142   protected int getNumReplicas() {
 143     return 1;
 144   }
 145
 146   @After
 147   public void tearDown() throws Exception {
 148     UTIL.deleteTable(TABLE_NAME);
 149     SnapshotTestingUtils.deleteAllSnapshots(UTIL.getAdmin());
 150     SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
 151   }
 152
 153   @AfterClass
 154   public static void cleanupTest() throws Exception {
 155     try {
 156       UTIL.shutdownMiniCluster();
 157     } catch (Exception e) {
 158       LOG.warn("failure shutting down cluster", e);
 159     }
 160   }
 161
 162   /**
 163    * Test snapshotting not allowed hbase:meta and -ROOT-
 164    */
 165   @Test(expected = IllegalArgumentException.class)
 166   public void testMetaTablesSnapshot() throws Exception {
 167     UTIL.getAdmin().snapshot("metaSnapshot", TableName.META_TABLE_NAME);
 168   }
 169
 170   /**
 171    * Test HBaseAdmin#deleteSnapshots(String) which deletes snapshots whose names match the parameter
 172    *
 173    * @throws Exception
 174    */
 175   @Test
 176   public void testSnapshotDeletionWithRegex() throws Exception {
 177     Admin admin = UTIL.getAdmin();
 178     // make sure we don't fail on listing snapshots
 179     SnapshotTestingUtils.assertNoSnapshots(admin);
 180
 181     // put some stuff in the table
 182     Table table = UTIL.getConnection().getTable(TABLE_NAME);
 183     UTIL.loadTable(table, TEST_FAM);
 184     table.close();
 185
 186     String snapshot1 = "TableSnapshot1";
 187     admin.snapshot(snapshot1, TABLE_NAME);
 188     LOG.debug("Snapshot1 completed.");
 189
 190     String snapshot2 = "TableSnapshot2";
 191     admin.snapshot(snapshot2, TABLE_NAME);
 192     LOG.debug("Snapshot2 completed.");
 193
 194     String snapshot3 = "3rdTableSnapshot";
 195     admin.snapshot(snapshot3, TABLE_NAME);
 196     LOG.debug(snapshot3 + " completed.");
 197
 198     // delete the first two snapshots
 199     admin.deleteSnapshots(Pattern.compile("TableSnapshot.*"));
 200     List<SnapshotDescription> snapshots = admin.listSnapshots();
 201     assertEquals(1, snapshots.size());
 202     assertEquals(snapshot3, snapshots.get(0).getName());
 203
 204     admin.deleteSnapshot(snapshot3);
 205     admin.close();
 206   }
 207   /**
 208    * Test snapshotting a table that is offline
 209    * @throws Exception
 210    */
 211   @Test
 212   public void testOfflineTableSnapshot() throws Exception {
 213     Admin admin = UTIL.getAdmin();
 214     // make sure we don't fail on listing snapshots
 215     SnapshotTestingUtils.assertNoSnapshots(admin);
 216
 217     // put some stuff in the table
 218     Table table = UTIL.getConnection().getTable(TABLE_NAME);
 219     UTIL.loadTable(table, TEST_FAM, false);
 220
 221     LOG.debug("FS state before disable:");
 222     CommonFSUtils.logFileSystemState(UTIL.getTestFileSystem(),
 223       CommonFSUtils.getRootDir(UTIL.getConfiguration()), LOG);
 224     // XXX if this is flakey, might want to consider using the async version and looping as
 225     // disableTable can succeed and still timeout.
 226     admin.disableTable(TABLE_NAME);
 227
 228     LOG.debug("FS state before snapshot:");
 229     CommonFSUtils.logFileSystemState(UTIL.getTestFileSystem(),
 230       CommonFSUtils.getRootDir(UTIL.getConfiguration()), LOG);
 231
 232     // take a snapshot of the disabled table
 233     final String SNAPSHOT_NAME = "offlineTableSnapshot";
 234     String snapshot = SNAPSHOT_NAME;
 235
 236     admin.snapshot(new SnapshotDescription(SNAPSHOT_NAME, TABLE_NAME,
 237         SnapshotType.DISABLED, null, -1, SnapshotManifestV1.DESCRIPTOR_VERSION, null));
 238     LOG.debug("Snapshot completed.");
 239
 240     // make sure we have the snapshot
 241     List<SnapshotDescription> snapshots =
 242         SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot, TABLE_NAME);
 243
 244     // make sure its a valid snapshot
 245     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
 246     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
 247     LOG.debug("FS state after snapshot:");
 248     CommonFSUtils.logFileSystemState(UTIL.getTestFileSystem(),
 249       CommonFSUtils.getRootDir(UTIL.getConfiguration()), LOG);
 250     SnapshotTestingUtils.confirmSnapshotValid(
 251       ProtobufUtil.createHBaseProtosSnapshotDesc(snapshots.get(0)), TABLE_NAME, TEST_FAM,
 252       rootDir, admin, fs);
 253
 254     admin.deleteSnapshot(snapshot);
 255     snapshots = admin.listSnapshots();
 256     SnapshotTestingUtils.assertNoSnapshots(admin);
 257   }
 258
 259   @Test
 260   public void testSnapshotFailsOnNonExistantTable() throws Exception {
 261     Admin admin = UTIL.getAdmin();
 262     // make sure we don't fail on listing snapshots
 263     SnapshotTestingUtils.assertNoSnapshots(admin);
 264     String tableName = "_not_a_table";
 265
 266     // make sure the table doesn't exist
 267     boolean fail = false;
 268     do {
 269       try {
 270         admin.getDescriptor(TableName.valueOf(tableName));
 271         fail = true;
 272         LOG.error("Table:" + tableName + " already exists, checking a new name");
 273         tableName = tableName + "!";
 274       } catch (TableNotFoundException e) {
 275         fail = false;
 276       }
 277     } while (fail);
 278
 279     // snapshot the non-existant table
 280     try {
 281       admin.snapshot("fail", TableName.valueOf(tableName));
 282       fail("Snapshot succeeded even though there is not table.");
 283     } catch (SnapshotCreationException e) {
 284       LOG.info("Correctly failed to snapshot a non-existant table:" + e.getMessage());
 285     }
 286   }
 287
 288   @Test
 289   public void testOfflineTableSnapshotWithEmptyRegions() throws Exception {
 290     // test with an empty table with one region
 291
 292     Admin admin = UTIL.getAdmin();
 293     // make sure we don't fail on listing snapshots
 294     SnapshotTestingUtils.assertNoSnapshots(admin);
 295
 296     LOG.debug("FS state before disable:");
 297     CommonFSUtils.logFileSystemState(UTIL.getTestFileSystem(),
 298       CommonFSUtils.getRootDir(UTIL.getConfiguration()), LOG);
 299     admin.disableTable(TABLE_NAME);
 300
 301     LOG.debug("FS state before snapshot:");
 302     CommonFSUtils.logFileSystemState(UTIL.getTestFileSystem(),
 303       CommonFSUtils.getRootDir(UTIL.getConfiguration()), LOG);
 304
 305     // take a snapshot of the disabled table
 306     String snapshot = "testOfflineTableSnapshotWithEmptyRegions";
 307     admin.snapshot(snapshot, TABLE_NAME);
 308     LOG.debug("Snapshot completed.");
 309
 310     // make sure we have the snapshot
 311     List<SnapshotDescription> snapshots =
 312         SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot, TABLE_NAME);
 313
 314     // make sure its a valid snapshot
 315     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
 316     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
 317     LOG.debug("FS state after snapshot:");
 318     CommonFSUtils.logFileSystemState(UTIL.getTestFileSystem(),
 319       CommonFSUtils.getRootDir(UTIL.getConfiguration()), LOG);
 320
 321     List<byte[]> emptyCfs = Lists.newArrayList(TEST_FAM); // no file in the region
 322     List<byte[]> nonEmptyCfs = Lists.newArrayList();
 323     SnapshotTestingUtils.confirmSnapshotValid(
 324       ProtobufUtil.createHBaseProtosSnapshotDesc(snapshots.get(0)), TABLE_NAME, nonEmptyCfs,
 325       emptyCfs, rootDir, admin, fs);
 326
 327     admin.deleteSnapshot(snapshot);
 328     snapshots = admin.listSnapshots();
 329     SnapshotTestingUtils.assertNoSnapshots(admin);
 330   }
 331
 332   @Test
 333   public void testListTableSnapshots() throws Exception {
 334     Admin admin = null;
 335     final TableName tableName = name.getTableName();
 336     try {
 337       admin = UTIL.getAdmin();
 338
 339       TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName).build();
 340       UTIL.createTable(htd, new byte[][] { TEST_FAM }, UTIL.getConfiguration());
 341
 342       String table1Snapshot1 = "Table1Snapshot1";
 343       admin.snapshot(table1Snapshot1, TABLE_NAME);
 344       LOG.debug("Snapshot1 completed.");
 345
 346       String table1Snapshot2 = "Table1Snapshot2";
 347       admin.snapshot(table1Snapshot2, TABLE_NAME);
 348       LOG.debug("Snapshot2 completed.");
 349
 350       String table2Snapshot1 = "Table2Snapshot1";
 351       admin.snapshot(table2Snapshot1, tableName);
 352       LOG.debug(table2Snapshot1 + " completed.");
 353
 354       List<SnapshotDescription> listTableSnapshots =
 355           admin.listTableSnapshots(Pattern.compile("test.*"), MATCH_ALL);
 356       List<String> listTableSnapshotNames = new ArrayList<>();
 357       assertEquals(3, listTableSnapshots.size());
 358       for (SnapshotDescription s : listTableSnapshots) {
 359         listTableSnapshotNames.add(s.getName());
 360       }
 361       assertTrue(listTableSnapshotNames.contains(table1Snapshot1));
 362       assertTrue(listTableSnapshotNames.contains(table1Snapshot2));
 363       assertTrue(listTableSnapshotNames.contains(table2Snapshot1));
 364     } finally {
 365       if (admin != null) {
 366         try {
 367           admin.deleteSnapshots(Pattern.compile("Table.*"));
 368         } catch (SnapshotDoesNotExistException ignore) {
 369         }
 370         if (admin.tableExists(tableName)) {
 371           UTIL.deleteTable(tableName);
 372         }
 373         admin.close();
 374       }
 375     }
 376   }
 377
 378   @Test
 379   public void testListTableSnapshotsWithRegex() throws Exception {
 380     Admin admin = null;
 381     try {
 382       admin = UTIL.getAdmin();
 383
 384       String table1Snapshot1 = "Table1Snapshot1";
 385       admin.snapshot(table1Snapshot1, TABLE_NAME);
 386       LOG.debug("Snapshot1 completed.");
 387
 388       String table1Snapshot2 = "Table1Snapshot2";
 389       admin.snapshot(table1Snapshot2, TABLE_NAME);
 390       LOG.debug("Snapshot2 completed.");
 391
 392       String table2Snapshot1 = "Table2Snapshot1";
 393       admin.snapshot(table2Snapshot1, TABLE_NAME);
 394       LOG.debug(table2Snapshot1 + " completed.");
 395
 396       List<SnapshotDescription> listTableSnapshots =
 397           admin.listTableSnapshots(Pattern.compile("test.*"), Pattern.compile("Table1.*"));
 398       List<String> listTableSnapshotNames = new ArrayList<>();
 399       assertEquals(2, listTableSnapshots.size());
 400       for (SnapshotDescription s : listTableSnapshots) {
 401         listTableSnapshotNames.add(s.getName());
 402       }
 403       assertTrue(listTableSnapshotNames.contains(table1Snapshot1));
 404       assertTrue(listTableSnapshotNames.contains(table1Snapshot2));
 405       assertFalse(listTableSnapshotNames.contains(table2Snapshot1));
 406     } finally {
 407       if (admin != null) {
 408         try {
 409           admin.deleteSnapshots(Pattern.compile("Table.*"));
 410         } catch (SnapshotDoesNotExistException ignore) {
 411         }
 412         admin.close();
 413       }
 414     }
 415   }
 416
 417   @Test
 418   public void testDeleteTableSnapshots() throws Exception {
 419     Admin admin = null;
 420     final TableName tableName = name.getTableName();
 421     try {
 422       admin = UTIL.getAdmin();
 423
 424       TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName).build();
 425       UTIL.createTable(htd, new byte[][] { TEST_FAM }, UTIL.getConfiguration());
 426
 427       String table1Snapshot1 = "Table1Snapshot1";
 428       admin.snapshot(table1Snapshot1, TABLE_NAME);
 429       LOG.debug("Snapshot1 completed.");
 430
 431       String table1Snapshot2 = "Table1Snapshot2";
 432       admin.snapshot(table1Snapshot2, TABLE_NAME);
 433       LOG.debug("Snapshot2 completed.");
 434
 435       String table2Snapshot1 = "Table2Snapshot1";
 436       admin.snapshot(table2Snapshot1, tableName);
 437       LOG.debug(table2Snapshot1 + " completed.");
 438
 439       Pattern tableNamePattern = Pattern.compile("test.*");
 440       admin.deleteTableSnapshots(tableNamePattern, MATCH_ALL);
 441       assertEquals(0, admin.listTableSnapshots(tableNamePattern, MATCH_ALL).size());
 442     } finally {
 443       if (admin != null) {
 444         if (admin.tableExists(tableName)) {
 445           UTIL.deleteTable(tableName);
 446         }
 447         admin.close();
 448       }
 449     }
 450   }
 451
 452   @Test
 453   public void testDeleteTableSnapshotsWithRegex() throws Exception {
 454     Admin admin = null;
 455     Pattern tableNamePattern = Pattern.compile("test.*");
 456     try {
 457       admin = UTIL.getAdmin();
 458
 459       String table1Snapshot1 = "Table1Snapshot1";
 460       admin.snapshot(table1Snapshot1, TABLE_NAME);
 461       LOG.debug("Snapshot1 completed.");
 462
 463       String table1Snapshot2 = "Table1Snapshot2";
 464       admin.snapshot(table1Snapshot2, TABLE_NAME);
 465       LOG.debug("Snapshot2 completed.");
 466
 467       String table2Snapshot1 = "Table2Snapshot1";
 468       admin.snapshot(table2Snapshot1, TABLE_NAME);
 469       LOG.debug(table2Snapshot1 + " completed.");
 470
 471       admin.deleteTableSnapshots(tableNamePattern, Pattern.compile("Table1.*"));
 472       assertEquals(1, admin.listTableSnapshots(tableNamePattern, MATCH_ALL).size());
 473     } finally {
 474       if (admin != null) {
 475         try {
 476           admin.deleteTableSnapshots(tableNamePattern, MATCH_ALL);
 477         } catch (SnapshotDoesNotExistException ignore) {
 478         }
 479         admin.close();
 480       }
 481     }
 482   }
 483 }