hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackup.java

   1 /**
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase.backup;
  19
  20 import static org.junit.Assert.assertTrue;
  21
  22 import java.util.ArrayList;
  23 import java.util.Collection;
  24 import java.util.List;
  25 import org.apache.hadoop.hbase.HBaseClassTestRule;
  26 import org.apache.hadoop.hbase.HBaseTestingUtility;
  27 import org.apache.hadoop.hbase.HColumnDescriptor;
  28 import org.apache.hadoop.hbase.MiniHBaseCluster;
  29 import org.apache.hadoop.hbase.TableName;
  30 import org.apache.hadoop.hbase.backup.impl.BackupAdminImpl;
  31 import org.apache.hadoop.hbase.backup.util.BackupUtils;
  32 import org.apache.hadoop.hbase.client.Connection;
  33 import org.apache.hadoop.hbase.client.ConnectionFactory;
  34 import org.apache.hadoop.hbase.client.HBaseAdmin;
  35 import org.apache.hadoop.hbase.client.HTable;
  36 import org.apache.hadoop.hbase.client.Put;
  37 import org.apache.hadoop.hbase.regionserver.HRegion;
  38 import org.apache.hadoop.hbase.testclassification.LargeTests;
  39 import org.apache.hadoop.hbase.util.Bytes;
  40 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
  41 import org.junit.Assert;
  42 import org.junit.ClassRule;
  43 import org.junit.Test;
  44 import org.junit.experimental.categories.Category;
  45 import org.junit.runner.RunWith;
  46 import org.junit.runners.Parameterized;
  47 import org.slf4j.Logger;
  48 import org.slf4j.LoggerFactory;
  49
  50 import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
  51
  52 @Category(LargeTests.class)
  53 @RunWith(Parameterized.class)
  54 public class TestIncrementalBackup extends TestBackupBase {
  55
  56   @ClassRule
  57   public static final HBaseClassTestRule CLASS_RULE =
  58       HBaseClassTestRule.forClass(TestIncrementalBackup.class);
  59
  60   private static final Logger LOG = LoggerFactory.getLogger(TestIncrementalBackup.class);
  61
  62   @Parameterized.Parameters
  63   public static Collection<Object[]> data() {
  64     provider = "multiwal";
  65     List<Object[]> params = new ArrayList<>();
  66     params.add(new Object[] { Boolean.TRUE });
  67     return params;
  68   }
  69
  70   public TestIncrementalBackup(Boolean b) {
  71   }
  72
  73   // implement all test cases in 1 test since incremental
  74   // backup/restore has dependencies
  75   @Test
  76   public void TestIncBackupRestore() throws Exception {
  77     int ADD_ROWS = 99;
  78
  79     // #1 - create full backup for all tables
  80     LOG.info("create full backup image for all tables");
  81     List<TableName> tables = Lists.newArrayList(table1, table2);
  82     final byte[] fam3Name = Bytes.toBytes("f3");
  83     final byte[] mobName = Bytes.toBytes("mob");
  84
  85     table1Desc.addFamily(new HColumnDescriptor(fam3Name));
  86     HColumnDescriptor mobHcd = new HColumnDescriptor(mobName);
  87     mobHcd.setMobEnabled(true);
  88     mobHcd.setMobThreshold(5L);
  89     table1Desc.addFamily(mobHcd);
  90     HBaseTestingUtility.modifyTableSync(TEST_UTIL.getAdmin(), table1Desc);
  91
  92     try (Connection conn = ConnectionFactory.createConnection(conf1)) {
  93       int NB_ROWS_FAM3 = 6;
  94       insertIntoTable(conn, table1, fam3Name, 3, NB_ROWS_FAM3).close();
  95       insertIntoTable(conn, table1, mobName, 3, NB_ROWS_FAM3).close();
  96       HBaseAdmin admin = null;
  97       admin = (HBaseAdmin) conn.getAdmin();
  98       BackupAdminImpl client = new BackupAdminImpl(conn);
  99       BackupRequest request = createBackupRequest(BackupType.FULL, tables, BACKUP_ROOT_DIR);
 100       String backupIdFull = client.backupTables(request);
 101       assertTrue(checkSucceeded(backupIdFull));
 102
 103       // #2 - insert some data to table
 104       HTable t1 = insertIntoTable(conn, table1, famName, 1, ADD_ROWS);
 105       LOG.debug("writing " + ADD_ROWS + " rows to " + table1);
 106       Assert.assertEquals(HBaseTestingUtility.countRows(t1),
 107               NB_ROWS_IN_BATCH + ADD_ROWS + NB_ROWS_FAM3);
 108       LOG.debug("written " + ADD_ROWS + " rows to " + table1);
 109
 110       // additionally, insert rows to MOB cf
 111       int NB_ROWS_MOB = 111;
 112       insertIntoTable(conn, table1, mobName, 3, NB_ROWS_MOB);
 113       LOG.debug("written " + NB_ROWS_MOB + " rows to " + table1 + " to Mob enabled CF");
 114       t1.close();
 115       Assert.assertEquals(HBaseTestingUtility.countRows(t1),
 116               NB_ROWS_IN_BATCH + ADD_ROWS + NB_ROWS_MOB);
 117
 118       HTable t2 = (HTable) conn.getTable(table2);
 119       Put p2;
 120       for (int i = 0; i < 5; i++) {
 121         p2 = new Put(Bytes.toBytes("row-t2" + i));
 122         p2.addColumn(famName, qualName, Bytes.toBytes("val" + i));
 123         t2.put(p2);
 124       }
 125       Assert.assertEquals(NB_ROWS_IN_BATCH + 5, HBaseTestingUtility.countRows(t2));
 126       t2.close();
 127       LOG.debug("written " + 5 + " rows to " + table2);
 128
 129       // split table1
 130       MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
 131       List<HRegion> regions = cluster.getRegions(table1);
 132       byte[] name = regions.get(0).getRegionInfo().getRegionName();
 133       long startSplitTime = EnvironmentEdgeManager.currentTime();
 134
 135       try {
 136         admin.splitRegionAsync(name, null).get();
 137       } catch (Exception e) {
 138         // although split fail, this may not affect following check in current API,
 139         // exception will be thrown.
 140         LOG.debug("region is not splittable, because " + e);
 141       }
 142       while (!admin.isTableAvailable(table1)) {
 143         Thread.sleep(100);
 144       }
 145
 146       long endSplitTime = EnvironmentEdgeManager.currentTime();
 147       // split finished
 148       LOG.debug("split finished in =" + (endSplitTime - startSplitTime));
 149
 150       // #3 - incremental backup for multiple tables
 151       tables = Lists.newArrayList(table1, table2);
 152       request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR);
 153       String backupIdIncMultiple = client.backupTables(request);
 154       assertTrue(checkSucceeded(backupIdIncMultiple));
 155
 156       // add column family f2 to table1
 157       final byte[] fam2Name = Bytes.toBytes("f2");
 158       table1Desc.addFamily(new HColumnDescriptor(fam2Name));
 159
 160       // drop column family f3
 161       table1Desc.removeFamily(fam3Name);
 162       HBaseTestingUtility.modifyTableSync(TEST_UTIL.getAdmin(), table1Desc);
 163
 164       int NB_ROWS_FAM2 = 7;
 165       HTable t3 = insertIntoTable(conn, table1, fam2Name, 2, NB_ROWS_FAM2);
 166       t3.close();
 167
 168       // Wait for 5 sec to make sure that old WALs were deleted
 169       Thread.sleep(5000);
 170
 171       // #4 - additional incremental backup for multiple tables
 172       request = createBackupRequest(BackupType.INCREMENTAL, tables, BACKUP_ROOT_DIR);
 173       String backupIdIncMultiple2 = client.backupTables(request);
 174       assertTrue(checkSucceeded(backupIdIncMultiple2));
 175
 176       // #5 - restore full backup for all tables
 177       TableName[] tablesRestoreFull = new TableName[] { table1, table2 };
 178       TableName[] tablesMapFull = new TableName[] { table1_restore, table2_restore };
 179
 180       LOG.debug("Restoring full " + backupIdFull);
 181       client.restore(BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, backupIdFull, false,
 182                 tablesRestoreFull, tablesMapFull, true));
 183
 184       // #6.1 - check tables for full restore
 185       HBaseAdmin hAdmin = TEST_UTIL.getHBaseAdmin();
 186       assertTrue(hAdmin.tableExists(table1_restore));
 187       assertTrue(hAdmin.tableExists(table2_restore));
 188       hAdmin.close();
 189
 190       // #6.2 - checking row count of tables for full restore
 191       HTable hTable = (HTable) conn.getTable(table1_restore);
 192       Assert.assertEquals(HBaseTestingUtility.countRows(hTable), NB_ROWS_IN_BATCH + NB_ROWS_FAM3);
 193       hTable.close();
 194
 195       hTable = (HTable) conn.getTable(table2_restore);
 196       Assert.assertEquals(NB_ROWS_IN_BATCH, HBaseTestingUtility.countRows(hTable));
 197       hTable.close();
 198
 199       // #7 - restore incremental backup for multiple tables, with overwrite
 200       TableName[] tablesRestoreIncMultiple = new TableName[] { table1, table2 };
 201       TableName[] tablesMapIncMultiple = new TableName[] { table1_restore, table2_restore };
 202       client.restore(BackupUtils.createRestoreRequest(BACKUP_ROOT_DIR, backupIdIncMultiple2,
 203               false, tablesRestoreIncMultiple, tablesMapIncMultiple, true));
 204       hTable = (HTable) conn.getTable(table1_restore);
 205
 206       LOG.debug("After incremental restore: " + hTable.getDescriptor());
 207       int countFamName = TEST_UTIL.countRows(hTable, famName);
 208       LOG.debug("f1 has " + countFamName + " rows");
 209       Assert.assertEquals(countFamName, NB_ROWS_IN_BATCH + ADD_ROWS);
 210
 211       int countFam2Name = TEST_UTIL.countRows(hTable, fam2Name);
 212       LOG.debug("f2 has " + countFam2Name + " rows");
 213       Assert.assertEquals(countFam2Name, NB_ROWS_FAM2);
 214
 215       int countMobName = TEST_UTIL.countRows(hTable, mobName);
 216       LOG.debug("mob has " + countMobName + " rows");
 217       Assert.assertEquals(countMobName, NB_ROWS_MOB);
 218       hTable.close();
 219
 220       hTable = (HTable) conn.getTable(table2_restore);
 221       Assert.assertEquals(NB_ROWS_IN_BATCH + 5, HBaseTestingUtility.countRows(hTable));
 222       hTable.close();
 223       admin.close();
 224     }
 225   }
 226 }