hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestSpaceQuotasWithSnapshots.java

   1 /**
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase.quotas;
  19
  20 import static org.junit.Assert.assertEquals;
  21 import static org.junit.Assert.assertNotNull;
  22 import static org.junit.Assert.assertTrue;
  23
  24 import java.io.IOException;
  25 import java.util.Map;
  26 import java.util.concurrent.atomic.AtomicInteger;
  27 import java.util.concurrent.atomic.AtomicLong;
  28 import org.apache.hadoop.conf.Configuration;
  29 import org.apache.hadoop.hbase.Cell;
  30 import org.apache.hadoop.hbase.CellScanner;
  31 import org.apache.hadoop.hbase.HBaseClassTestRule;
  32 import org.apache.hadoop.hbase.HBaseTestingUtil;
  33 import org.apache.hadoop.hbase.TableName;
  34 import org.apache.hadoop.hbase.Waiter;
  35 import org.apache.hadoop.hbase.Waiter.Predicate;
  36 import org.apache.hadoop.hbase.client.Admin;
  37 import org.apache.hadoop.hbase.client.Connection;
  38 import org.apache.hadoop.hbase.client.Result;
  39 import org.apache.hadoop.hbase.client.ResultScanner;
  40 import org.apache.hadoop.hbase.client.Scan;
  41 import org.apache.hadoop.hbase.client.SnapshotType;
  42 import org.apache.hadoop.hbase.client.Table;
  43 import org.apache.hadoop.hbase.quotas.SpaceQuotaHelperForTests.SpaceQuotaSnapshotPredicate;
  44 import org.apache.hadoop.hbase.testclassification.LargeTests;
  45 import org.junit.AfterClass;
  46 import org.junit.Before;
  47 import org.junit.BeforeClass;
  48 import org.junit.ClassRule;
  49 import org.junit.Rule;
  50 import org.junit.Test;
  51 import org.junit.experimental.categories.Category;
  52 import org.junit.rules.TestName;
  53 import org.slf4j.Logger;
  54 import org.slf4j.LoggerFactory;
  55
  56 import org.apache.hbase.thirdparty.com.google.common.collect.Iterables;
  57 import org.apache.hbase.thirdparty.com.google.protobuf.UnsafeByteOperations;
  58
  59 import org.apache.hadoop.hbase.shaded.protobuf.generated.QuotaProtos;
  60
  61 /**
  62  * Test class to exercise the inclusion of snapshots in space quotas
  63  */
  64 @Category({LargeTests.class})
  65 public class TestSpaceQuotasWithSnapshots {
  66
  67   @ClassRule
  68   public static final HBaseClassTestRule CLASS_RULE =
  69       HBaseClassTestRule.forClass(TestSpaceQuotasWithSnapshots.class);
  70
  71   private static final Logger LOG = LoggerFactory.getLogger(TestSpaceQuotasWithSnapshots.class);
  72   private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
  73   // Global for all tests in the class
  74   private static final AtomicLong COUNTER = new AtomicLong(0);
  75   private static final long FUDGE_FOR_TABLE_SIZE = 500L * SpaceQuotaHelperForTests.ONE_KILOBYTE;
  76
  77   @Rule
  78   public TestName testName = new TestName();
  79   private SpaceQuotaHelperForTests helper;
  80   private Connection conn;
  81   private Admin admin;
  82
  83   @BeforeClass
  84   public static void setUp() throws Exception {
  85     Configuration conf = TEST_UTIL.getConfiguration();
  86     SpaceQuotaHelperForTests.updateConfigForQuotas(conf);
  87     TEST_UTIL.startMiniCluster(1);
  88     // Wait till quota table onlined.
  89     TEST_UTIL.waitFor(10000, new Waiter.Predicate<Exception>() {
  90       @Override
  91       public boolean evaluate() throws Exception {
  92         return TEST_UTIL.getAdmin().tableExists(QuotaTableUtil.QUOTA_TABLE_NAME);
  93       }
  94     });
  95   }
  96
  97   @AfterClass
  98   public static void tearDown() throws Exception {
  99     TEST_UTIL.shutdownMiniCluster();
 100   }
 101
 102   @Before
 103   public void removeAllQuotas() throws Exception {
 104     helper = new SpaceQuotaHelperForTests(TEST_UTIL, testName, COUNTER);
 105     conn = TEST_UTIL.getConnection();
 106     admin = TEST_UTIL.getAdmin();
 107   }
 108
 109   @Test
 110   public void testTablesInheritSnapshotSize() throws Exception {
 111     TableName tn = helper.createTableWithRegions(1);
 112     LOG.info("Writing data");
 113     // Set a quota
 114     QuotaSettings settings = QuotaSettingsFactory.limitTableSpace(
 115         tn, SpaceQuotaHelperForTests.ONE_GIGABYTE, SpaceViolationPolicy.NO_INSERTS);
 116     admin.setQuota(settings);
 117     // Write some data
 118     final long initialSize = 2L * SpaceQuotaHelperForTests.ONE_MEGABYTE;
 119     helper.writeData(tn, initialSize);
 120
 121     LOG.info("Waiting until table size reflects written data");
 122     // Wait until that data is seen by the master
 123     TEST_UTIL.waitFor(30 * 1000, 500, new SpaceQuotaSnapshotPredicate(conn, tn) {
 124       @Override boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception {
 125         return snapshot.getUsage() >= initialSize;
 126       }
 127     });
 128
 129     // Make sure we see the final quota usage size
 130     waitForStableQuotaSize(conn, tn, null);
 131
 132     // The actual size on disk after we wrote our data the first time
 133     final long actualInitialSize = conn.getAdmin().getCurrentSpaceQuotaSnapshot(tn).getUsage();
 134     LOG.info("Initial table size was " + actualInitialSize);
 135
 136     LOG.info("Snapshot the table");
 137     final String snapshot1 = tn.toString() + "_snapshot1";
 138     admin.snapshot(snapshot1, tn);
 139
 140     // Write the same data again, then flush+compact. This should make sure that
 141     // the snapshot is referencing files that the table no longer references.
 142     LOG.info("Write more data");
 143     helper.writeData(tn, initialSize);
 144     LOG.info("Flush the table");
 145     admin.flush(tn);
 146     LOG.info("Synchronously compacting the table");
 147     TEST_UTIL.compact(tn, true);
 148
 149     final long upperBound = initialSize + FUDGE_FOR_TABLE_SIZE;
 150     final long lowerBound = initialSize - FUDGE_FOR_TABLE_SIZE;
 151
 152     // Store the actual size after writing more data and then compacting it down to one file
 153     LOG.info("Waiting for the region reports to reflect the correct size, between ("
 154         + lowerBound + ", " + upperBound + ")");
 155     TEST_UTIL.waitFor(30 * 1000, 500, new Predicate<Exception>() {
 156       @Override
 157       public boolean evaluate() throws Exception {
 158         long size = getRegionSizeReportForTable(conn, tn);
 159         return size < upperBound && size > lowerBound;
 160       }
 161     });
 162
 163     // Make sure we see the "final" new size for the table, not some intermediate
 164     waitForStableRegionSizeReport(conn, tn);
 165     final long finalSize = getRegionSizeReportForTable(conn, tn);
 166     assertNotNull("Did not expect to see a null size", finalSize);
 167     LOG.info("Last seen size: " + finalSize);
 168
 169     // Make sure the QuotaObserverChore has time to reflect the new region size reports
 170     // (we saw above). The usage of the table should *not* decrease when we check it below,
 171     // though, because the snapshot on our table will cause the table to "retain" the size.
 172     TEST_UTIL.waitFor(20 * 1000, 500, new SpaceQuotaSnapshotPredicate(conn, tn) {
 173       @Override
 174       public boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception {
 175         return snapshot.getUsage() >= finalSize;
 176       }
 177     });
 178
 179     // The final usage should be the sum of the initial size (referenced by the snapshot) and the
 180     // new size we just wrote above.
 181     long expectedFinalSize = actualInitialSize + finalSize;
 182     LOG.info(
 183         "Expecting table usage to be " + actualInitialSize + " + " + finalSize
 184         + " = " + expectedFinalSize);
 185     // The size of the table (WRT quotas) should now be approximately double what it was previously
 186     TEST_UTIL.waitFor(30 * 1000, 1000, new SpaceQuotaSnapshotPredicate(conn, tn) {
 187       @Override boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception {
 188         LOG.debug("Checking for " + expectedFinalSize + " == " + snapshot.getUsage());
 189         return expectedFinalSize == snapshot.getUsage();
 190       }
 191     });
 192
 193     Map<String,Long> snapshotSizes = QuotaTableUtil.getObservedSnapshotSizes(conn);
 194     Long size = snapshotSizes.get(snapshot1);
 195     assertNotNull("Did not observe the size of the snapshot", size);
 196     assertEquals(
 197         "The recorded size of the HBase snapshot was not the size we expected", actualInitialSize,
 198         size.longValue());
 199   }
 200
 201   @Test
 202   public void testNamespacesInheritSnapshotSize() throws Exception {
 203     String ns = helper.createNamespace().getName();
 204     TableName tn = helper.createTableWithRegions(ns, 1);
 205     LOG.info("Writing data");
 206     // Set a quota
 207     QuotaSettings settings = QuotaSettingsFactory.limitNamespaceSpace(
 208         ns, SpaceQuotaHelperForTests.ONE_GIGABYTE, SpaceViolationPolicy.NO_INSERTS);
 209     admin.setQuota(settings);
 210
 211     // Write some data and flush it to disk
 212     final long initialSize = 2L * SpaceQuotaHelperForTests.ONE_MEGABYTE;
 213     helper.writeData(tn, initialSize);
 214     admin.flush(tn);
 215
 216     LOG.info("Waiting until namespace size reflects written data");
 217     // Wait until that data is seen by the master
 218     TEST_UTIL.waitFor(30 * 1000, 500, new SpaceQuotaSnapshotPredicate(conn, ns) {
 219       @Override boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception {
 220         return snapshot.getUsage() >= initialSize;
 221       }
 222     });
 223
 224     // Make sure we see the "final" new size for the table, not some intermediate
 225     waitForStableQuotaSize(conn, null, ns);
 226
 227     // The actual size on disk after we wrote our data the first time
 228     final long actualInitialSize = conn.getAdmin().getCurrentSpaceQuotaSnapshot(ns).getUsage();
 229     LOG.info("Initial table size was " + actualInitialSize);
 230
 231     LOG.info("Snapshot the table");
 232     final String snapshot1 = tn.getQualifierAsString() + "_snapshot1";
 233     admin.snapshot(snapshot1, tn);
 234
 235     // Write the same data again, then flush+compact. This should make sure that
 236     // the snapshot is referencing files that the table no longer references.
 237     LOG.info("Write more data");
 238     helper.writeData(tn, initialSize);
 239     LOG.info("Flush the table");
 240     admin.flush(tn);
 241     LOG.info("Synchronously compacting the table");
 242     TEST_UTIL.compact(tn, true);
 243
 244     final long upperBound = initialSize + FUDGE_FOR_TABLE_SIZE;
 245     final long lowerBound = initialSize - FUDGE_FOR_TABLE_SIZE;
 246
 247     LOG.info("Waiting for the region reports to reflect the correct size, between ("
 248         + lowerBound + ", " + upperBound + ")");
 249     TEST_UTIL.waitFor(30 * 1000, 500, new Predicate<Exception>() {
 250       @Override
 251       public boolean evaluate() throws Exception {
 252         Map<TableName, Long> sizes = conn.getAdmin().getSpaceQuotaTableSizes();
 253         LOG.debug("Master observed table sizes from region size reports: " + sizes);
 254         Long size = sizes.get(tn);
 255         if (null == size) {
 256           return false;
 257         }
 258         return size < upperBound && size > lowerBound;
 259       }
 260     });
 261
 262     // Make sure we see the "final" new size for the table, not some intermediate
 263     waitForStableRegionSizeReport(conn, tn);
 264     final long finalSize = getRegionSizeReportForTable(conn, tn);
 265     assertNotNull("Did not expect to see a null size", finalSize);
 266     LOG.info("Final observed size of table: " + finalSize);
 267
 268     // Make sure the QuotaObserverChore has time to reflect the new region size reports
 269     // (we saw above). The usage of the table should *not* decrease when we check it below,
 270     // though, because the snapshot on our table will cause the table to "retain" the size.
 271     TEST_UTIL.waitFor(20 * 1000, 500, new SpaceQuotaSnapshotPredicate(conn, ns) {
 272       @Override
 273       public boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception {
 274         return snapshot.getUsage() >= finalSize;
 275       }
 276     });
 277
 278     // The final usage should be the sum of the initial size (referenced by the snapshot) and the
 279     // new size we just wrote above.
 280     long expectedFinalSize = actualInitialSize + finalSize;
 281     LOG.info(
 282         "Expecting namespace usage to be " + actualInitialSize + " + " + finalSize
 283         + " = " + expectedFinalSize);
 284     // The size of the table (WRT quotas) should now be approximately double what it was previously
 285     TEST_UTIL.waitFor(30 * 1000, 1000, new SpaceQuotaSnapshotPredicate(conn, ns) {
 286       @Override boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception {
 287         LOG.debug("Checking for " + expectedFinalSize + " == " + snapshot.getUsage());
 288         return expectedFinalSize == snapshot.getUsage();
 289       }
 290     });
 291
 292     Map<String,Long> snapshotSizes = QuotaTableUtil.getObservedSnapshotSizes(conn);
 293     Long size = snapshotSizes.get(snapshot1);
 294     assertNotNull("Did not observe the size of the snapshot", size);
 295     assertEquals(
 296         "The recorded size of the HBase snapshot was not the size we expected", actualInitialSize,
 297         size.longValue());
 298   }
 299
 300   @Test
 301   public void testTablesWithSnapshots() throws Exception {
 302     final Connection conn = TEST_UTIL.getConnection();
 303     final SpaceViolationPolicy policy = SpaceViolationPolicy.NO_INSERTS;
 304     final TableName tn = helper.createTableWithRegions(10);
 305
 306     // 3MB limit on the table
 307     final long tableLimit = 3L * SpaceQuotaHelperForTests.ONE_MEGABYTE;
 308     TEST_UTIL.getAdmin().setQuota(QuotaSettingsFactory.limitTableSpace(tn, tableLimit, policy));
 309
 310     LOG.info("Writing first data set");
 311     // Write more data than should be allowed and flush it to disk
 312     helper.writeData(tn, 1L * SpaceQuotaHelperForTests.ONE_MEGABYTE, "q1");
 313
 314     LOG.info("Creating snapshot");
 315     TEST_UTIL.getAdmin().snapshot(tn.toString() + "snap1", tn, SnapshotType.FLUSH);
 316
 317     LOG.info("Writing second data set");
 318     // Write some more data
 319     helper.writeData(tn, 1L * SpaceQuotaHelperForTests.ONE_MEGABYTE, "q2");
 320
 321     LOG.info("Flushing and major compacting table");
 322     // Compact the table to force the snapshot to own all of its files
 323     TEST_UTIL.getAdmin().flush(tn);
 324     TEST_UTIL.compact(tn, true);
 325
 326     LOG.info("Checking for quota violation");
 327     // Wait to observe the quota moving into violation
 328     TEST_UTIL.waitFor(60_000, 1_000, new Predicate<Exception>() {
 329       @Override
 330       public boolean evaluate() throws Exception {
 331         Scan s = QuotaTableUtil.makeQuotaSnapshotScanForTable(tn);
 332         try (Table t = conn.getTable(QuotaTableUtil.QUOTA_TABLE_NAME)) {
 333           ResultScanner rs = t.getScanner(s);
 334           try {
 335             Result r = Iterables.getOnlyElement(rs);
 336             CellScanner cs = r.cellScanner();
 337             assertTrue(cs.advance());
 338             Cell c = cs.current();
 339             SpaceQuotaSnapshot snapshot = SpaceQuotaSnapshot.toSpaceQuotaSnapshot(
 340                 QuotaProtos.SpaceQuotaSnapshot.parseFrom(
 341                   UnsafeByteOperations.unsafeWrap(
 342                       c.getValueArray(), c.getValueOffset(), c.getValueLength())));
 343             LOG.info(
 344                 snapshot.getUsage() + "/" + snapshot.getLimit() + " " + snapshot.getQuotaStatus());
 345             // We expect to see the table move to violation
 346             return snapshot.getQuotaStatus().isInViolation();
 347           } finally {
 348             if (null != rs) {
 349               rs.close();
 350             }
 351           }
 352         }
 353       }
 354     });
 355   }
 356
 357   @Test
 358   public void testRematerializedTablesDoNoInheritSpace() throws Exception {
 359     TableName tn = helper.createTableWithRegions(1);
 360     TableName tn2 = helper.getNextTableName();
 361     LOG.info("Writing data");
 362     // Set a quota on both tables
 363     QuotaSettings settings = QuotaSettingsFactory.limitTableSpace(
 364         tn, SpaceQuotaHelperForTests.ONE_GIGABYTE, SpaceViolationPolicy.NO_INSERTS);
 365     admin.setQuota(settings);
 366     QuotaSettings settings2 = QuotaSettingsFactory.limitTableSpace(
 367         tn2, SpaceQuotaHelperForTests.ONE_GIGABYTE, SpaceViolationPolicy.NO_INSERTS);
 368     admin.setQuota(settings2);
 369     // Write some data
 370     final long initialSize = 2L * SpaceQuotaHelperForTests.ONE_MEGABYTE;
 371     helper.writeData(tn, initialSize);
 372
 373     LOG.info("Waiting until table size reflects written data");
 374     // Wait until that data is seen by the master
 375     TEST_UTIL.waitFor(30 * 1000, 500, new SpaceQuotaSnapshotPredicate(conn, tn) {
 376       @Override boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception {
 377         return snapshot.getUsage() >= initialSize;
 378       }
 379     });
 380
 381     // Make sure we see the final quota usage size
 382     waitForStableQuotaSize(conn, tn, null);
 383
 384     // The actual size on disk after we wrote our data the first time
 385     final long actualInitialSize = conn.getAdmin().getCurrentSpaceQuotaSnapshot(tn).getUsage();
 386     LOG.info("Initial table size was " + actualInitialSize);
 387
 388     LOG.info("Snapshot the table");
 389     final String snapshot1 = tn.toString() + "_snapshot1";
 390     admin.snapshot(snapshot1, tn);
 391
 392     admin.cloneSnapshot(snapshot1, tn2);
 393
 394     // Write some more data to the first table
 395     helper.writeData(tn, initialSize, "q2");
 396     admin.flush(tn);
 397
 398     // Watch the usage of the first table with some more data to know when the new
 399     // region size reports were sent to the master
 400     TEST_UTIL.waitFor(30_000, 1_000, new SpaceQuotaSnapshotPredicate(conn, tn) {
 401       @Override
 402       boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception {
 403         return snapshot.getUsage() >= actualInitialSize * 2;
 404       }
 405     });
 406
 407     // We know that reports were sent by our RS, verify that they take up zero size.
 408     SpaceQuotaSnapshot snapshot =
 409       (SpaceQuotaSnapshot) conn.getAdmin().getCurrentSpaceQuotaSnapshot(tn2);
 410     assertNotNull(snapshot);
 411     assertEquals(0, snapshot.getUsage());
 412
 413     // Compact the cloned table to force it to own its own files.
 414     TEST_UTIL.compact(tn2, true);
 415     // After the table is compacted, it should have its own files and be the same size as originally
 416     // But The compaction result file has an additional compaction event tracker
 417     TEST_UTIL.waitFor(30_000, 1_000, new SpaceQuotaSnapshotPredicate(conn, tn2) {
 418       @Override
 419       boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception {
 420         return snapshot.getUsage() >= actualInitialSize;
 421       }
 422     });
 423   }
 424
 425   void waitForStableQuotaSize(Connection conn, TableName tn, String ns) throws Exception {
 426     // For some stability in the value before proceeding
 427     // Helps make sure that we got the actual last value, not some inbetween
 428     AtomicLong lastValue = new AtomicLong(-1);
 429     AtomicInteger counter = new AtomicInteger(0);
 430     TEST_UTIL.waitFor(15_000, 500, new SpaceQuotaSnapshotPredicate(conn, tn, ns) {
 431       @Override boolean evaluate(SpaceQuotaSnapshot snapshot) throws Exception {
 432         LOG.debug("Last observed size=" + lastValue.get());
 433         if (snapshot.getUsage() == lastValue.get()) {
 434           int numMatches = counter.incrementAndGet();
 435           if (numMatches >= 5) {
 436             return true;
 437           }
 438           // Not yet..
 439           return false;
 440         }
 441         counter.set(0);
 442         lastValue.set(snapshot.getUsage());
 443         return false;
 444       }
 445     });
 446   }
 447
 448   long getRegionSizeReportForTable(Connection conn, TableName tn) throws IOException {
 449     Map<TableName, Long> sizes = conn.getAdmin().getSpaceQuotaTableSizes();
 450     Long value = sizes.get(tn);
 451     if (null == value) {
 452       return 0L;
 453     }
 454     return value.longValue();
 455   }
 456
 457   void waitForStableRegionSizeReport(Connection conn, TableName tn) throws Exception {
 458     // For some stability in the value before proceeding
 459     // Helps make sure that we got the actual last value, not some inbetween
 460     AtomicLong lastValue = new AtomicLong(-1);
 461     AtomicInteger counter = new AtomicInteger(0);
 462     TEST_UTIL.waitFor(15_000, 500, new Predicate<Exception>() {
 463       @Override public boolean evaluate() throws Exception {
 464         LOG.debug("Last observed size=" + lastValue.get());
 465         long actual = getRegionSizeReportForTable(conn, tn);
 466         if (actual == lastValue.get()) {
 467           int numMatches = counter.incrementAndGet();
 468           if (numMatches >= 5) {
 469             return true;
 470           }
 471           // Not yet..
 472           return false;
 473         }
 474         counter.set(0);
 475         lastValue.set(actual);
 476         return false;
 477       }
 478     });
 479   }
 480 }