HBASE-20821 Space Quota: Re-creating a dropped namespace and contained table inherits previously set space quota settings

hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/MasterQuotasObserver.java

@@ -65,6 +65,7 @@ public void postDeleteTable(
     }
     final Connection conn = ctx.getEnvironment().getConnection();
     Quotas quotas = QuotaUtil.getTableQuota(conn, tableName);
+    Quotas quotasAtNamespace = QuotaUtil.getNamespaceQuota(conn, tableName.getNamespaceAsString());
     if (quotas != null){
       if (quotas.hasSpace()){
         QuotaSettings settings = QuotaSettingsFactory.removeTableSpaceLimit(tableName);
@@ -78,6 +79,20 @@ public void postDeleteTable(
           admin.setQuota(settings);
         }
       }
+    } else if (quotasAtNamespace != null) {
+      // If quota present at namespace level remove the table entry from 'hbase:quota'
+      if (quotasAtNamespace.hasSpace()) {
+        QuotaSettings settings = QuotaSettingsFactory.removeTableSpaceLimit(tableName);
+        try (Admin admin = conn.getAdmin()) {
+          admin.setQuota(settings);
+        }
+      }
+      if (quotasAtNamespace.hasThrottle()) {
+        QuotaSettings settings = QuotaSettingsFactory.unthrottleTable(tableName);
+        try (Admin admin = conn.getAdmin()) {
+          admin.setQuota(settings);
+        }
+      }
     }
   }
 

hbase-server/src/main/java/org/apache/hadoop/hbase/quotas/QuotaUtil.java

@@ -30,6 +30,7 @@
 import org.apache.hadoop.hbase.HColumnDescriptor;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.NamespaceNotFoundException;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.TableNotDisabledException;
 import org.apache.hadoop.hbase.TableNotEnabledException;
@@ -117,6 +118,19 @@ public static void addNamespaceQuota(final Connection connection, final String n
 
   public static void deleteNamespaceQuota(final Connection connection, final String namespace)
       throws IOException {
+    // Before removing namespace quota , remove quota from the tables inside the namespace
+    // which does not have explicit space quotas defined on them.
+    TableName[] tableNames = QuotaUtil.listTableNamesByNamepsace(connection, namespace);
+    if (tableNames != null) {
+      for (int i = 0; i < tableNames.length; i++) {
+        QuotaSettings quotaSettings = getTableSpaceQuota(connection, tableNames[i]);
+        // if quotasettings is null, no space quota explicitly set on it.
+        // Remove entry from 'hbase:quota'
+        if (quotaSettings == null) {
+          deleteQuotas(connection, getTableRowKey(tableNames[i]));
+        }
+      }
+    }
     deleteQuotas(connection, getNamespaceRowKey(namespace));
   }
 
@@ -456,6 +470,46 @@ private static interface KeyFromRow<T> {
     double getFactor(T t);
   }
 
+  /**
+   * Method to return the space quotas defined on a given table.
+   *
+   * @param conn connection
+   * @param tn   tablename
+   * @return returns space quota settings defined on the table tn otherwise null.
+   * @throws IOException throws IOException
+   */
+  public static QuotaSettings getTableSpaceQuota(Connection conn, TableName tn) throws IOException {
+    try (QuotaRetriever scanner = QuotaRetriever
+        .open(conn.getConfiguration(), new QuotaFilter().setTableFilter(tn.getNameAsString()))) {
+      for (QuotaSettings setting : scanner) {
+        if (setting.getTableName().equals(tn) && setting.getQuotaType() == QuotaType.SPACE) {
+          return setting;
+        }
+      }
+      return null;
+    }
+  }
+
+  /**
+   * Retrieve list of tables for the given namespace.
+   *
+   * @param connection Connection
+   * @param namespace  name of the namespace
+   * @return list of tables present inside the namespace otherwise returns null.
+   * @throws IOException throws IOException
+   */
+  public static TableName[] listTableNamesByNamepsace(final Connection connection, String namespace)
+      throws IOException {
+    try {
+      TableName[] tableNames = connection.getAdmin().listTableNamesByNamespace(namespace);
+      return tableNames;
+    } catch (NamespaceNotFoundException ns) {
+      LOG.warn(
+          " Namespace " + namespace + "does not exist. Can't return tables inside the namespace");
+      return null;
+    }
+  }
+
   /* =========================================================================
    *  HTable helpers
    */

hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/SpaceQuotaHelperForTests.java

@@ -271,6 +271,17 @@ void setQuotaLimit(final TableName tn, SpaceViolationPolicy policy, long sizeInM
     LOG.debug("Quota limit set for table = {}, limit = {}", tn, sizeLimit);
   }
 
+  /**
+   * Sets the given quota (policy & limit) on the passed namespace.
+   */
+  void setQuotaLimitNamespace(final String namespace, SpaceViolationPolicy policy, long sizeInMBs)
+      throws Exception {
+    final long sizeLimit = sizeInMBs * SpaceQuotaHelperForTests.ONE_MEGABYTE;
+    QuotaSettings settings = QuotaSettingsFactory.limitNamespaceSpace(namespace, sizeLimit, policy);
+    testUtil.getAdmin().setQuota(settings);
+    LOG.debug("Quota limit set for namespace = {}, limit = {}", namespace, sizeLimit);
+  }
+
   /**
    * Removes the space quota from the given table
    */
@@ -280,6 +291,15 @@ void removeQuotaFromtable(final TableName tn) throws Exception {
     LOG.debug("Space quota settings removed from the table ", tn);
   }
 
+  /**
+   * Removes the space quota from the given namespace
+   */
+  void removeQuotaFromNamespace(final String namespace) throws Exception {
+    QuotaSettings removeQuota = QuotaSettingsFactory.removeNamespaceSpaceLimit(namespace);
+    testUtil.getAdmin().setQuota(removeQuota);
+    LOG.debug("Space quota settings removed from the namespace ", namespace);
+  }
+
   /**
    * Removes all quotas defined in the HBase quota table.
    */

hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestSpaceQuotasAtNamespaceLevel.java

@@ -0,0 +1,143 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.quotas;
+
+
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.NamespaceDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@Category(MediumTests.class)
+public class TestSpaceQuotasAtNamespaceLevel {
+
+  @ClassRule
+  public static final HBaseClassTestRule CLASS_RULE =
+      HBaseClassTestRule.forClass(TestSpaceQuotasAtNamespaceLevel.class);
+
+  private static final Logger LOG = LoggerFactory.getLogger(TestSpaceQuotasAtNamespaceLevel.class);
+  private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
+  @Rule
+  public TestName testName = new TestName();
+  private SpaceQuotaHelperForTests helper;
+
+  @BeforeClass
+  public static void setUp() throws Exception {
+    Configuration conf = TEST_UTIL.getConfiguration();
+    SpaceQuotaHelperForTests.updateConfigForQuotas(conf);
+    TEST_UTIL.startMiniCluster(1);
+  }
+
+  @AfterClass
+  public static void tearDown() throws Exception {
+    TEST_UTIL.shutdownMiniCluster();
+  }
+
+  @Before
+  public void removeAllQuotas() throws Exception {
+    helper = new SpaceQuotaHelperForTests(TEST_UTIL, testName, new AtomicLong(0));
+    helper.removeAllQuotas();
+  }
+
+  @After
+  public void removeQuotas() throws Exception {
+    helper.removeAllQuotas();
+  }
+
+  @Test
+  public void testSetNamespaceQuotaAndRemove() throws Exception {
+    NamespaceDescriptor nd = helper.createNamespace();
+    TableName table = helper.createTableInNamespace(nd);
+
+    // Set quota on namespace.
+    helper.setQuotaLimitNamespace(nd.getName(), SpaceViolationPolicy.NO_WRITES, 2L);
+
+    // Sufficient time for all the chores to run
+    Thread.sleep(5000);
+
+    // Get Current Snapshot from 'hbase:quota'
+    Map<TableName, SpaceQuotaSnapshot> snapshotMap =
+        QuotaTableUtil.getSnapshots(TEST_UTIL.getConnection());
+
+    // After setting quota on namespace, 'hbase:quota' should have some entries present.
+    Assert.assertEquals(1, snapshotMap.size());
+
+    helper.removeQuotaFromNamespace(nd.getName());
+
+    // Get Current Snapshot from 'hbase:quota'
+    snapshotMap = QuotaTableUtil.getSnapshots(TEST_UTIL.getConnection());
+
+    // After removing quota on namespace, 'hbase:quota' should not have any entry present.
+    Assert.assertEquals(0, snapshotMap.size());
+
+    // drop table and namespace.
+    TEST_UTIL.getAdmin().disableTable(table);
+    TEST_UTIL.getAdmin().deleteTable(table);
+    TEST_UTIL.getAdmin().deleteNamespace(nd.getName());
+  }
+
+  @Test
+  public void testDropTableInNamespaceQuota() throws Exception {
+    NamespaceDescriptor nd = helper.createNamespace();
+    TableName table = helper.createTableInNamespace(nd);
+
+    // Set quota on namespace.
+    helper.setQuotaLimitNamespace(nd.getName(), SpaceViolationPolicy.NO_WRITES, 2L);
+
+    // write some data.
+    helper.writeData(table,SpaceQuotaHelperForTests.ONE_KILOBYTE);
+
+    // Sufficient time for all the chores to run
+    Thread.sleep(5000);
+
+    // Get Current Snapshot from 'hbase:quota'
+    Map<TableName, SpaceQuotaSnapshot> snapshotMap =
+        QuotaTableUtil.getSnapshots(TEST_UTIL.getConnection());
+
+    // Table before drop should have entry in 'hbase:quota'
+    Assert.assertTrue(snapshotMap.containsKey(table));
+
+    TEST_UTIL.getAdmin().disableTable(table);
+    TEST_UTIL.getAdmin().deleteTable(table);
+
+    // Get Current Snapshot from 'hbase:quota'
+    snapshotMap = QuotaTableUtil.getSnapshots(TEST_UTIL.getConnection());
+
+    // Table after drop should not have entry in 'hbase:quota'
+    Assert.assertFalse(snapshotMap.containsKey(table));
+
+    //drop  Namepsace.
+    TEST_UTIL.getAdmin().deleteNamespace(nd.getName());
+  }
+}