Revert "HBASE-27064 CME in TestRegionNormalizerWorkQueue (apache#4468)"

This reverts commit a93ea9e.

Author: wenwj0

hbase-server/src/main/java/org/apache/hadoop/hbase/master/normalizer/RegionNormalizerWorkQueue.java

@@ -25,7 +25,7 @@
 import java.util.Set;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.locks.Condition;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.concurrent.locks.ReentrantLock;
 import org.apache.yetus.audience.InterfaceAudience;
 
 /**
@@ -39,22 +39,62 @@
  * {@link BlockingQueue}.</li>
  * <li>Allows a producer to insert an item at the head of the queue, if desired.</li>
  * </ul>
+ * Assumes low-frequency and low-parallelism concurrent access, so protects state using a simplistic
+ * synchronization strategy.
  */
 @InterfaceAudience.Private
 class RegionNormalizerWorkQueue<E> {
 
   /** Underlying storage structure that gives us the Set behavior and FIFO retrieval policy. */
   private LinkedHashSet<E> delegate;
 
-  /** Lock for puts and takes **/
-  private final ReentrantReadWriteLock lock;
+  // the locking structure used here follows the example found in LinkedBlockingQueue. The
+  // difference is that our locks guard access to `delegate` rather than the head node.
+
+  /** Lock held by take, poll, etc */
+  private final ReentrantLock takeLock;
+
   /** Wait queue for waiting takes */
   private final Condition notEmpty;
 
+  /** Lock held by put, offer, etc */
+  private final ReentrantLock putLock;
+
   RegionNormalizerWorkQueue() {
     delegate = new LinkedHashSet<>();
-    lock = new ReentrantReadWriteLock();
-    notEmpty = lock.writeLock().newCondition();
+    takeLock = new ReentrantLock();
+    notEmpty = takeLock.newCondition();
+    putLock = new ReentrantLock();
+  }
+
+  /**
+   * Signals a waiting take. Called only from put/offer (which do not otherwise ordinarily lock
+   * takeLock.)
+   */
+  private void signalNotEmpty() {
+    final ReentrantLock takeLock = this.takeLock;
+    takeLock.lock();
+    try {
+      notEmpty.signal();
+    } finally {
+      takeLock.unlock();
+    }
+  }
+
+  /**
+   * Locks to prevent both puts and takes.
+   */
+  private void fullyLock() {
+    putLock.lock();
+    takeLock.lock();
+  }
+
+  /**
+   * Unlocks to allow both puts and takes.
+   */
+  private void fullyUnlock() {
+    takeLock.unlock();
+    putLock.unlock();
   }
 
   /**
@@ -65,14 +105,16 @@ public void put(E e) {
     if (e == null) {
       throw new NullPointerException();
     }
-    lock.writeLock().lock();
+
+    putLock.lock();
     try {
       delegate.add(e);
-      if (!delegate.isEmpty()) {
-        notEmpty.signal();
-      }
     } finally {
-      lock.writeLock().unlock();
+      putLock.unlock();
+    }
+
+    if (!delegate.isEmpty()) {
+      signalNotEmpty();
     }
   }
 
@@ -96,14 +138,16 @@ public void putAll(Collection<? extends E> c) {
     if (c == null) {
       throw new NullPointerException();
     }
-    lock.writeLock().lock();
+
+    putLock.lock();
     try {
       delegate.addAll(c);
-      if (!delegate.isEmpty()) {
-        notEmpty.signal();
-      }
     } finally {
-      lock.writeLock().unlock();
+      putLock.unlock();
+    }
+
+    if (!delegate.isEmpty()) {
+      signalNotEmpty();
     }
   }
 
@@ -115,17 +159,19 @@ public void putAllFirst(Collection<? extends E> c) {
     if (c == null) {
       throw new NullPointerException();
     }
-    lock.writeLock().lock();
+
+    fullyLock();
     try {
       final LinkedHashSet<E> copy = new LinkedHashSet<>(c.size() + delegate.size());
       copy.addAll(c);
       copy.addAll(delegate);
       delegate = copy;
-      if (!delegate.isEmpty()) {
-        notEmpty.signal();
-      }
     } finally {
-      lock.writeLock().unlock();
+      fullyUnlock();
+    }
+
+    if (!delegate.isEmpty()) {
+      signalNotEmpty();
     }
   }
 
@@ -137,13 +183,10 @@ public void putAllFirst(Collection<? extends E> c) {
    */
   public E take() throws InterruptedException {
     E x;
-    // Take a write lock. If the delegate's queue is empty we need it to await(), which will
-    // drop the lock, then reacquire it; or if the queue is not empty we will use an iterator
-    // to mutate the head.
-    lock.writeLock().lockInterruptibly();
+    takeLock.lockInterruptibly();
     try {
       while (delegate.isEmpty()) {
-        notEmpty.await(); // await drops the lock, then reacquires it
+        notEmpty.await();
       }
       final Iterator<E> iter = delegate.iterator();
       x = iter.next();
@@ -152,7 +195,7 @@ public E take() throws InterruptedException {
         notEmpty.signal();
       }
     } finally {
-      lock.writeLock().unlock();
+      takeLock.unlock();
     }
     return x;
   }
@@ -162,11 +205,11 @@ public E take() throws InterruptedException {
    * returns.
    */
   public void clear() {
-    lock.writeLock().lock();
+    putLock.lock();
     try {
       delegate.clear();
     } finally {
-      lock.writeLock().unlock();
+      putLock.unlock();
     }
   }
 
@@ -175,21 +218,21 @@ public void clear() {
    * @return the number of elements in this queue
    */
   public int size() {
-    lock.readLock().lock();
+    takeLock.lock();
     try {
       return delegate.size();
     } finally {
-      lock.readLock().unlock();
+      takeLock.unlock();
     }
   }
 
   @Override
   public String toString() {
-    lock.readLock().lock();
+    takeLock.lock();
     try {
       return delegate.toString();
     } finally {
-      lock.readLock().unlock();
+      takeLock.unlock();
     }
   }
 }