Make HashTable entries use Tag instead of a full hash

`VacantEntry` now stores a `Tag` instead of a full `hash: u64`. This
means that `OccupiedEntry` doesn't need to store anything, because it
can get that tag from the control byte for `remove -> VacantEntry`.
The `get_bucket_entry` method doesn't need a hash argument either.

Also, since `OccupiedEntry` is now smaller, `enum Entry` will be the
same size as `VacantEntry` by using a niche for the discriminant.
(Although this is not _guaranteed_ by the compiler.)

Author: cuviper

src/raw/mod.rs

@@ -838,6 +838,19 @@ impl<T, A: Allocator> RawTable<T, A> {
         (item.read(), self.bucket_index(&item))
     }
 
+    /// Removes an element from the table, returning it.
+    ///
+    /// This also returns an index to the newly free bucket
+    /// and the former `Tag` for that bucket.
+    #[cfg_attr(feature = "inline-more", inline)]
+    #[allow(clippy::needless_pass_by_value)]
+    pub(crate) unsafe fn remove_tagged(&mut self, item: Bucket<T>) -> (T, usize, Tag) {
+        let index = self.bucket_index(&item);
+        let tag = *self.table.ctrl(index);
+        self.table.erase(index);
+        (item.read(), index, tag)
+    }
+
     /// Finds and removes an element from the table, returning it.
     #[cfg_attr(feature = "inline-more", inline)]
     pub fn remove_entry(&mut self, hash: u64, eq: impl FnMut(&T) -> bool) -> Option<T> {
@@ -1172,8 +1185,8 @@ impl<T, A: Allocator> RawTable<T, A> {
         }
     }
 
-    /// Inserts a new element into the table at the given index, and returns its
-    /// raw bucket.
+    /// Inserts a new element into the table at the given index with the given hash,
+    /// and returns its raw bucket.
     ///
     /// # Safety
     ///
@@ -1182,8 +1195,26 @@ impl<T, A: Allocator> RawTable<T, A> {
     /// occurred since that call.
     #[inline]
     pub unsafe fn insert_at_index(&mut self, hash: u64, index: usize, value: T) -> Bucket<T> {
+        self.insert_tagged_at_index(Tag::full(hash), index, value)
+    }
+
+    /// Inserts a new element into the table at the given index with the given tag,
+    /// and returns its raw bucket.
+    ///
+    /// # Safety
+    ///
+    /// `index` must point to a slot previously returned by
+    /// `find_or_find_insert_index`, and no mutation of the table must have
+    /// occurred since that call.
+    #[inline]
+    pub(crate) unsafe fn insert_tagged_at_index(
+        &mut self,
+        tag: Tag,
+        index: usize,
+        value: T,
+    ) -> Bucket<T> {
         let old_ctrl = *self.table.ctrl(index);
-        self.table.record_item_insert_at(index, old_ctrl, hash);
+        self.table.record_item_insert_at(index, old_ctrl, tag);
 
         let bucket = self.bucket(index);
         bucket.write(value);
@@ -1258,11 +1289,11 @@ impl<T, A: Allocator> RawTable<T, A> {
     }
 
     /// Returns a pointer to an element in the table, but only after verifying that
-    /// the index is in-bounds and that its control byte matches the given hash.
+    /// the index is in-bounds and the bucket is occupied.
     #[inline]
-    pub fn checked_bucket(&self, hash: u64, index: usize) -> Option<Bucket<T>> {
+    pub fn checked_bucket(&self, index: usize) -> Option<Bucket<T>> {
         unsafe {
-            if index < self.buckets() && *self.table.ctrl(index) == Tag::full(hash) {
+            if index < self.buckets() && self.is_bucket_full(index) {
                 Some(self.bucket(index))
             } else {
                 None
@@ -2442,9 +2473,9 @@ impl RawTableInner {
     }
 
     #[inline]
-    unsafe fn record_item_insert_at(&mut self, index: usize, old_ctrl: Tag, hash: u64) {
+    unsafe fn record_item_insert_at(&mut self, index: usize, old_ctrl: Tag, new_ctrl: Tag) {
         self.growth_left -= usize::from(old_ctrl.special_is_empty());
-        self.set_ctrl_hash(index, hash);
+        self.set_ctrl(index, new_ctrl);
         self.items += 1;
     }
 

src/table.rs

@@ -1,6 +1,7 @@
 use core::{fmt, iter::FusedIterator, marker::PhantomData};
 
 use crate::{
+    control::Tag,
     raw::{
         Allocator, Bucket, Global, RawDrain, RawExtractIf, RawIntoIter, RawIter, RawIterHash,
         RawTable,
@@ -303,7 +304,6 @@ where
     ) -> Result<OccupiedEntry<'_, T, A>, AbsentEntry<'_, T, A>> {
         match self.raw.find(hash, eq) {
             Some(bucket) => Ok(OccupiedEntry {
-                hash,
                 bucket,
                 table: self,
             }),
@@ -413,24 +413,19 @@ where
     ) -> Entry<'_, T, A> {
         match self.raw.find_or_find_insert_index(hash, eq, hasher) {
             Ok(bucket) => Entry::Occupied(OccupiedEntry {
-                hash,
                 bucket,
                 table: self,
             }),
             Err(insert_index) => Entry::Vacant(VacantEntry {
-                hash,
+                tag: Tag::full(hash),
                 index: insert_index,
                 table: self,
             }),
         }
     }
 
-    /// Returns an `OccupiedEntry` for a bucket index in the table with the given hash,
-    /// or `None` if the index is out of bounds or if its hash doesn't match.
-    ///
-    /// However, note that the hash is only compared for the few bits that are directly stored in
-    /// the table, and even in full this could not guarantee equality. Use [`OccupiedEntry::get`]
-    /// if you need to further validate a match.
+    /// Returns an `OccupiedEntry` for the given bucket index in the table,
+    /// or `None` if it is unoccupied or out of bounds.
     ///
     /// # Examples
     ///
@@ -447,29 +442,25 @@ where
     /// table.insert_unique(hasher(&2), (2, 'b'), |val| hasher(&val.0));
     /// table.insert_unique(hasher(&3), (3, 'c'), |val| hasher(&val.0));
     ///
-    /// let hash = hasher(&2);
-    /// let index = table.find_bucket_index(hash, |val| val.0 == 2).unwrap();
+    /// let index = table.find_bucket_index(hasher(&2), |val| val.0 == 2).unwrap();
     ///
-    /// let bad_hash = !hash;
-    /// assert!(table.get_bucket_entry(bad_hash, index).is_none());
-    /// assert!(table.get_bucket_entry(hash, usize::MAX).is_none());
+    /// assert!(table.get_bucket_entry(usize::MAX).is_none());
     ///
-    /// let occupied_entry = table.get_bucket_entry(hash, index).unwrap();
+    /// let occupied_entry = table.get_bucket_entry(index).unwrap();
     /// assert_eq!(occupied_entry.get(), &(2, 'b'));
     /// assert_eq!(occupied_entry.remove().0, (2, 'b'));
     ///
-    /// assert!(table.find(hash, |val| val.0 == 2).is_none());
+    /// assert!(table.find(hasher(&2), |val| val.0 == 2).is_none());
     /// # }
     /// # fn main() {
     /// #     #[cfg(feature = "nightly")]
     /// #     test()
     /// # }
     /// ```
     #[inline]
-    pub fn get_bucket_entry(&mut self, hash: u64, index: usize) -> Option<OccupiedEntry<'_, T, A>> {
+    pub fn get_bucket_entry(&mut self, index: usize) -> Option<OccupiedEntry<'_, T, A>> {
         Some(OccupiedEntry {
-            hash,
-            bucket: self.raw.checked_bucket(hash, index)?,
+            bucket: self.raw.checked_bucket(index)?,
             table: self,
         })
     }
@@ -573,7 +564,6 @@ where
     ) -> OccupiedEntry<'_, T, A> {
         let bucket = self.raw.insert(hash, value, hasher);
         OccupiedEntry {
-            hash,
             bucket,
             table: self,
         }
@@ -1771,7 +1761,6 @@ pub struct OccupiedEntry<'a, T, A = Global>
 where
     A: Allocator,
 {
-    hash: u64,
     bucket: Bucket<T>,
     table: &'a mut HashTable<T, A>,
 }
@@ -1840,11 +1829,11 @@ where
     /// ```
     #[cfg_attr(feature = "inline-more", inline)]
     pub fn remove(self) -> (T, VacantEntry<'a, T, A>) {
-        let (val, index) = unsafe { self.table.raw.remove(self.bucket) };
+        let (val, index, tag) = unsafe { self.table.raw.remove_tagged(self.bucket) };
         (
             val,
             VacantEntry {
-                hash: self.hash,
+                tag,
                 index,
                 table: self.table,
             },
@@ -2083,7 +2072,7 @@ pub struct VacantEntry<'a, T, A = Global>
 where
     A: Allocator,
 {
-    hash: u64,
+    tag: Tag,
     index: usize,
     table: &'a mut HashTable<T, A>,
 }
@@ -2131,9 +2120,12 @@ where
     /// ```
     #[inline]
     pub fn insert(self, value: T) -> OccupiedEntry<'a, T, A> {
-        let bucket = unsafe { self.table.raw.insert_at_index(self.hash, self.index, value) };
+        let bucket = unsafe {
+            self.table
+                .raw
+                .insert_tagged_at_index(self.tag, self.index, value)
+        };
         OccupiedEntry {
-            hash: self.hash,
             bucket,
             table: self.table,
         }