assert,util: fix deep comparison for sets and maps with mixed types

When comparing a Set or a Map and they contain primitives as keys
as well as objects, the primitives would always be skipped. That
is not correct, since that may only be skipped in case all keys
are objects.

Fixes: #61386

Author: BridgeAR

lib/internal/util/comparisons.js

@@ -669,8 +669,10 @@ function setObjectEquiv(array, a, b, mode, memo) {
         if (b.has(val1)) {
           continue;
         }
-      } else if (mode !== kLoose || b.has(val1)) {
+      } else if (b.has(val1)) {
         continue;
+      } else if (mode !== kLoose) {
+        return false;
       }
     }
 
@@ -816,11 +818,16 @@ function mapObjectEquiv(array, a, b, mode, memo) {
   const extraChecks = mode === kLoose || array.length !== a.size;
 
   for (const { 0: key1, 1: item1 } of a) {
-    if (extraChecks &&
-        (typeof key1 !== 'object' || key1 === null) &&
-        (mode !== kLoose ||
-          (b.has(key1) && innerDeepEqual(item1, b.get(key1), mode, memo)))) { // Mixed mode
-      continue;
+    if (extraChecks && (typeof key1 !== 'object' || key1 === null)) {
+      if (mode !== kLoose) {
+        if (!b.has(key1)) {
+          return false;
+        }
+        continue;
+      }
+      if (b.has(key1) && innerDeepEqual(item1, b.get(key1), mode, memo)) { // Mixed mode
+        continue;
+      }
     }
 
     let innerStart = start;

test/parallel/test-assert-deep.js

@@ -634,6 +634,21 @@ test('Handle sparse arrays', () => {
   assertNotDeepOrStrict(a, b, AssertionError, { partial: 'pass' });
 });
 
+test('Handle sets and maps with mixed keys', () => {
+  // https://github.com/nodejs/node/issues/61386
+  const aSet = new Set([0, new Set([1, 2, 3]), new Set([4, 5, 6])]);
+  const bSet = new Set([
+    0,
+    new Set([1, new Set([2, 3]), new Set([20, 30])]),
+    new Set([4, new Set([5, 6]), new Set([50, 60])]),
+  ]);
+  assertNotDeepOrStrict(aSet, bSet);
+
+  const aMap = new Map([[0, 'zero'], [1, 'one'], [new Set([1, 2, 3]), 'A']]);
+  const bMap = new Map([[0, 'zero'], [new Set([1, 2, 3]), 'A'], [new Set([9]), 'B']]);
+  assertNotDeepOrStrict(aMap, bMap);
+});
+
 test('Handle different error messages', () => {
   const err1 = new Error('foo1');
   assertNotDeepOrStrict(err1, new Error('foo2'), assert.AssertionError);