diff --git a/src/planner/binder/expression/bind_macro_expression.cpp b/src/planner/binder/expression/bind_macro_expression.cpp
index d6f48cc12667..c83a29d79262 100644
--- a/src/planner/binder/expression/bind_macro_expression.cpp
+++ b/src/planner/binder/expression/bind_macro_expression.cpp
@@ -1,4 +1,5 @@
 #include "duckdb/catalog/catalog_entry/scalar_macro_catalog_entry.hpp"
+#include "duckdb/common/reference_map.hpp"
 #include "duckdb/common/string_util.hpp"
 #include "duckdb/function/scalar_macro_function.hpp"
 #include "duckdb/parser/expression/function_expression.hpp"
@@ -44,7 +45,7 @@ void ExpressionBinder::ReplaceMacroParametersRecursive(unique_ptr<ParsedExpressi
 }
 
 static void DetectInfiniteMacroRecursion(ClientContext &context, unique_ptr<ParsedExpression> &expr,
-                                         unordered_set<CatalogEntry *> &expanded_macros) {
+                                         reference_set_t<CatalogEntry> &expanded_macros) {
 	optional_ptr<ScalarMacroCatalogEntry> recursive_macro;
 	switch (expr->GetExpressionClass()) {
 	case ExpressionClass::FUNCTION: {
@@ -52,7 +53,7 @@ static void DetectInfiniteMacroRecursion(ClientContext &context, unique_ptr<Pars
 		auto function = Catalog::GetEntry(context, CatalogType::SCALAR_FUNCTION_ENTRY, func.catalog, func.schema,
 		                                  func.function_name, OnEntryNotFound::RETURN_NULL);
 		if (function && function->type == CatalogType::MACRO_ENTRY) {
-			if (expanded_macros.find(function.get()) != expanded_macros.end()) {
+			if (expanded_macros.find(*function) != expanded_macros.end()) {
 				throw BinderException("Infinite recursion detected in macro \"%s\"", func.function_name);
 			} else {
 				recursive_macro = &function->Cast<ScalarMacroCatalogEntry>();
@@ -75,11 +76,11 @@ static void DetectInfiniteMacroRecursion(ClientContext &context, unique_ptr<Pars
 	if (recursive_macro) {
 		auto &macro_def = recursive_macro->function->Cast<ScalarMacroFunction>();
 		auto rec_expr = macro_def.expression->Copy();
-		expanded_macros.insert(recursive_macro.get());
+		expanded_macros.insert(*recursive_macro);
 		ParsedExpressionIterator::EnumerateChildren(*rec_expr, [&](unique_ptr<ParsedExpression> &child) {
 			DetectInfiniteMacroRecursion(context, child, expanded_macros);
 		});
-		expanded_macros.erase(recursive_macro.get());
+		expanded_macros.erase(*recursive_macro);
 	} else {
 		ParsedExpressionIterator::EnumerateChildren(*expr, [&](unique_ptr<ParsedExpression> &child) {
 			DetectInfiniteMacroRecursion(context, child, expanded_macros);
@@ -126,8 +127,8 @@ BindResult ExpressionBinder::BindMacro(FunctionExpression &function, ScalarMacro
 	expr = macro_def.expression->Copy();
 
 	// detect infinite recursion
-	unordered_set<CatalogEntry *> expanded_macros;
-	expanded_macros.insert(&macro_func);
+	reference_set_t<CatalogEntry> expanded_macros;
+	expanded_macros.insert(macro_func);
 	DetectInfiniteMacroRecursion(context, expr, expanded_macros);
 
 	// now replace the parameters