diff --git a/src/Middleware/OutputCaching/src/OutputCacheAttribute.cs b/src/Middleware/OutputCaching/src/OutputCacheAttribute.cs
index 1409f20f5bdc..7194b63948d2 100644
--- a/src/Middleware/OutputCaching/src/OutputCacheAttribute.cs
+++ b/src/Middleware/OutputCaching/src/OutputCacheAttribute.cs
@@ -66,12 +66,18 @@ internal IOutputCachePolicy BuildPolicy()
             return _builtPolicy;
         }
 
-        var builder = new OutputCachePolicyBuilder();
+        OutputCachePolicyBuilder builder;
 
         if (PolicyName != null)
         {
+            // Don't add the default policy if a named one is used as it could already contain it
+            builder = new OutputCachePolicyBuilder(excludeDefaultPolicy: true);
             builder.AddPolicy(new NamedPolicy(PolicyName));
         }
+        else
+        {
+            builder = new();
+        }
 
         if (_noCache != null && _noCache.Value)
         {
diff --git a/src/Middleware/OutputCaching/test/OutputCacheAttributeTests.cs b/src/Middleware/OutputCaching/test/OutputCacheAttributeTests.cs
index cd43d2d58a34..26c4c5bd49b3 100644
--- a/src/Middleware/OutputCaching/test/OutputCacheAttributeTests.cs
+++ b/src/Middleware/OutputCaching/test/OutputCacheAttributeTests.cs
@@ -48,7 +48,7 @@ public async Task Attribute_CreatesNamedPolicy()
         var options = new OutputCacheOptions();
         options.AddPolicy("MyPolicy", b => b.Expire(TimeSpan.FromSeconds(42)));
 
-        var context = TestUtils.CreateTestContext(options: options);
+        var context = TestUtils.CreateUninitializedContext(options: options);
 
         var attribute = OutputCacheMethods.GetAttribute(nameof(OutputCacheMethods.PolicyName));
         await attribute.BuildPolicy().CacheRequestAsync(context, cancellation: default);
@@ -57,6 +57,20 @@ public async Task Attribute_CreatesNamedPolicy()
         Assert.Equal(42, context.ResponseExpirationTimeSpan?.TotalSeconds);
     }
 
+    [Fact]
+    public async Task Attribute_NamedPolicyDoesNotInjectDefaultPolicy()
+    {
+        var options = new OutputCacheOptions();
+        options.AddPolicy("MyPolicy", b => b.With(x => false).Cache());
+
+        var context = TestUtils.CreateUninitializedContext(options: options);
+
+        var attribute = OutputCacheMethods.GetAttribute(nameof(OutputCacheMethods.PolicyName));
+        await attribute.BuildPolicy().CacheRequestAsync(context, cancellation: default);
+
+        Assert.False(context.EnableOutputCaching);
+    }
+
     [Fact]
     public async Task Attribute_CreatesVaryByHeaderPolicy()
     {