Use private instance of MemoryCache and impose size limit

src/Microsoft.AspNetCore.ResponseCaching/Internal/CacheEntry/CacheEntryHelpers .cs

@@ -0,0 +1,88 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using Microsoft.Extensions.Primitives;
+
+namespace Microsoft.AspNetCore.ResponseCaching.Internal
+{
+    internal static class CacheEntryHelpers
+    {
+
+        internal static long EstimateCachedResponseSize(CachedResponse cachedResponse)
+        {
+            if (cachedResponse == null)
+            {
+                return 0L;
+            }
+
+            checked
+            {
+                // StatusCode
+                long size = sizeof(int);
+
+                // Headers
+                if (cachedResponse.Headers != null)
+                {
+                    foreach (var item in cachedResponse.Headers)
+                    {
+                        size += item.Key.Length * sizeof(char) + EstimateStringValuesSize(item.Value);
+                    }
+                }
+
+                // Body
+                if (cachedResponse.Body != null)
+                {
+                    size += cachedResponse.Body.Length;
+                }
+
+                return size;
+            }
+        }
+
+        internal static long EstimateCachedVaryByRulesySize(CachedVaryByRules cachedVaryByRules)
+        {
+            if (cachedVaryByRules == null)
+            {
+                return 0L;
+            }
+
+            checked
+            {
+                var size = 0L;
+
+                // VaryByKeyPrefix
+                if (!string.IsNullOrEmpty(cachedVaryByRules.VaryByKeyPrefix))
+                {
+                    size = cachedVaryByRules.VaryByKeyPrefix.Length * sizeof(char);
+                }
+
+                // Headers
+                size += EstimateStringValuesSize(cachedVaryByRules.Headers);
+
+                // QueryKeys
+                size += EstimateStringValuesSize(cachedVaryByRules.QueryKeys);
+
+                return size;
+            }
+        }
+
+        internal static long EstimateStringValuesSize(StringValues stringValues)
+        {
+            checked
+            {
+                var size = 0L;
+
+                for (var i = 0; i < stringValues.Count; i++)
+                {
+                    var stringValue = stringValues[i];
+                    if (!string.IsNullOrEmpty(stringValue))
+                    {
+                        size += stringValues[i].Length * sizeof(char);
+                    }
+                }
+
+                return size;
+            }
+        }
+    }
+}

src/Microsoft.AspNetCore.ResponseCaching/Internal/MemoryResponseCache.cs

@@ -67,7 +67,8 @@ namespace Microsoft.AspNetCore.ResponseCaching.Internal
                     },
                     new MemoryCacheEntryOptions
                     {
-                        AbsoluteExpirationRelativeToNow = validFor
+                        AbsoluteExpirationRelativeToNow = validFor,
+                        Size = CacheEntryHelpers.EstimateCachedResponseSize(cachedResponse)
                     });
             }
             else
@@ -77,7 +78,8 @@ namespace Microsoft.AspNetCore.ResponseCaching.Internal
                     entry,
                     new MemoryCacheEntryOptions
                     {
-                        AbsoluteExpirationRelativeToNow = validFor
+                        AbsoluteExpirationRelativeToNow = validFor,
+                        Size = CacheEntryHelpers.EstimateCachedVaryByRulesySize(entry as CachedVaryByRules)
                     });
             }
         }

src/Microsoft.AspNetCore.ResponseCaching/ResponseCachingMiddleware.cs

@@ -7,6 +7,7 @@ using System.Threading.Tasks;
 using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Http.Features;
 using Microsoft.AspNetCore.ResponseCaching.Internal;
+using Microsoft.Extensions.Caching.Memory;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 using Microsoft.Extensions.Primitives;
@@ -26,6 +27,24 @@ namespace Microsoft.AspNetCore.ResponseCaching
         private readonly IResponseCachingKeyProvider _keyProvider;
 
         public ResponseCachingMiddleware(
+            RequestDelegate next,
+            IOptions<ResponseCachingOptions> options,
+            ILoggerFactory loggerFactory,
+            IResponseCachingPolicyProvider policyProvider,
+            IResponseCachingKeyProvider keyProvider)
+            : this(
+                next,
+                options,
+                loggerFactory,
+                policyProvider,
+                new MemoryResponseCache(new MemoryCache(new MemoryCacheOptions
+                {
+                    SizeLimit = options.Value.SizeLimit
+                })), keyProvider)
+        { }
+
+        // for testing
+        internal ResponseCachingMiddleware(
             RequestDelegate next,
             IOptions<ResponseCachingOptions> options,
             ILoggerFactory loggerFactory,

src/Microsoft.AspNetCore.ResponseCaching/ResponseCachingOptions.cs

@@ -8,6 +8,11 @@ namespace Microsoft.AspNetCore.ResponseCaching
 {
     public class ResponseCachingOptions
     {
+        /// <summary>
+        /// The size limit for the response cache middleware in bytes. The default is set to 100 MB.
+        /// </summary>
+        public long SizeLimit { get; set; } = 100 * 1024 * 1024;
+
         /// <summary>
         /// The largest cacheable size for the response body in bytes. The default is set to 64 MB.
         /// </summary>

src/Microsoft.AspNetCore.ResponseCaching/ResponseCachingServicesExtensions.cs

@@ -28,7 +28,6 @@ namespace Microsoft.Extensions.DependencyInjection
             services.AddMemoryCache();
             services.TryAdd(ServiceDescriptor.Singleton<IResponseCachingPolicyProvider, ResponseCachingPolicyProvider>());
             services.TryAdd(ServiceDescriptor.Singleton<IResponseCachingKeyProvider, ResponseCachingKeyProvider>());
-            services.TryAdd(ServiceDescriptor.Singleton<IResponseCache, MemoryResponseCache>());
 
             return services;
         }

test/Microsoft.AspNetCore.ResponseCaching.Tests/ResponseCachingMiddlewareTests.cs

@@ -7,6 +7,7 @@ using System.Threading.Tasks;
 using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Http.Features;
 using Microsoft.AspNetCore.ResponseCaching.Internal;
+using Microsoft.Extensions.Caching.Memory;
 using Microsoft.Extensions.Logging.Testing;
 using Microsoft.Extensions.Primitives;
 using Microsoft.Net.Http.Headers;
@@ -823,6 +824,38 @@ namespace Microsoft.AspNetCore.ResponseCaching.Tests
                 LoggedMessage.ResponseNotCached);
         }
 
+        [Fact]
+        public async Task FinalizeCacheBody_DoNotCache_IfSizeTooBig()
+        {
+            var sink = new TestSink();
+            var middleware = TestUtils.CreateTestMiddleware(
+                testSink: sink,
+                keyProvider: new TestResponseCachingKeyProvider("BaseKey"),
+                cache: new MemoryResponseCache(new MemoryCache(new MemoryCacheOptions
+                {
+                    SizeLimit = 100
+                })));
+            var context = TestUtils.CreateTestContext();
+
+            context.ShouldCacheResponse = true;
+            middleware.ShimResponseStream(context);
+
+            await context.HttpContext.Response.WriteAsync(new string('0', 101));
+
+            context.CachedResponse = new CachedResponse() { Headers = new HeaderDictionary() };
+            context.CachedResponseValidFor = TimeSpan.FromSeconds(10);
+
+            await middleware.FinalizeCacheBodyAsync(context);
+
+            // The response cached message will be logged but the adding of the entry will no-op
+            TestUtils.AssertLoggedMessages(
+                sink.Writes,
+                LoggedMessage.ResponseCached);
+
+            // The entry cannot be retrieved
+            Assert.False(await middleware.TryServeFromCacheAsync(context));
+        }
+
         [Fact]
         public void AddResponseCachingFeature_SecondInvocation_Throws()
         {