diff --git a/src/Servers/Kestrel/Core/src/Internal/ConfigurationReader.cs b/src/Servers/Kestrel/Core/src/Internal/ConfigurationReader.cs
index f54f43ca7a..cf37f5ec73 100644
--- a/src/Servers/Kestrel/Core/src/Internal/ConfigurationReader.cs
+++ b/src/Servers/Kestrel/Core/src/Internal/ConfigurationReader.cs
@@ -19,19 +19,20 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal
 
         private readonly IConfiguration _configuration;
 
+        private IDictionary<string, CertificateConfig> _certificates;
+        private EndpointDefaults _endpointDefaults;
+        private IEnumerable<EndpointConfig> _endpoints;
+        private bool? _latin1RequestHeaders;
+
         public ConfigurationReader(IConfiguration configuration)
         {
             _configuration = configuration ?? throw new ArgumentNullException(nameof(configuration));
-            Certificates = ReadCertificates();
-            EndpointDefaults = ReadEndpointDefaults();
-            Endpoints = ReadEndpoints();
-            Latin1RequestHeaders = _configuration.GetValue<bool>(Latin1RequestHeadersKey);
         }
 
-        public IDictionary<string, CertificateConfig> Certificates { get; }
-        public EndpointDefaults EndpointDefaults { get; }
-        public IEnumerable<EndpointConfig> Endpoints { get; }
-        public bool Latin1RequestHeaders  { get; }
+        public IDictionary<string, CertificateConfig> Certificates => _certificates ??= ReadCertificates();
+        public EndpointDefaults EndpointDefaults => _endpointDefaults ??= ReadEndpointDefaults();
+        public IEnumerable<EndpointConfig> Endpoints => _endpoints ??= ReadEndpoints();
+        public bool Latin1RequestHeaders => _latin1RequestHeaders ??= _configuration.GetValue<bool>(Latin1RequestHeadersKey);
 
         private IDictionary<string, CertificateConfig> ReadCertificates()
         {
diff --git a/src/Servers/Kestrel/Core/src/KestrelConfigurationLoader.cs b/src/Servers/Kestrel/Core/src/KestrelConfigurationLoader.cs
index 518f908535..f68b324053 100644
--- a/src/Servers/Kestrel/Core/src/KestrelConfigurationLoader.cs
+++ b/src/Servers/Kestrel/Core/src/KestrelConfigurationLoader.cs
@@ -29,6 +29,8 @@ namespace Microsoft.AspNetCore.Server.Kestrel
             Options = options ?? throw new ArgumentNullException(nameof(options));
             Configuration = configuration ?? throw new ArgumentNullException(nameof(configuration));
             ReloadOnChange = reloadOnChange;
+
+            ConfigurationReader = new ConfigurationReader(configuration);
         }
 
         public KestrelServerOptions Options { get; }
diff --git a/src/Servers/Kestrel/Core/test/KestrelServerOptionsTests.cs b/src/Servers/Kestrel/Core/test/KestrelServerOptionsTests.cs
index e9d99c764a..f070266ec5 100644
--- a/src/Servers/Kestrel/Core/test/KestrelServerOptionsTests.cs
+++ b/src/Servers/Kestrel/Core/test/KestrelServerOptionsTests.cs
@@ -49,5 +49,16 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests
             });
             Assert.Equal(HttpProtocols.Http2, options.CodeBackedListenOptions[3].Protocols);
         }
+
+        [Fact]
+        public void CanCallListenAfterConfigure()
+        {
+            var options = new KestrelServerOptions();
+            options.Configure();
+
+            // This is a regression test to verify the Listen* methods don't throw a NullReferenceException if called after Configure().
+            // https://github.com/dotnet/aspnetcore/issues/21423
+            options.ListenLocalhost(5000);
+        }
     }
 }
diff --git a/src/Servers/Kestrel/Kestrel/test/ConfigurationReaderTests.cs b/src/Servers/Kestrel/Kestrel/test/ConfigurationReaderTests.cs
index 22a24014ce..7097c16553 100644
--- a/src/Servers/Kestrel/Kestrel/test/ConfigurationReaderTests.cs
+++ b/src/Servers/Kestrel/Kestrel/test/ConfigurationReaderTests.cs
@@ -97,7 +97,8 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Tests
             {
                 new KeyValuePair<string, string>("Endpoints:End1", ""),
             }).Build();
-            Assert.Throws<InvalidOperationException>(() => new ConfigurationReader(config));
+            var reader = new ConfigurationReader(config);
+            Assert.Throws<InvalidOperationException>(() => reader.Endpoints);
         }
 
         [Fact]
@@ -107,7 +108,8 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Tests
             {
                 new KeyValuePair<string, string>("Endpoints:End1:Url", ""),
             }).Build();
-            Assert.Throws<InvalidOperationException>(() => new ConfigurationReader(config));
+            var reader = new ConfigurationReader(config);
+            Assert.Throws<InvalidOperationException>(() => reader.Endpoints);
         }
 
         [Fact]