diff --git a/src/Microsoft.AspNetCore.Dispatcher.Abstractions/EndpointHandlerFactory.cs b/src/Microsoft.AspNetCore.Dispatcher.Abstractions/EndpointHandlerFactory.cs
deleted file mode 100644
index 54f5c2676b..0000000000
--- a/src/Microsoft.AspNetCore.Dispatcher.Abstractions/EndpointHandlerFactory.cs
+++ /dev/null
@@ -1,17 +0,0 @@
-// Copyright (c) .NET Foundation. All rights reserved.
-// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
-
-using System;
-using Microsoft.AspNetCore.Http;
-
-namespace Microsoft.AspNetCore.Dispatcher
-{
-    /// <summary>
-    /// A delegate which attempts to create a <see cref="Func{RequestDelegate, RequestDelegate}"/> for the selected <see cref="Endpoint"/>.
-    /// </summary>
-    /// <param name="endpoint">The <see cref="Endpoint"/> selected by the dispatcher.</param>
-    /// <returns>
-    /// A <see cref="Func{RequestDelegate, RequestDelegate}"/>  that invokes the operation represented by the <see cref="Endpoint"/>, or <c>null</c>.
-    /// </returns>
-    public delegate Func<RequestDelegate, RequestDelegate> EndpointHandlerFactory(Endpoint endpoint);
-}
diff --git a/src/Microsoft.AspNetCore.Dispatcher/CompositeHandlerFactory.cs b/src/Microsoft.AspNetCore.Dispatcher/CompositeHandlerFactory.cs
new file mode 100644
index 0000000000..ab7ae23b81
--- /dev/null
+++ b/src/Microsoft.AspNetCore.Dispatcher/CompositeHandlerFactory.cs
@@ -0,0 +1,44 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Microsoft.AspNetCore.Http;
+
+namespace Microsoft.AspNetCore.Dispatcher
+{
+    public class CompositeHandlerFactory : IHandlerFactory
+    {
+        private readonly IHandlerFactory[] _factories;
+
+        public CompositeHandlerFactory(IEnumerable<IHandlerFactory> factories)
+        {
+            if (factories == null)
+            {
+                throw new ArgumentNullException(nameof(factories));
+            }
+
+            _factories = factories.ToArray();
+        }
+
+        public Func<RequestDelegate, RequestDelegate> CreateHandler(Endpoint endpoint)
+        {
+            if (endpoint == null)
+            {
+                throw new ArgumentNullException(nameof(endpoint));
+            }
+
+            for (var i = 0; i < _factories.Length; i++)
+            {
+                var handler = _factories[i].CreateHandler(endpoint);
+                if (handler != null)
+                {
+                    return handler;
+                }
+            }
+
+            return null;
+        }
+    }
+}
diff --git a/src/Microsoft.AspNetCore.Dispatcher/DefaultDispatcherConfigureOptions.cs b/src/Microsoft.AspNetCore.Dispatcher/DefaultDispatcherConfigureOptions.cs
index f372a2812f..7b54aaf76d 100644
--- a/src/Microsoft.AspNetCore.Dispatcher/DefaultDispatcherConfigureOptions.cs
+++ b/src/Microsoft.AspNetCore.Dispatcher/DefaultDispatcherConfigureOptions.cs
@@ -12,13 +12,13 @@ namespace Microsoft.AspNetCore.Dispatcher
         private readonly IEnumerable<DispatcherDataSource> _dataSources;
         private readonly IDefaultMatcherFactory _dispatcherFactory;
         private readonly IEnumerable<EndpointSelector> _endpointSelectors;
-        private readonly IEnumerable<EndpointHandlerFactoryBase> _handlerFactories;
+        private readonly IEnumerable<IHandlerFactory> _handlerFactories;
 
         public DefaultDispatcherConfigureOptions(
             IDefaultMatcherFactory dispatcherFactory,
             IEnumerable<DispatcherDataSource> dataSources,
             IEnumerable<EndpointSelector> endpointSelectors,
-            IEnumerable<EndpointHandlerFactoryBase> handlerFactories)
+            IEnumerable<IHandlerFactory> handlerFactories)
         {
             _dispatcherFactory = dispatcherFactory;
             _dataSources = dataSources;
@@ -33,12 +33,15 @@ namespace Microsoft.AspNetCore.Dispatcher
                 throw new ArgumentNullException(nameof(options));
             }
 
-            options.Matchers.Add(_dispatcherFactory.CreateDispatcher(new CompositeDispatcherDataSource(_dataSources), _endpointSelectors));
-
-            foreach (var handlerFactory in _handlerFactories)
+            var matcher = _dispatcherFactory.CreateMatcher(new CompositeDispatcherDataSource(_dataSources), _endpointSelectors);
+            
+            options.Matchers.Add(new MatcherEntry()
             {
-                options.HandlerFactories.Add(handlerFactory.CreateHandler);
-            }
+                Matcher = matcher,
+                AddressProvider = matcher as IAddressCollectionProvider,
+                EndpointProvider = matcher as IEndpointCollectionProvider,
+                HandlerFactory = new CompositeHandlerFactory(_handlerFactories),
+            });
         }
     }
 }
diff --git a/src/Microsoft.AspNetCore.Dispatcher/DispatcherMiddleware.cs b/src/Microsoft.AspNetCore.Dispatcher/DispatcherMiddleware.cs
index 211061e088..c13ad2fefd 100644
--- a/src/Microsoft.AspNetCore.Dispatcher/DispatcherMiddleware.cs
+++ b/src/Microsoft.AspNetCore.Dispatcher/DispatcherMiddleware.cs
@@ -63,15 +63,10 @@ namespace Microsoft.AspNetCore.Dispatcher
                     feature.Endpoint = context.Endpoint;
                     feature.Values = context.Values;
 
-                    // Associate this with the DispatcherEntry, not global
-                    for (var i = 0; i < _options.HandlerFactories.Count; i++)
+                    feature.Handler = entry.HandlerFactory.CreateHandler(feature.Endpoint);
+                    if (feature.Handler == null)
                     {
-                        var middleware = _options.HandlerFactories[i](feature.Endpoint);
-                        if (middleware != null)
-                        {
-                            feature.Handler = middleware;
-                            break;
-                        }
+                        throw new InvalidOperationException("Couldn't create a handler, that's bad.");
                     }
 
                     break;
diff --git a/src/Microsoft.AspNetCore.Dispatcher/DispatcherOptions.cs b/src/Microsoft.AspNetCore.Dispatcher/DispatcherOptions.cs
index cffea2f891..1df9f50cc9 100644
--- a/src/Microsoft.AspNetCore.Dispatcher/DispatcherOptions.cs
+++ b/src/Microsoft.AspNetCore.Dispatcher/DispatcherOptions.cs
@@ -8,7 +8,5 @@ namespace Microsoft.AspNetCore.Dispatcher
     public class DispatcherOptions
     {
         public MatcherCollection Matchers { get; } = new MatcherCollection();
-
-        public IList<EndpointHandlerFactory> HandlerFactories { get; } = new List<EndpointHandlerFactory>();
     }
 }
diff --git a/src/Microsoft.AspNetCore.Dispatcher/DispatcherServiceCollectionExtensions.cs b/src/Microsoft.AspNetCore.Dispatcher/DispatcherServiceCollectionExtensions.cs
index 31e9ef190c..914b6f6106 100644
--- a/src/Microsoft.AspNetCore.Dispatcher/DispatcherServiceCollectionExtensions.cs
+++ b/src/Microsoft.AspNetCore.Dispatcher/DispatcherServiceCollectionExtensions.cs
@@ -27,7 +27,7 @@ namespace Microsoft.Extensions.DependencyInjection
             services.AddSingleton<AddressTable, DefaultAddressTable>();
             services.AddSingleton<TemplateAddressSelector>();
 
-            services.TryAddEnumerable(ServiceDescriptor.Singleton<EndpointHandlerFactoryBase, TemplateEndpointHandlerFactory>());
+            services.TryAddEnumerable(ServiceDescriptor.Singleton<IHandlerFactory, TemplateEndpointHandlerFactory>());
 
             return services;
         }
diff --git a/src/Microsoft.AspNetCore.Dispatcher/HandlerFactory.cs b/src/Microsoft.AspNetCore.Dispatcher/HandlerFactory.cs
new file mode 100644
index 0000000000..98d6d83ce4
--- /dev/null
+++ b/src/Microsoft.AspNetCore.Dispatcher/HandlerFactory.cs
@@ -0,0 +1,42 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using Microsoft.AspNetCore.Http;
+
+namespace Microsoft.AspNetCore.Dispatcher
+{
+    /// <summary>
+    /// A simple implementation of <see cref="IHandlerFactory"/> that adapts a particular endpoint
+    /// type to a handler.
+    /// </summary>
+    public sealed class HandlerFactory<TEndpoint> : IHandlerFactory
+    {
+        private readonly Func<TEndpoint, Func<RequestDelegate, RequestDelegate>> _adapter;
+
+        public HandlerFactory(Func<TEndpoint, Func<RequestDelegate, RequestDelegate>> adapter)
+        {
+            if (adapter == null)
+            {
+                throw new ArgumentNullException(nameof(adapter));
+            }
+
+            _adapter = adapter;
+        }
+
+        public Func<RequestDelegate, RequestDelegate> CreateHandler(Endpoint endpoint)
+        {
+            if (endpoint == null)
+            {
+                throw new ArgumentNullException(nameof(endpoint));
+            }
+
+            if (endpoint is TEndpoint myTypeOfEndpoint)
+            {
+                return _adapter(myTypeOfEndpoint);
+            }
+
+            return null;
+        }
+    }
+}
diff --git a/src/Microsoft.AspNetCore.Dispatcher/IDefaultMatcherFactory.cs b/src/Microsoft.AspNetCore.Dispatcher/IDefaultMatcherFactory.cs
index a98b281af8..45a7ace09d 100644
--- a/src/Microsoft.AspNetCore.Dispatcher/IDefaultMatcherFactory.cs
+++ b/src/Microsoft.AspNetCore.Dispatcher/IDefaultMatcherFactory.cs
@@ -7,6 +7,6 @@ namespace Microsoft.AspNetCore.Dispatcher
 {
     public interface IDefaultMatcherFactory
     {
-        MatcherEntry CreateDispatcher(DispatcherDataSource dataSource, IEnumerable<EndpointSelector> endpointSelectors);
+        IMatcher CreateMatcher(DispatcherDataSource dataSource, IEnumerable<EndpointSelector> endpointSelectors);
     }
 }
diff --git a/src/Microsoft.AspNetCore.Dispatcher/EndpointHandlerFactoryBase.cs b/src/Microsoft.AspNetCore.Dispatcher/IHandlerFactory.cs
similarity index 65%
rename from src/Microsoft.AspNetCore.Dispatcher/EndpointHandlerFactoryBase.cs
rename to src/Microsoft.AspNetCore.Dispatcher/IHandlerFactory.cs
index 6aeadb855c..f534cfb642 100644
--- a/src/Microsoft.AspNetCore.Dispatcher/EndpointHandlerFactoryBase.cs
+++ b/src/Microsoft.AspNetCore.Dispatcher/IHandlerFactory.cs
@@ -8,20 +8,20 @@ namespace Microsoft.AspNetCore.Dispatcher
 {
     /// <summary>
     /// <para>
-    /// Base class for implementations that can create a middleware-like delegate from an <see cref="Endpoint"/>.
+    /// An interface for components that can create a middleware-like delegate from an <see cref="Endpoint"/>.
     /// </para>
     /// <para>
-    /// Implementations registered in the application services using the service type <see cref="EndpointHandlerFactoryBase"/>
-    /// will be automatically added to <see cref="DispatcherOptions.HandlerFactories"/>.
+    /// Implementations registered in the application services using the service type <see cref="IHandlerFactory"/>
+    /// will be automatically added to set of handler factories used by the default dispatcher.
     /// </para>
     /// </summary>
-    public abstract class EndpointHandlerFactoryBase
+    public interface IHandlerFactory
     {
         /// <summary>
         /// Creates a middleware-like delegate for the provided <see cref="Endpoint"/>.
         /// </summary>
         /// <param name="endpoint">The <see cref="Endpoint"/> that will execute for the current request.</param>
         /// <returns>An <see cref="Func{RequestDelegate, RequestDelegate}"/> or <c>null</c>.</returns>
-        public abstract Func<RequestDelegate, RequestDelegate> CreateHandler(Endpoint endpoint);
+        Func<RequestDelegate, RequestDelegate> CreateHandler(Endpoint endpoint);
     }
 }
diff --git a/src/Microsoft.AspNetCore.Dispatcher/MatcherCollection.cs b/src/Microsoft.AspNetCore.Dispatcher/MatcherCollection.cs
index 586caafeb9..b2dbe8a6bf 100644
--- a/src/Microsoft.AspNetCore.Dispatcher/MatcherCollection.cs
+++ b/src/Microsoft.AspNetCore.Dispatcher/MatcherCollection.cs
@@ -8,7 +8,7 @@ namespace Microsoft.AspNetCore.Dispatcher
 {
     public class MatcherCollection : Collection<MatcherEntry>
     {
-        public void Add(MatcherBase matcher)
+        public void Add(IMatcher matcher, IHandlerFactory handerFactory)
         {
             if (matcher == null)
             {
@@ -18,21 +18,10 @@ namespace Microsoft.AspNetCore.Dispatcher
             Add(new MatcherEntry()
             {
                 Matcher = matcher,
-                AddressProvider  = matcher,
-                EndpointProvider = matcher,
-            });
-        }
+                AddressProvider = matcher as IAddressCollectionProvider,
+                EndpointProvider = matcher as IEndpointCollectionProvider,
 
-        public void Add(IMatcher matcher)
-        {
-            if (matcher == null)
-            {
-                throw new ArgumentNullException(nameof(matcher));
-            }
-
-            Add(new MatcherEntry()
-            {
-                Matcher = matcher,
+                HandlerFactory = handerFactory,
             });
         }
     }
diff --git a/src/Microsoft.AspNetCore.Dispatcher/MatcherEntry.cs b/src/Microsoft.AspNetCore.Dispatcher/MatcherEntry.cs
index 0f32c5eaea..2bc36a44b9 100644
--- a/src/Microsoft.AspNetCore.Dispatcher/MatcherEntry.cs
+++ b/src/Microsoft.AspNetCore.Dispatcher/MatcherEntry.cs
@@ -5,6 +5,8 @@ namespace Microsoft.AspNetCore.Dispatcher
 {
     public class MatcherEntry
     {
+        public IHandlerFactory HandlerFactory { get; set; }
+
         public IMatcher Matcher { get; set; }
 
         public IAddressCollectionProvider AddressProvider { get; set; }
diff --git a/src/Microsoft.AspNetCore.Dispatcher/TemplateEndpointHandlerFactory.cs b/src/Microsoft.AspNetCore.Dispatcher/TemplateEndpointHandlerFactory.cs
index a3544d1e89..f90c146a4c 100644
--- a/src/Microsoft.AspNetCore.Dispatcher/TemplateEndpointHandlerFactory.cs
+++ b/src/Microsoft.AspNetCore.Dispatcher/TemplateEndpointHandlerFactory.cs
@@ -6,9 +6,9 @@ using Microsoft.AspNetCore.Http;
 
 namespace Microsoft.AspNetCore.Dispatcher
 {
-    public class TemplateEndpointHandlerFactory : EndpointHandlerFactoryBase
+    public sealed class TemplateEndpointHandlerFactory : IHandlerFactory
     {
-        public override Func<RequestDelegate, RequestDelegate> CreateHandler(Endpoint endpoint)
+        public Func<RequestDelegate, RequestDelegate> CreateHandler(Endpoint endpoint)
         {
             if (endpoint == null)
             {
diff --git a/src/Microsoft.AspNetCore.Routing/Dispatcher/TreeMatcherFactory.cs b/src/Microsoft.AspNetCore.Routing/Dispatcher/TreeMatcherFactory.cs
index 213a0791a4..694ebe6b67 100644
--- a/src/Microsoft.AspNetCore.Routing/Dispatcher/TreeMatcherFactory.cs
+++ b/src/Microsoft.AspNetCore.Routing/Dispatcher/TreeMatcherFactory.cs
@@ -9,7 +9,7 @@ namespace Microsoft.AspNetCore.Routing.Dispatcher
 {
     public class TreeMatcherFactory : IDefaultMatcherFactory
     {
-        public MatcherEntry CreateDispatcher(DispatcherDataSource dataSource, IEnumerable<EndpointSelector> endpointSelectors)
+        public IMatcher CreateMatcher(DispatcherDataSource dataSource, IEnumerable<EndpointSelector> endpointSelectors)
         {
             if (dataSource == null)
             {
@@ -26,12 +26,7 @@ namespace Microsoft.AspNetCore.Routing.Dispatcher
                 matcher.Selectors.Add(endpointSelector);
             }
 
-            return new MatcherEntry()
-            {
-                AddressProvider = matcher,
-                Matcher = matcher,
-                EndpointProvider = matcher,
-            };
+            return matcher;
         }
     }
 }
diff --git a/test/Microsoft.AspNetCore.Dispatcher.FunctionalTest/ApiAppStartup.cs b/test/Microsoft.AspNetCore.Dispatcher.FunctionalTest/ApiAppStartup.cs
index bc3e8017d1..1b3d707636 100644
--- a/test/Microsoft.AspNetCore.Dispatcher.FunctionalTest/ApiAppStartup.cs
+++ b/test/Microsoft.AspNetCore.Dispatcher.FunctionalTest/ApiAppStartup.cs
@@ -72,9 +72,8 @@ namespace Microsoft.AspNetCore.Dispatcher.FunctionalTest
                 Selectors =
                 {
                     new HttpMethodEndpointSelector(),
-                }
-            });
-            options.HandlerFactories.Add(endpoint => (endpoint as TemplateEndpoint)?.HandlerFactory);
+                },
+            }, new TemplateEndpointHandlerFactory());
         }
 
         private Task Products_Fallback(HttpContext httpContext) => httpContext.Response.WriteAsync("Hello, Products_Fallback");