diff --git a/src/Testing/src/AzurePipelines.cs b/src/Testing/src/AzurePipelines.cs
deleted file mode 100644
index ae1eac3b90..0000000000
--- a/src/Testing/src/AzurePipelines.cs
+++ /dev/null
@@ -1,17 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Reflection;
-
-namespace Microsoft.AspNetCore.Testing
-{
-    public static class AzurePipelines
-    {
-        public const string All = Prefix + "All";
-        public const string Windows = OsPrefix + "Windows_NT";
-        public const string macOS = OsPrefix + "Darwin";
-        public const string Linux = OsPrefix + "Linux";
-
-        private const string Prefix = "AzP:";
-        private const string OsPrefix = Prefix + "OS:";
-    }
-}
diff --git a/src/Testing/src/FlakyOn.cs b/src/Testing/src/FlakyOn.cs
new file mode 100644
index 0000000000..81d9299043
--- /dev/null
+++ b/src/Testing/src/FlakyOn.cs
@@ -0,0 +1,41 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace Microsoft.AspNetCore.Testing
+{
+    public static class FlakyOn
+    {
+        public const string All = "All";
+
+        public static class Helix
+        {
+            public const string All = QueuePrefix + "All";
+
+            public const string Fedora28Amd64 = QueuePrefix + HelixQueues.Fedora28Amd64;
+            public const string Fedora27Amd64 = QueuePrefix + HelixQueues.Fedora27Amd64;
+            public const string Redhat7Amd64 = QueuePrefix + HelixQueues.Redhat7Amd64;
+            public const string Debian9Amd64 = QueuePrefix + HelixQueues.Debian9Amd64;
+            public const string Debian8Amd64 = QueuePrefix + HelixQueues.Debian8Amd64;
+            public const string Centos7Amd64 = QueuePrefix + HelixQueues.Centos7Amd64;
+            public const string Ubuntu1604Amd64 = QueuePrefix + HelixQueues.Ubuntu1604Amd64;
+            public const string Ubuntu1810Amd64 = QueuePrefix + HelixQueues.Ubuntu1810Amd64;
+            public const string macOS1012Amd64 = QueuePrefix + HelixQueues.macOS1012Amd64;
+            public const string Windows10Amd64 = QueuePrefix + HelixQueues.Windows10Amd64;
+
+            private const string Prefix = "Helix:";
+            private const string QueuePrefix = Prefix + "Queue:";
+        }
+
+        public static class AzP
+        {
+            public const string All = Prefix + "All";
+            public const string Windows = OsPrefix + "Windows_NT";
+            public const string macOS = OsPrefix + "Darwin";
+            public const string Linux = OsPrefix + "Linux";
+
+            private const string Prefix = "AzP:";
+            private const string OsPrefix = Prefix + "OS:";
+        }
+    }
+}
diff --git a/src/Testing/src/HelixQueues.cs b/src/Testing/src/HelixQueues.cs
index 84828b6b83..ef5e4d1f5a 100644
--- a/src/Testing/src/HelixQueues.cs
+++ b/src/Testing/src/HelixQueues.cs
@@ -1,26 +1,18 @@
-using System;
-using System.Collections.Generic;
-using System.Reflection;
-
 namespace Microsoft.AspNetCore.Testing
 {
     public static class HelixQueues
     {
-        public const string All = Prefix + "All";
+        public const string Fedora28Amd64 = "Fedora.28." + Amd64Suffix;
+        public const string Fedora27Amd64 = "Fedora.27." + Amd64Suffix;
+        public const string Redhat7Amd64 = "Redhat.7." + Amd64Suffix;
+        public const string Debian9Amd64 = "Debian.9." + Amd64Suffix;
+        public const string Debian8Amd64 = "Debian.8." + Amd64Suffix;
+        public const string Centos7Amd64 = "Centos.7." + Amd64Suffix;
+        public const string Ubuntu1604Amd64 = "Ubuntu.1604." + Amd64Suffix;
+        public const string Ubuntu1810Amd64 = "Ubuntu.1810." + Amd64Suffix;
+        public const string macOS1012Amd64 = "OSX.1012." + Amd64Suffix;
+        public const string Windows10Amd64 = "Windows.10.Amd64.ClientRS4.VS2017.Open"; // Doesn't have the default suffix!
 
-        public const string Fedora28Amd64 = QueuePrefix + "Fedora.28." + Amd64Suffix;
-        public const string Fedora27Amd64 = QueuePrefix + "Fedora.27." + Amd64Suffix;
-        public const string Redhat7Amd64 = QueuePrefix + "Redhat.7." + Amd64Suffix;
-        public const string Debian9Amd64 = QueuePrefix + "Debian.9." + Amd64Suffix;
-        public const string Debian8Amd64 = QueuePrefix + "Debian.8." + Amd64Suffix;
-        public const string Centos7Amd64 = QueuePrefix + "Centos.7." + Amd64Suffix;
-        public const string Ubuntu1604Amd64 = QueuePrefix + "Ubuntu.1604." + Amd64Suffix;
-        public const string Ubuntu1810Amd64 = QueuePrefix + "Ubuntu.1810." + Amd64Suffix;
-        public const string macOS1012Amd64 = QueuePrefix + "OSX.1012." + Amd64Suffix;
-        public const string Windows10Amd64 = QueuePrefix + "Windows.10.Amd64.ClientRS4.VS2017.Open"; // Doesn't have the default suffix!
-
-        private const string Prefix = "Helix:";
-        private const string QueuePrefix = Prefix + "Queue:";
         private const string Amd64Suffix = "Amd64.Open";
     }
 }
diff --git a/src/Testing/src/xunit/FlakyAttribute.cs b/src/Testing/src/xunit/FlakyAttribute.cs
index b613a9bf4d..f58026c7ca 100644
--- a/src/Testing/src/xunit/FlakyAttribute.cs
+++ b/src/Testing/src/xunit/FlakyAttribute.cs
@@ -64,12 +64,29 @@ namespace Microsoft.AspNetCore.Testing.xunit
         /// Initializes a new instance of the <see cref="FlakyAttribute"/> class with the specified <see cref="GitHubIssueUrl"/> and a list of <see cref="Filters"/>. If no
         /// filters are provided, the test is considered flaky in all environments.
         /// </summary>
+        /// <remarks>
+        /// At least one filter is required.
+        /// </remarks>
         /// <param name="gitHubIssueUrl">The URL to a GitHub issue tracking this flaky test.</param>
-        /// <param name="filters">A list of filters that define where this test is flaky. Use values in <see cref="AzurePipelines"/> and <see cref="HelixQueues"/>.</param>
-        public FlakyAttribute(string gitHubIssueUrl, params string[] filters)
+        /// <param name="firstFilter">The first filter that indicates where the test is flaky. Use a value from <see cref="FlakyOn"/>.</param>
+        /// <param name="additionalFilters">A list of additional filters that define where this test is flaky. Use values in <see cref="FlakyOn"/>.</param>
+        public FlakyAttribute(string gitHubIssueUrl, string firstFilter, params string[] additionalFilters)
         {
+            if(string.IsNullOrEmpty(gitHubIssueUrl))
+            {
+                throw new ArgumentNullException(nameof(gitHubIssueUrl));
+            }
+
+            if(string.IsNullOrEmpty(firstFilter))
+            {
+                throw new ArgumentNullException(nameof(firstFilter));
+            }
+
             GitHubIssueUrl = gitHubIssueUrl;
-            Filters = new List<string>(filters);
+            var filters = new List<string>();
+            filters.Add(firstFilter);
+            filters.AddRange(additionalFilters);
+            Filters = filters;
         }
     }
 }
diff --git a/src/Testing/test/FlakyAttributeTest.cs b/src/Testing/test/FlakyAttributeTest.cs
index e9accf6274..7837bd8711 100644
--- a/src/Testing/test/FlakyAttributeTest.cs
+++ b/src/Testing/test/FlakyAttributeTest.cs
@@ -8,8 +8,8 @@ namespace Microsoft.AspNetCore.Testing.Tests
     public class FlakyAttributeTest
     {
         [Fact]
-        [Flaky("http://example.com")]
-        public void AlwaysFlaky()
+        [Flaky("http://example.com", FlakyOn.All)]
+        public void AlwaysFlakyInCI()
         {
             if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("HELIX")) || !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("AGENT_OS")))
             {
@@ -18,7 +18,7 @@ namespace Microsoft.AspNetCore.Testing.Tests
         }
 
         [Fact]
-        [Flaky("http://example.com", HelixQueues.All)]
+        [Flaky("http://example.com", FlakyOn.Helix.All)]
         public void FlakyInHelixOnly()
         {
             if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("HELIX")))
@@ -28,7 +28,7 @@ namespace Microsoft.AspNetCore.Testing.Tests
         }
 
         [Fact]
-        [Flaky("http://example.com", HelixQueues.macOS1012Amd64, HelixQueues.Fedora28Amd64)]
+        [Flaky("http://example.com", FlakyOn.Helix.macOS1012Amd64, FlakyOn.Helix.Fedora28Amd64)]
         public void FlakyInSpecificHelixQueue()
         {
             // Today we don't run Extensions tests on Helix, but this test should light up when we do.
@@ -44,7 +44,7 @@ namespace Microsoft.AspNetCore.Testing.Tests
         }
 
         [Fact]
-        [Flaky("http://example.com", AzurePipelines.All)]
+        [Flaky("http://example.com", FlakyOn.AzP.All)]
         public void FlakyInAzPOnly()
         {
             if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("AGENT_OS")))
@@ -54,41 +54,41 @@ namespace Microsoft.AspNetCore.Testing.Tests
         }
 
         [Fact]
-        [Flaky("http://example.com", AzurePipelines.Windows)]
+        [Flaky("http://example.com", FlakyOn.AzP.Windows)]
         public void FlakyInAzPWindowsOnly()
         {
-            if (string.Equals(Environment.GetEnvironmentVariable("AGENT_OS"), AzurePipelines.Windows))
+            if (string.Equals(Environment.GetEnvironmentVariable("AGENT_OS"), "Windows_NT"))
             {
                 throw new Exception("Flaky on AzP Windows!");
             }
         }
 
         [Fact]
-        [Flaky("http://example.com", AzurePipelines.macOS)]
+        [Flaky("http://example.com", FlakyOn.AzP.macOS)]
         public void FlakyInAzPmacOSOnly()
         {
-            if (string.Equals(Environment.GetEnvironmentVariable("AGENT_OS"), AzurePipelines.macOS))
+            if (string.Equals(Environment.GetEnvironmentVariable("AGENT_OS"), "Darwin"))
             {
                 throw new Exception("Flaky on AzP macOS!");
             }
         }
 
         [Fact]
-        [Flaky("http://example.com", AzurePipelines.Linux)]
+        [Flaky("http://example.com", FlakyOn.AzP.Linux)]
         public void FlakyInAzPLinuxOnly()
         {
-            if (string.Equals(Environment.GetEnvironmentVariable("AGENT_OS"), AzurePipelines.Linux))
+            if (string.Equals(Environment.GetEnvironmentVariable("AGENT_OS"), "Linux"))
             {
                 throw new Exception("Flaky on AzP Linux!");
             }
         }
 
         [Fact]
-        [Flaky("http://example.com", AzurePipelines.Linux, AzurePipelines.macOS)]
+        [Flaky("http://example.com", FlakyOn.AzP.Linux, FlakyOn.AzP.macOS)]
         public void FlakyInAzPNonWindowsOnly()
         {
             var agentOs = Environment.GetEnvironmentVariable("AGENT_OS");
-            if (string.Equals(agentOs, "Linux") || string.Equals(agentOs, AzurePipelines.macOS))
+            if (string.Equals(agentOs, "Linux") || string.Equals(agentOs, "Darwin"))
             {
                 throw new Exception("Flaky on AzP non-Windows!");
             }