Properly parse quotes and white space

src/Microsoft.AspNetCore.Rewrite/Internal/ModRewrite/Tokenizer.cs

@@ -3,6 +3,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.Text.RegularExpressions;
 
 namespace Microsoft.AspNetCore.Rewrite.Internal.ModRewrite
 {
@@ -30,7 +31,7 @@ namespace Microsoft.AspNetCore.Rewrite.Internal.ModRewrite
             }
             var context = new ParserContext(rule);
             context.Next();
-            
+
             var tokens = new List<string>();
             context.Mark();
             while (true)
@@ -44,6 +45,21 @@ namespace Microsoft.AspNetCore.Rewrite.Internal.ModRewrite
                         throw new FormatException($"Invalid escaper character in string: {rule}");
                     }
                 }
+                else if (context.Current == '"')
+                {
+                    // Ignore all characters until the next quote is hit
+                    if (!context.Next())
+                    {
+                        throw new FormatException($"Mismatched number of quotes: {rule}");
+                    }
+                    while (context.Current != '"')
+                    {
+                        if (!context.Next())
+                        {
+                            throw new FormatException($"Mismatched number of quotes: {rule}");
+                        }
+                    }
+                }
                 else if (context.Current == Space || context.Current == Tab)
                 {
                     // time to capture!
@@ -56,6 +72,7 @@ namespace Microsoft.AspNetCore.Rewrite.Internal.ModRewrite
                             if (!context.Next())
                             {
                                 // At end of string, we can return at this point.
+                                RemoveQuotesAndEscapeCharacters(tokens);
                                 return tokens;
                             }
                         }
@@ -74,8 +91,22 @@ namespace Microsoft.AspNetCore.Rewrite.Internal.ModRewrite
             {
                 tokens.Add(done);
             }
-            
+
+            RemoveQuotesAndEscapeCharacters(tokens);
             return tokens;
         }
+
+        // Need to remove leading and trailing slashes if they exist.
+        // This is on start-up, so more forgivening towards substrings/ new strings
+        // If this is a perf/memory problem, discuss later.
+        private void RemoveQuotesAndEscapeCharacters(List<string> tokens)
+        {
+            for (int i = 0; i < tokens.Count; i++)
+            {
+                var token = tokens[i];
+                var trimmed = token.Trim('\"');
+                tokens[i] = Regex.Unescape(trimmed);
+            }
+        }
     }
 }

test/Microsoft.AspNetCore.Rewrite.Tests/ModRewrite/RewriteTokenizerTest.cs

@@ -31,7 +31,7 @@ namespace Microsoft.AspNetCore.Rewrite.Tests.ModRewrite
 
             var expected = new List<string>();
             expected.Add("RewriteCond");
-            expected.Add(@"%{HTTPS}\ what");
+            expected.Add(@"%{HTTPS} what");
             expected.Add("!-f");
             Assert.Equal(tokens, expected);
         }
@@ -45,7 +45,7 @@ namespace Microsoft.AspNetCore.Rewrite.Tests.ModRewrite
             var expected = new List<string>();
             expected.Add(@"RewriteCond");
             expected.Add(@"%{HTTPS}");
-            expected.Add(@"\ what");
+            expected.Add(@" what");
             expected.Add(@"!-f");
             Assert.Equal(tokens, expected);
         }
@@ -59,7 +59,21 @@ namespace Microsoft.AspNetCore.Rewrite.Tests.ModRewrite
             var expected = new List<string>();
             expected.Add(@"RewriteCond");
             expected.Add(@"%{HTTPS}");
-            expected.Add(@"\ what");
+            expected.Add(@" what");
+            expected.Add(@"!-f");
+            Assert.Equal(expected, tokens);
+        }
+
+        [Fact]
+        public void Tokenize_CheckQuotesAreProperlyRemovedFromString()
+        {
+            var testString = "RewriteCond \"%{HTTPS}\" \"\\ what\" \"!-f\"    ";
+            var tokens = new Tokenizer().Tokenize(testString);
+
+            var expected = new List<string>();
+            expected.Add(@"RewriteCond");
+            expected.Add(@"%{HTTPS}");
+            expected.Add(@" what");
             expected.Add(@"!-f");
             Assert.Equal(expected, tokens);
         }
@@ -67,9 +81,15 @@ namespace Microsoft.AspNetCore.Rewrite.Tests.ModRewrite
         [Fact]
         public void Tokenize_AssertFormatExceptionWhenEscapeCharacterIsAtEndOfString()
         {
-
             var ex = Assert.Throws<FormatException>(() => new Tokenizer().Tokenize("\\"));
             Assert.Equal(@"Invalid escaper character in string: \", ex.Message);
         }
+
+        [Fact]
+        public void Tokenize_AssertFormatExceptionWhenUnevenNumberOfQuotes()
+        {
+            var ex = Assert.Throws<FormatException>(() => new Tokenizer().Tokenize("\""));
+            Assert.Equal("Mismatched number of quotes: \"", ex.Message);
+        }
     }
 }