Progress on flow control

* Dealing with race conditions
* Reworking iterator methods
* Shutdown and Close need to be passed throught the write-behind mechanism

src/Microsoft.AspNet.Server.Kestrel/Http/Connection.cs

@@ -82,15 +82,7 @@ namespace Microsoft.AspNet.Server.Kestrel.Http
             {
                 SocketInput.RemoteIntakeFin = true;
                 _socket.ReadStop();
-
+                Log.ConnectionReadFin(_connectionId);
-                if (errorDone && error != null)
-                {
-                    Log.LogError("Connection.OnRead", error);
-                }
-                else
-                {
-                    Log.ConnectionReadFin(_connectionId);
-                }
             }
 
             SocketInput.SetCompleted(errorDone ? error : null);
@@ -122,19 +114,7 @@ namespace Microsoft.AspNet.Server.Kestrel.Http
                         _connectionState = ConnectionState.Shutdown;
 
                         Log.ConnectionWriteFin(_connectionId);
-                        Thread.Post(
+                        SocketOutput.End(endType);
-                            _this =>
-                            {
-                                var shutdown = new UvShutdownReq(_this.Log);
-                                shutdown.Init(_this.Thread.Loop);
-                                shutdown.Shutdown(_this._socket, (req, status, state2) =>
-                                {
-                                    var __this = (Connection)state2;
-                                    __this.Log.ConnectionWroteFin(__this._connectionId, status);
-                                    req.Dispose();
-                                }, _this);
-                            },
-                            this);
                         break;
                     case ProduceEndType.ConnectionKeepAlive:
                         if (_connectionState != ConnectionState.Open)
@@ -152,13 +132,7 @@ namespace Microsoft.AspNet.Server.Kestrel.Http
                         _connectionState = ConnectionState.Disconnected;
 
                         Log.ConnectionDisconnect(_connectionId);
-                        Thread.Post(
+                        SocketOutput.End(endType);
-                            _this =>
-                            {
-                                Log.ConnectionStop(_this._connectionId);
-                                _this._socket.Dispose();
-                            },
-                            this);
                         break;
                 }
             }

src/Microsoft.AspNet.Server.Kestrel/Http/Frame.cs

@@ -113,23 +113,15 @@ namespace Microsoft.AspNet.Server.Kestrel.Http
                     {
                         await SocketInput;
                     }
-                    else
-                    {
-                        var x = 5;
-                    }
                 }
 
-                while (!terminated && !TakeMessageHeader2(SocketInput))
+                while (!terminated && !TakeMessageHeaders(SocketInput))
                 {
                     terminated = SocketInput.RemoteIntakeFin;
                     if (!terminated)
                     {
                         await SocketInput;
                     }
-                    else
-                    {
-                        var x = 5;
-                    }
                 }
 
                 if (!terminated)
@@ -171,6 +163,10 @@ namespace Microsoft.AspNet.Server.Kestrel.Http
 
             // Connection Terminated!
             ConnectionControl.End(ProduceEndType.SocketShutdownSend);
+
+            // Wait for client to disconnect, or to receive unexpected data
+            await SocketInput;
+
             ConnectionControl.End(ProduceEndType.SocketDisconnect);
         }
 
@@ -407,13 +403,7 @@ namespace Microsoft.AspNet.Server.Kestrel.Http
                 WriteChunkedResponseSuffix();
             }
 
-            if (!_keepAlive)
+            ConnectionControl.End(_keepAlive ? ProduceEndType.ConnectionKeepAlive : ProduceEndType.SocketShutdownSend);
-            {
-                ConnectionControl.End(ProduceEndType.SocketShutdownSend);
-            }
-
-            //NOTE: must finish reading request body
-            ConnectionControl.End(_keepAlive ? ProduceEndType.ConnectionKeepAlive : ProduceEndType.SocketDisconnect);
         }
 
         private Tuple<ArraySegment<byte>, IDisposable> CreateResponseHeader(
@@ -515,36 +505,51 @@ namespace Microsoft.AspNet.Server.Kestrel.Http
 
         private bool TakeStartLine(SocketInput input)
         {
-            var begin = input.GetIterator();
+            var scan = input.GetIterator();
-            if (begin.IsDefault) return false;
 
-            var end = begin.IndexOf(' ');
+            var begin = scan;
-            var method = begin.GetString(end);
+            if (scan.Seek(' ') == -1)
-
-            char chFound;
-            begin = end.Add(1);
-            end = begin.IndexOfAny(' ', '?', out chFound);
-            var requestUri = begin.GetString(end);
-
-            begin = end;
-            end = chFound == '?' ? begin.IndexOf(' ') : begin;
-            var queryString = begin.GetString(end);
-
-            begin = end.Add(1);
-            end = begin.IndexOf('\r');
-            var httpVersion = begin.GetString(end);
-
-            end = end.Add(1);
-            if (end.Peek() != '\n')
             {
                 return false;
             }
+            var method = begin.GetString(scan);
+
+            scan.Take();
+            begin = scan;
+            var chFound = scan.Seek(' ', '?');
+            if (chFound == -1)
+            {
+                return false;
+            }
+            var requestUri = begin.GetString(scan);
+
+            var queryString = "";
+            if (chFound == '?')
+            {
+                begin = scan;
+                if (scan.Seek(' ') != ' ')
+                {
+                    return false;
+                }
+                queryString = begin.GetString(scan);
+            }
+
+            scan.Take();
+            begin = scan;
+            if (scan.Seek('\r') == -1)
+            {
+                return false;
+            }
+            var httpVersion = begin.GetString(scan);
+
+            scan.Take();
+            if (scan.Take() != '\n') return false;
 
             Method = method;
             RequestUri = requestUri;
             QueryString = queryString;
             HttpVersion = httpVersion;
-            input.JumpTo(end.Add(1));
+            input.JumpTo(scan);
             return true;
         }
 
@@ -553,150 +558,98 @@ namespace Microsoft.AspNet.Server.Kestrel.Http
             return Encoding.UTF8.GetString(range.Array, range.Offset + startIndex, endIndex - startIndex);
         }
 
-        private bool TakeMessageHeader2(SocketInput baton)
+        private bool TakeMessageHeaders(SocketInput input)
         {
-            char chFirst;
+            int chFirst;
-            char chSecond;
+            int chSecond;
-            var scan = baton.GetIterator();
+            var scan = input.GetIterator();
-            while (!scan.IsDefault)
+            var consumed = scan;
+            try
             {
-                var beginName = scan;
+                while (!scan.IsEnd)
-                scan = scan.IndexOfAny(':', '\r', out chFirst);
-
-                var endName = scan;
-                chSecond = scan.MoveNext();
-
-                if (chFirst == '\r' && chSecond == '\n')
                 {
-                    baton.JumpTo(scan.Add(1));
+                    var beginName = scan;
-                    return true;
+                    scan.Seek(':', '\r');
-                }
+                    var endName = scan;
-                if (chFirst == char.MinValue)
-                {
-                    return false;
-                }
 
-                while (
+                    chFirst = scan.Take();
-                    chSecond == ' ' ||
+                    var beginValue = scan;
-                    chSecond == '\t' ||
+                    chSecond = scan.Take();
-                    chSecond == '\r' ||
-                    chSecond == '\n')
-                {
-                    chSecond = scan.MoveNext();
-                }
 
-                var beginValue = scan;
+                    if (chFirst == -1 || chSecond == -1)
-                var wrapping = false;
-                while (!scan.IsDefault)
-                {
-                    var endValue = scan = scan.IndexOf('\r');
-                    chFirst = scan.MoveNext();
-                    if (chFirst != '\n')
                     {
-                        continue;
+                        return false;
                     }
-                    chSecond = scan.MoveNext();
+                    if (chFirst == '\r')
-                    if (chSecond == ' ' || chSecond == '\t')
                     {
-                        wrapping = true;
+                        if (chSecond == '\n')
-                        continue;
+                        {
-                    }
+                            consumed = scan;
-                    var name = beginName.GetArraySegment(endName);
+                            return true;
-                    var value = beginValue.GetString(endValue);
+                        }
-                    if (wrapping)
+                        throw new Exception("Malformed request");
-                    {
-                        value = value.Replace("\r\n", " ");
                     }
 
-                    _requestHeaders.Append(name.Array, name.Offset, name.Count, value);
+                    while (
-                    break;
+                        chSecond == ' ' ||
+                        chSecond == '\t' ||
+                        chSecond == '\r' ||
+                        chSecond == '\n')
+                    {
+                        beginValue = scan;
+                        chSecond = scan.Take();
+                    }
+                    scan = beginValue;
+
+                    var wrapping = false;
+                    while (!scan.IsEnd)
+                    {
+                        if (scan.Seek('\r') == -1)
+                        {
+                            // no "\r" in sight, burn used bytes and go back to await more data
+                            return false;
+                        }
+
+                        var endValue = scan;
+                        chFirst = scan.Take(); // expecting: /r
+                        chSecond = scan.Take(); // expecting: /n
+
+                        if (chSecond == '\r')
+                        {
+                            // special case, "\r\r". move to the 2nd "\r" and try again
+                            scan = endValue;
+                            scan.Take();
+                            continue;
+                        }
+
+                        var chThird = scan.Peek();
+                        if (chThird == ' ' || chThird == '\t')
+                        {
+                            // special case, "\r\n " or "\r\n\t". 
+                            // this is considered wrapping"linear whitespace" and is actually part of the header value
+                            // continue past this for the next
+                            wrapping = true;
+                            continue;
+                        }
+
+                        var name = beginName.GetArraySegment(endName);
+                        var value = beginValue.GetString(endValue);
+                        if (wrapping)
+                        {
+                            value = value.Replace("\r\n", " ");
+                        }
+
+                        _requestHeaders.Append(name.Array, name.Offset, name.Count, value);
+                        consumed = scan;
+                        break;
+                    }
                 }
-            }
-
-            return false;
-        }
-
-        private bool TakeMessageHeader(SocketInput baton, out bool endOfHeaders)
-        {
-            var remaining = baton.Buffer;
-            endOfHeaders = false;
-            if (remaining.Count < 2)
-            {
                 return false;
             }
-            var ch0 = remaining.Array[remaining.Offset];
+            finally
-            var ch1 = remaining.Array[remaining.Offset + 1];
-            if (ch0 == '\r' && ch1 == '\n')
             {
-                endOfHeaders = true;
+                input.JumpTo(consumed);
-                baton.Skip(2);
-                return true;
             }
-
-            if (remaining.Count < 3)
-            {
-                return false;
-            }
-            var wrappedHeaders = false;
-            var colonIndex = -1;
-            var valueStartIndex = -1;
-            var valueEndIndex = -1;
-            for (var index = 0; index != remaining.Count - 2; ++index)
-            {
-                var ch2 = remaining.Array[remaining.Offset + index + 2];
-                if (ch0 == '\r' &&
-                    ch1 == '\n' &&
-                        ch2 != ' ' &&
-                            ch2 != '\t')
-                {
-                    var value = "";
-                    if (valueEndIndex != -1)
-                    {
-                        value = _ascii.GetString(
-                            remaining.Array, remaining.Offset + valueStartIndex, valueEndIndex - valueStartIndex);
-                    }
-                    if (wrappedHeaders)
-                    {
-                        value = value.Replace("\r\n", " ");
-                    }
-                    AddRequestHeader(remaining.Array, remaining.Offset, colonIndex, value);
-                    baton.Skip(index + 2);
-                    return true;
-                }
-                if (colonIndex == -1 && ch0 == ':')
-                {
-                    colonIndex = index;
-                }
-                else if (colonIndex != -1 &&
-                    ch0 != ' ' &&
-                        ch0 != '\t' &&
-                            ch0 != '\r' &&
-                                ch0 != '\n')
-                {
-                    if (valueStartIndex == -1)
-                    {
-                        valueStartIndex = index;
-                    }
-                    valueEndIndex = index + 1;
-                }
-                else if (!wrappedHeaders &&
-                    ch0 == '\r' &&
-                        ch1 == '\n' &&
-                            (ch2 == ' ' ||
-                                ch2 == '\t'))
-                {
-                    wrappedHeaders = true;
-                }
-
-                ch0 = ch1;
-                ch1 = ch2;
-            }
-            return false;
-        }
-
-        private void AddRequestHeader(byte[] keyBytes, int keyOffset, int keyLength, string value)
-        {
-            _requestHeaders.Append(keyBytes, keyOffset, keyLength, value);
         }
 
         public bool StatusCanHaveBody(int statusCode)

src/Microsoft.AspNet.Server.Kestrel/Http/ISocketOutput.cs

@@ -14,5 +14,6 @@ namespace Microsoft.AspNet.Server.Kestrel.Http
     {
         void Write(ArraySegment<byte> buffer, bool immediate = true);
         Task WriteAsync(ArraySegment<byte> buffer, bool immediate = true, CancellationToken cancellationToken = default(CancellationToken));
+        void End(ProduceEndType endType);
     }
 }

src/Microsoft.AspNet.Server.Kestrel/Http/MemoryPool2.cs

@@ -1,7 +1,5 @@
 using System;
 using System.Collections.Concurrent;
-using System.Diagnostics;
-using System.Runtime.InteropServices;
 
 namespace Microsoft.AspNet.Server.Kestrel.Http
 {
@@ -15,6 +13,7 @@ namespace Microsoft.AspNet.Server.Kestrel.Http
 
         private ConcurrentStack<MemoryPoolBlock2> _blocks = new ConcurrentStack<MemoryPoolBlock2>();
         private ConcurrentStack<MemoryPoolSlab2> _slabs = new ConcurrentStack<MemoryPoolSlab2>();
+        private bool disposedValue = false; // To detect redundant calls
 
         public MemoryPoolBlock2 Lease(int minimumSize)
         {
@@ -27,7 +26,7 @@ namespace Microsoft.AspNet.Server.Kestrel.Http
                     slab: null);
             }
 
-            for (;;)
+            while (true)
             {
                 MemoryPoolBlock2 block;
                 if (_blocks.TryPop(out block))
@@ -66,7 +65,6 @@ namespace Microsoft.AspNet.Server.Kestrel.Http
         }
 
         #region IDisposable Support
-        private bool disposedValue = false; // To detect redundant calls
 
         protected virtual void Dispose(bool disposing)
         {

src/Microsoft.AspNet.Server.Kestrel/Http/MemoryPoolBlock2.cs

@@ -135,246 +135,284 @@ namespace Microsoft.AspNet.Server.Kestrel.Http
 
             public bool IsDefault => _block == null;
 
+            public bool IsEnd
+            {
+                get
+                {
+                    if (_block == null)
+                    {
+                        return true;
+                    }
+                    else if (_index < _block.End)
+                    {
+                        return false;
+                    }
+                    else
+                    {
+                        for (var block = _block.Next; block != null; block = block.Next)
+                        {
+                            if (block.Start < block.End)
+                            {
+                                return true;
+                            }
+                        }
+                        return true;
+                    }
+                }
+            }
+
             public MemoryPoolBlock2 Block => _block;
 
             public int Index => _index;
 
-            public bool HasAtLeast(int count)
+            public int Take()
             {
-                var scan = _block;
+                if (_block == null)
-                var index = _index;
-                while (scan != null)
                 {
-                    if (count <= scan.End - index)
+                    return -1;
-                    {
+                }
-                        return true;
+                else if (_index < _block.End)
-                    }
+                {
-                    count -= scan.End - index;
+                    return _block.Array[_index++];
-                    scan = scan.Next;
-                    index = scan?.Start ?? 0;
                 }
-                return false;
-            }
 
-            public Iterator Add(int count)
-            {
                 var block = _block;
                 var index = _index;
-                while (block != null)
+                while (true)
                 {
-                    var tailCount = block.End - index;
+                    if (index < block.End)
-                    if (count < tailCount)
                     {
-                        return new Iterator(block, index + count);
+                        _block = block;
+                        _index = index + 1;
+                        return block.Array[index];
                     }
-                    count -= tailCount;
+                    else if (block.Next == null)
-                    block = block.Next;
+                    {
-                    index = block?.Start ?? 0;
+                        return -1;
-                }
+                    }
-                return new Iterator(block, index + count);
+                    else
-            }
-
-            public Iterator CopyTo(byte[] array, int offset, int count, out int actual)
-            {
-                var block = _block;
-                var index = _index;
-                var remaining = count;
-                while (block != null && remaining != 0)
-                {
-                    var copyLength = Math.Min(remaining, block.End - index);
-                    Buffer.BlockCopy(block.Array, index, array, offset, copyLength);
-                    index += copyLength;
-                    offset += copyLength;
-                    remaining -= copyLength;
-                    if (index == block.End)
                     {
                         block = block.Next;
-                        index = block?.Start ?? 0;
+                        index = block.Start;
                     }
                 }
-                actual = count - remaining;
-                return new Iterator(block, index);
-            }
-
-            public char MoveNext()
-            {
-                var block = _block;
-                var index = _index;
-                while (block != null && index == block.End)
-                {
-                    block = block.Next;
-                    index = block?.Start ?? 0;
-                }
-                if (block != null)
-                {
-                    ++index;
-                }
-                while (block != null && index == block.End)
-                {
-                    block = block.Next;
-                    index = block?.Start ?? 0;
-                }
-                _block = block;
-                _index = index;
-                return block != null ? (char)block.Array[index] : char.MinValue;
             }
 
             public int Peek()
             {
-                while (_block != null)
+                if (_block == null)
                 {
-                    if (_index < _block.End)
+                    return -1;
-                    {
+                }
-                        return _block.Data.Array[_index];
+                else if (_index < _block.End)
-                    }
+                {
-                    _block = _block.Next;
+                    return _block.Array[_index];
-                    _index = _block.Start;
+                }
+                else if (_block.Next == null)
+                {
+                    return -1;
+                }
+
+                var block = _block.Next;
+                var index = block.Start;
+                while (true)
+                {
+                    if (index < block.End)
+                    {
+                        return block.Array[index];
+                    }
+                    else if (block.Next == null)
+                    {
+                        return -1;
+                    }
+                    else
+                    {
+                        block = block.Next;
+                        index = block.Start;
+                    }
                 }
-                return -1;
             }
 
-            public Iterator IndexOf(char char0)
+            public int Seek(int char0)
             {
+                if (IsDefault)
+                {
+                    return -1;
+                }
+
                 var byte0 = (byte)char0;
                 var vectorStride = Vector<byte>.Count;
                 var ch0Vector = new Vector<byte>(byte0);
 
-                var scanBlock = _block;
+                var block = _block;
-                var scanArray = scanBlock?.Array;
+                var index = _index;
-                var scanIndex = _index;
+                var array = block.Array;
-                while (scanBlock != null)
+                while (true)
                 {
-                    var tailCount = scanBlock.End - scanIndex;
+                    while (block.End == index)
-                    if (tailCount == 0)
                     {
-                        scanBlock = scanBlock.Next;
+                        if (block.Next == null)
-                        scanArray = scanBlock?.Array;
+                        {
-                        scanIndex = scanBlock?.Start ?? 0;
+                            _block = block;
-                        continue;
+                            _index = index;
+                            return -1;
+                        }
+                        block = block.Next;
+                        index = block.Start;
+                        array = block.Array;
                     }
-                    if (tailCount >= vectorStride)
+                    while (block.End != index)
                     {
-                        var data = new Vector<byte>(scanBlock.Array, scanIndex);
+                        var following = block.End - index;
-                        var ch0Equals = Vector.Equals(data, ch0Vector);
+                        if (following >= vectorStride)
-                        var ch0Count = Vector.Dot(ch0Equals, _dotCount);
+                        {
+                            var data = new Vector<byte>(array, index);
+                            var ch0Equals = Vector.Equals(data, ch0Vector);
+                            var ch0Count = Vector.Dot(ch0Equals, _dotCount);
 
-                        if (ch0Count == 0)
+                            if (ch0Count == 0)
-                        {
+                            {
-                            scanIndex += vectorStride;
+                                index += vectorStride;
-                            continue;
+                                continue;
+                            }
+                            else if (ch0Count == 1)
+                            {
+                                _block = block;
+                                _index = index + Vector.Dot(ch0Equals, _dotIndex);
+                                return char0;
+                            }
+                            else
+                            {
+                                following = vectorStride;
+                            }
                         }
-                        else if (ch0Count == 1)
+                        for (; following != 0; following--, index++)
                         {
-                            return new Iterator(scanBlock, scanIndex + Vector.Dot(ch0Equals, _dotIndex));
+                            if (block.Array[index] == byte0)
-                        }
+                            {
-                        else
+                                _block = block;
-                        {
+                                _index = index;
-                            tailCount = vectorStride;
+                                return char0;
-                        }
+                            }
-                    }
-                    for (; tailCount != 0; tailCount--, scanIndex++)
-                    {
-                        var ch = scanBlock.Array[scanIndex];
-                        if (ch == byte0)
-                        {
-                            return new Iterator(scanBlock, scanIndex);
                         }
                     }
                 }
-                return new Iterator(null, 0);
             }
 
-            public Iterator IndexOfAny(char char0, char char1, out char chFound)
+            public int Seek(int char0, int char1)
             {
+                if (IsDefault)
+                {
+                    return -1;
+                }
+
                 var byte0 = (byte)char0;
                 var byte1 = (byte)char1;
                 var vectorStride = Vector<byte>.Count;
                 var ch0Vector = new Vector<byte>(byte0);
                 var ch1Vector = new Vector<byte>(byte1);
 
-                var scanBlock = _block;
+                var block = _block;
-                var scanArray = scanBlock?.Array;
+                var index = _index;
-                var scanIndex = _index;
+                var array = block.Array;
-                while (scanBlock != null)
+                while (true)
                 {
-                    var tailCount = scanBlock.End - scanIndex;
+                    while (block.End == index)
-                    if (tailCount == 0)
                     {
-                        scanBlock = scanBlock.Next;
+                        if (block.Next == null)
-                        scanArray = scanBlock?.Array;
-                        scanIndex = scanBlock?.Start ?? 0;
-                        continue;
-                    }
-                    if (tailCount >= vectorStride)
-                    {
-                        var data = new Vector<byte>(scanBlock.Array, scanIndex);
-                        var ch0Equals = Vector.Equals(data, ch0Vector);
-                        var ch0Count = Vector.Dot(ch0Equals, _dotCount);
-                        var ch1Equals = Vector.Equals(data, ch1Vector);
-                        var ch1Count = Vector.Dot(ch1Equals, _dotCount);
-
-                        if (ch0Count == 0 && ch1Count == 0)
                         {
-                            scanIndex += vectorStride;
+                            _block = block;
-                            continue;
+                            _index = index;
+                            return -1;
                         }
-                        else if (ch0Count < 2 && ch1Count < 2)
+                        block = block.Next;
+                        index = block.Start;
+                        array = block.Array;
+                    }
+                    while (block.End != index)
+                    {
+                        var following = block.End - index;
+                        if (following >= vectorStride)
                         {
-                            var ch0Index = ch0Count == 1 ? Vector.Dot(ch0Equals, _dotIndex) : byte.MaxValue;
+                            var data = new Vector<byte>(array, index);
-                            var ch1Index = ch1Count == 1 ? Vector.Dot(ch1Equals, _dotIndex) : byte.MaxValue;
+                            var ch0Equals = Vector.Equals(data, ch0Vector);
-                            if (ch0Index < ch1Index)
+                            var ch0Count = Vector.Dot(ch0Equals, _dotCount);
+                            var ch1Equals = Vector.Equals(data, ch1Vector);
+                            var ch1Count = Vector.Dot(ch1Equals, _dotCount);
+
+                            if (ch0Count == 0 && ch1Count == 0)
                             {
-                                chFound = char0;
+                                index += vectorStride;
-                                return new Iterator(scanBlock, scanIndex + ch0Index);
+                                continue;
+                            }
+                            else if (ch0Count < 2 && ch1Count < 2)
+                            {
+                                var ch0Index = ch0Count == 1 ? Vector.Dot(ch0Equals, _dotIndex) : byte.MaxValue;
+                                var ch1Index = ch1Count == 1 ? Vector.Dot(ch1Equals, _dotIndex) : byte.MaxValue;
+                                if (ch0Index < ch1Index)
+                                {
+                                    _block = block;
+                                    _index = index + ch0Index;
+                                    return char0;
+                                }
+                                else
+                                {
+                                    _block = block;
+                                    _index = index + ch1Index;
+                                    return char1;
+                                }
                             }
                             else
                             {
-                                chFound = char1;
+                                following = vectorStride;
-                                return new Iterator(scanBlock, scanIndex + ch1Index);
                             }
                         }
-                        else
+                        for (; following != 0; following--, index++)
                         {
-                            tailCount = vectorStride;
+                            var byteIndex = block.Array[index];
-                        }
+                            if (byteIndex == byte0)
-                    }
+                            {
-                    for (; tailCount != 0; tailCount--, scanIndex++)
+                                _block = block;
-                    {
+                                _index = index;
-                        var chIndex = scanBlock.Array[scanIndex];
+                                return char0;
-                        if (chIndex == byte0)
+                            }
-                        {
+                            else if (byteIndex == byte1)
-                            chFound = char0;
+                            {
-                            return new Iterator(scanBlock, scanIndex);
+                                _block = block;
-                        }
+                                _index = index;
-                        else if (chIndex == byte1)
+                                return char1;
-                        {
+                            }
-                            chFound = char1;
-                            return new Iterator(scanBlock, scanIndex);
                         }
                     }
                 }
-                chFound = char.MinValue;
-                return new Iterator(null, 0);
             }
 
             public int GetLength(Iterator end)
             {
-                var length = 0;
+                if (IsDefault || end.IsDefault)
+                {
+                    return -1;
+                }
+
                 var block = _block;
                 var index = _index;
-                for (;;)
+                var length = 0;
+                while (true)
                 {
                     if (block == end._block)
                     {
                         return length + end._index - index;
                     }
-                    if (block == null)
+                    else if (block.Next == null)
                     {
-                        throw new Exception("end was not after iterator");
+                        throw new Exception("end did not follow iterator");
+                    }
+                    else
+                    {
+                        length += block.End - index;
+                        block = block.Next;
+                        index = block.Start;
                     }
-                    length += block.End - index;
-                    block = block.Next;
-                    index = block?.Start ?? 0;
                 }
             }
 
@@ -386,53 +424,74 @@ namespace Microsoft.AspNet.Server.Kestrel.Http
                 }
                 if (end._block == _block)
                 {
-                    return Encoding.ASCII.GetString(_block.Array, _index, end._index - _index);
+                    return Encoding.UTF8.GetString(_block.Array, _index, end._index - _index);
-                }
-                if (end._block == _block.Next && end._index == end._block.Start)
-                {
-                    return Encoding.ASCII.GetString(_block.Array, _index, _block.End - _index);
                 }
 
-                var length = GetLength(end);
-                var result = new char[length];
-                var offset = 0;
                 var decoder = Encoding.ASCII.GetDecoder();
 
+                var length = GetLength(end);
+                var charLength = length * 2;
+                var chars = new char[charLength];
+                var charIndex = 0;
+
                 var block = _block;
                 var index = _index;
-                while (length != 0)
+                var remaining = length;
+                while (true)
                 {
-                    if (block == null)
-                    {
-                        throw new Exception("Unexpected end of data");
-                    }
-
-                    var count = Math.Min(block.End - index, length);
-
                     int bytesUsed;
-                    int textAdded;
+                    int charsUsed;
                     bool completed;
-                    decoder.Convert(
+                    var following = block.End - index;
-                        block.Array,
+                    if (remaining <= following)
-                        index,
+                    {
-                        count,
+                        decoder.Convert(
-                        result,
+                            block.Array,
-                        offset,
+                            index,
-                        length,
+                            remaining,
-                        count == length,
+                            chars,
-                        out bytesUsed,
+                            charIndex,
-                        out textAdded,
+                            charLength - charIndex,
-                        out completed);
+                            true,
-
+                            out bytesUsed,
-                    Debug.Assert(bytesUsed == count);
+                            out charsUsed,
-                    Debug.Assert(textAdded == count);
+                            out completed);
-                    offset += count;
+                        return new string(chars, 0, charIndex + charsUsed);
-                    length -= count;
+                    }
-
+                    else if (block.Next == null)
-                    block = block.Next;
+                    {
-                    index = block?.Start ?? 0;
+                        decoder.Convert(
+                            block.Array,
+                            index,
+                            following,
+                            chars,
+                            charIndex,
+                            charLength - charIndex,
+                            true,
+                            out bytesUsed,
+                            out charsUsed,
+                            out completed);
+                        return new string(chars, 0, charIndex + charsUsed);
+                    }
+                    else
+                    {
+                        decoder.Convert(
+                            block.Array,
+                            index,
+                            following,
+                            chars,
+                            charIndex,
+                            charLength - charIndex,
+                            false,
+                            out bytesUsed,
+                            out charsUsed,
+                            out completed);
+                        charIndex += charsUsed;
+                        remaining -= following;
+                        block = block.Next;
+                        index = block.Start;
+                    }
                 }
-                return new string(result);
             }
 
             public ArraySegment<byte> GetArraySegment(Iterator end)
@@ -445,34 +504,47 @@ namespace Microsoft.AspNet.Server.Kestrel.Http
                 {
                     return new ArraySegment<byte>(_block.Array, _index, end._index - _index);
                 }
-                if (end._block == _block.Next && end._index == end._block.Start)
-                {
-                    return new ArraySegment<byte>(_block.Array, _index, _block.End - _index);
-                }
 
                 var length = GetLength(end);
-                var result = new ArraySegment<byte>(new byte[length]);
+                var array = new byte[length];
-                var offset = result.Offset;
+                CopyTo(array, 0, length, out length);
+                return new ArraySegment<byte>(array, 0, length);
+            }
+
+            public Iterator CopyTo(byte[] array, int offset, int count, out int actual)
+            {
+                if (IsDefault)
+                {
+                    actual = 0;
+                    return this;
+                }
 
                 var block = _block;
                 var index = _index;
-                while (length != 0)
+                var remaining = count;
+                while (true)
                 {
-                    if (block == null)
+                    var following = block.End - index;
+                    if (remaining <= following)
                     {
-                        throw new Exception("Unexpected end of data");
+                        actual = count;
+                        Buffer.BlockCopy(block.Array, index, array, offset, remaining);
+                        return new Iterator(block, index + remaining);
+                    }
+                    else if (block.Next == null)
+                    {
+                        actual = count - remaining + following;
+                        Buffer.BlockCopy(block.Array, index, array, offset, following);
+                        return new Iterator(block, index + following);
+                    }
+                    else
+                    {
+                        Buffer.BlockCopy(block.Array, index, array, offset, following);
+                        remaining -= following;
+                        block = block.Next;
+                        index = block.Start;
                     }
-
-                    var count = Math.Min(block.End - index, length);
-                    Buffer.BlockCopy(block.Array, index, result.Array, offset, count);
-                    offset += count;
-                    length -= count;
-
-                    block = block.Next;
-                    index = block?.Start ?? 0;
                 }
-
-                return result;
             }
         }
     }

src/Microsoft.AspNet.Server.Kestrel/Http/MemoryPoolSlab2.cs

@@ -11,6 +11,7 @@ namespace Microsoft.AspNet.Server.Kestrel.Http
         public byte[] Array;
         public IntPtr ArrayPtr;
         public bool IsActive;
+        private bool disposedValue = false; // To detect redundant calls
 
         public static MemoryPoolSlab2 Create(int length)
         {
@@ -26,7 +27,6 @@ namespace Microsoft.AspNet.Server.Kestrel.Http
         }
 
         #region IDisposable Support
-        private bool disposedValue = false; // To detect redundant calls
 
         protected virtual void Dispose(bool disposing)
         {

src/Microsoft.AspNet.Server.Kestrel/Http/MessageBody.cs

@@ -86,14 +86,24 @@ namespace Microsoft.AspNet.Server.Kestrel.Http
             public override async Task<int> ReadAsyncImplementation(ArraySegment<byte> buffer, CancellationToken cancellationToken)
             {
                 var input = _context.SocketInput;
+                while (true)
+                {
+                    await input;
 
-                await input;
+                    var begin = input.GetIterator();
+                    int actual;
+                    var end = begin.CopyTo(buffer.Array, buffer.Offset, buffer.Count, out actual);
+                    input.JumpTo(end);
 
-                var begin = input.GetIterator();
+                    if (actual != 0)
-                int actual;
+                    {
-                var end = begin.CopyTo(buffer.Array, buffer.Offset, buffer.Count, out actual);
+                        return actual;
-                input.JumpTo(end);
+                    }
-                return actual;
+                    if (input.RemoteIntakeFin)
+                    {
+                        return 0;
+                    }
+                }
             }
         }
 
@@ -114,18 +124,30 @@ namespace Microsoft.AspNet.Server.Kestrel.Http
             {
                 var input = _context.SocketInput;
 
-                var limit = Math.Min(buffer.Count, _inputLength);
+                while (true)
-                if (limit != 0)
                 {
-                    await input;
+                    var limit = Math.Min(buffer.Count, _inputLength);
-                }
+                    if (limit == 0)
+                    {
+                        return 0;
+                    }
 
-                var begin = input.GetIterator();
+                    await input;
-                int actual;
+
-                var end = begin.CopyTo(buffer.Array, buffer.Offset, limit, out actual);
+                    var begin = input.GetIterator();
-                _inputLength -= actual;
+                    int actual;
-                input.JumpTo(end);
+                    var end = begin.CopyTo(buffer.Array, buffer.Offset, limit, out actual);
-                return actual;
+                    _inputLength -= actual;
+                    input.JumpTo(end);
+                    if (actual != 0)
+                    {
+                        return actual;
+                    }
+                    if (input.RemoteIntakeFin)
+                    {
+                        throw new Exception("Unexpected end of request content");
+                    }
+                }
             }
         }
 
@@ -186,21 +208,23 @@ namespace Microsoft.AspNet.Server.Kestrel.Http
                         {
                             _mode = Mode.ChunkSuffix;
                         }
-
+                        if (actual != 0)
-                        return actual;
+                        {
+                            return actual;
+                        }
                     }
                     while (_mode == Mode.ChunkSuffix)
                     {
-                        var begin = input.GetIterator();
+                        var scan = input.GetIterator();
-                        var ch1 = begin.Peek();
+                        var ch1 = scan.Take();
-                        var ch2 = begin.MoveNext();
+                        var ch2 = scan.Take();
-                        if (ch1 == char.MinValue || ch2 == char.MinValue)
+                        if (ch1 == -1 || ch2 == -1)
                         {
                             await input;
                         }
                         else if (ch1 == '\r' && ch2 == '\n')
                         {
-                            input.JumpTo(begin.Add(1));
+                            input.JumpTo(scan);
                             _mode = Mode.ChunkPrefix;
                         }
                         else
@@ -215,13 +239,17 @@ namespace Microsoft.AspNet.Server.Kestrel.Http
 
             private static bool TakeChunkedLine(SocketInput baton, ref int chunkSizeOut)
             {
-                var remaining = baton.GetIterator();
+                var scan = baton.GetIterator();
-                var ch0 = remaining.Peek();
+                var ch0 = scan.Take();
                 var chunkSize = 0;
                 var mode = 0;
-                while(ch0 != -1)
+                while (ch0 != -1)
                 {
-                    var ch1 = remaining.MoveNext();
+                    var ch1 = scan.Take();
+                    if (ch1 == -1)
+                    {
+                        return false;
+                    }
 
                     if (mode == 0)
                     {
@@ -263,7 +291,7 @@ namespace Microsoft.AspNet.Server.Kestrel.Http
                         }
                         else if (ch0 == '\r' && ch1 == '\n')
                         {
-                            baton.JumpTo(remaining.Add(1));
+                            baton.JumpTo(scan);
                             chunkSizeOut = chunkSize;
                             return true;
                         }
@@ -276,7 +304,7 @@ namespace Microsoft.AspNet.Server.Kestrel.Http
                     {
                         if (ch0 == '\r' && ch1 == '\n')
                         {
-                            baton.JumpTo(remaining.Add(1));
+                            baton.JumpTo(scan);
                             chunkSizeOut = chunkSize;
                             return true;
                         }

src/Microsoft.AspNet.Server.Kestrel/Http/SocketInput.cs

@@ -206,19 +206,16 @@ namespace Microsoft.AspNet.Server.Kestrel.Http
             lock (_syncHeadAndTail)
             {
                 // TODO: leave _pinned intact
+                // TODO: return when empty
 
                 returnStart = _head;
                 returnEnd = iterator.Block;
                 _head = iterator.Block;
-                if (_head == null)
+                _head.Start = iterator.Index;
+                if (iterator.IsEnd)
                 {
-                    _tail = null;
                     SetNotCompleted();
                 }
-                else
-                {
-                    _head.Start = iterator.Index;
-                }
             }
             while (returnStart != returnEnd)
             {

src/Microsoft.AspNet.Server.Kestrel/Http/SocketOutput.cs

@@ -42,14 +42,22 @@ namespace Microsoft.AspNet.Server.Kestrel.Http
             _callbacksPending = new Queue<CallbackContext>();
         }
 
-        public void Write(ArraySegment<byte> buffer, Action<Exception, object> callback, object state, bool immediate = true)
+        public void Write(
+            ArraySegment<byte> buffer,
+            Action<Exception, object> callback,
+            object state,
+            bool immediate = true,
+            bool socketShutdownSend = false,
+            bool socketDisconnect = false)
         {
             //TODO: need buffering that works
-            var copy = new byte[buffer.Count];
+            if (buffer.Array != null)
-            Array.Copy(buffer.Array, buffer.Offset, copy, 0, buffer.Count);
+            {
-            buffer = new ArraySegment<byte>(copy);
+                var copy = new byte[buffer.Count];
-
+                Array.Copy(buffer.Array, buffer.Offset, copy, 0, buffer.Count);
-            _log.ConnectionWrite(_connectionId, buffer.Count);
+                buffer = new ArraySegment<byte>(copy);
+                _log.ConnectionWrite(_connectionId, buffer.Count);
+            }
 
             bool triggerCallbackNow = false;
 
@@ -60,8 +68,18 @@ namespace Microsoft.AspNet.Server.Kestrel.Http
                     _nextWriteContext = new WriteContext(this);
                 }
 
-                _nextWriteContext.Buffers.Enqueue(buffer);
+                if (buffer.Array != null)
-
+                {
+                    _nextWriteContext.Buffers.Enqueue(buffer);
+                }
+                if (socketShutdownSend)
+                {
+                    _nextWriteContext.SocketShutdownSend = true;
+                }
+                if (socketDisconnect)
+                {
+                    _nextWriteContext.SocketDisconnect = true;
+                }
                 // Complete the write task immediately if all previous write tasks have been completed,
                 // the buffers haven't grown too large, and the last write to the socket succeeded.
                 triggerCallbackNow = _lastWriteError == null &&
@@ -121,22 +139,7 @@ namespace Microsoft.AspNet.Server.Kestrel.Http
 
             try
             {
-                var buffers = new ArraySegment<byte>[writingContext.Buffers.Count];
+                writingContext.Execute();
-
-                var i = 0;
-                foreach (var buffer in writingContext.Buffers)
-                {
-                    buffers[i++] = buffer;
-                }
-
-                var writeReq = new UvWriteReq(_log);
-                writeReq.Init(_thread.Loop);
-
-                writeReq.Write(_socket, new ArraySegment<ArraySegment<byte>>(buffers), (r, status, error, state) =>
-                {
-                    var writtenContext = (WriteContext)state;
-                    writtenContext.Self.OnWriteCompleted(writtenContext.Buffers, r, status, error);
-                }, writingContext);
             }
             catch
             {
@@ -152,7 +155,7 @@ namespace Microsoft.AspNet.Server.Kestrel.Http
         }
 
         // This is called on the libuv event loop
-        private void OnWriteCompleted(Queue<ArraySegment<byte>> writtenBuffers, UvWriteReq req, int status, Exception error)
+        private void OnWriteCompleted(Queue<ArraySegment<byte>> writtenBuffers, int status, Exception error)
         {
             _log.ConnectionWriteCallback(_connectionId, status);
 
@@ -194,8 +197,6 @@ namespace Microsoft.AspNet.Server.Kestrel.Http
                 Trace.Assert(_numBytesPreCompleted >= 0);
                 Trace.Assert(_numBytesPreCompleted <= _maxBytesPreCompleted);
             }
-
-            req.Dispose();
         }
 
         private void TriggerCallback(CallbackContext context)
@@ -237,6 +238,25 @@ namespace Microsoft.AspNet.Server.Kestrel.Http
             return tcs.Task;
         }
 
+        void ISocketOutput.End(ProduceEndType endType)
+        {
+            switch (endType)
+            {
+                case ProduceEndType.SocketShutdownSend:
+                    Write(default(ArraySegment<byte>), (error, state) => { }, null,
+                        immediate: true,
+                        socketShutdownSend: true,
+                        socketDisconnect: false);
+                    break;
+                case ProduceEndType.SocketDisconnect:
+                    Write(default(ArraySegment<byte>), (error, state) => { }, null,
+                        immediate: true,
+                        socketShutdownSend: false,
+                        socketDisconnect: true);
+                    break;
+            }
+        }
+
         private class CallbackContext
         {
             public Exception Error;
@@ -248,13 +268,89 @@ namespace Microsoft.AspNet.Server.Kestrel.Http
         private class WriteContext
         {
             public SocketOutput Self;
+
             public Queue<ArraySegment<byte>> Buffers;
+            public bool SocketShutdownSend;
+            public bool SocketDisconnect;
+
+            public int WriteStatus;
+            public Exception WriteError;
+
+            public int ShutdownSendStatus;
 
             public WriteContext(SocketOutput self)
             {
                 Self = self;
                 Buffers = new Queue<ArraySegment<byte>>();
             }
+
+            public void Execute()
+            {
+                if (Buffers.Count == 0 || Self._socket.IsClosed)
+                {
+                    StageTwo();
+                    return;
+                }
+
+                var buffers = new ArraySegment<byte>[Buffers.Count];
+
+                var i = 0;
+                foreach (var buffer in Buffers)
+                {
+                    buffers[i++] = buffer;
+                }
+
+                var writeReq = new UvWriteReq(Self._log);
+                writeReq.Init(Self._thread.Loop);
+                writeReq.Write(Self._socket, new ArraySegment<ArraySegment<byte>>(buffers), (_writeReq, status, error, state) =>
+                {
+                    _writeReq.Dispose();
+                    var _this = (WriteContext)state;
+                    _this.WriteStatus = status;
+                    _this.WriteError = error;
+                    StageTwo();
+                }, this);
+            }
+
+            public void StageTwo()
+            {
+                if (SocketShutdownSend == false || Self._socket.IsClosed)
+                {
+                    StageThree();
+                    return;
+                }
+
+                var shutdownReq = new UvShutdownReq(Self._log);
+                shutdownReq.Init(Self._thread.Loop);
+                shutdownReq.Shutdown(Self._socket, (_shutdownReq, status, state) =>
+                {
+                    _shutdownReq.Dispose();
+                    var _this = (WriteContext)state;
+                    _this.ShutdownSendStatus = status;
+
+                    Self._log.ConnectionWroteFin(Self._connectionId, status);
+
+                    StageThree();
+                }, this);
+            }
+
+            public void StageThree()
+            {
+                if (SocketDisconnect == false || Self._socket.IsClosed)
+                {
+                    Complete();
+                    return;
+                }
+
+                Self._socket.Dispose();
+                Self._log.ConnectionStop(Self._connectionId);
+                Complete();
+            }
+
+            public void Complete()
+            {
+                Self.OnWriteCompleted(Buffers, WriteStatus, WriteError);
+            }
         }
     }
 }

test/Microsoft.AspNet.Server.KestrelTests/EngineTests.cs

@@ -12,6 +12,7 @@ using Microsoft.AspNet.Server.Kestrel.Http;
 using Microsoft.Dnx.Runtime;
 using Microsoft.Dnx.Runtime.Infrastructure;
 using Xunit;
+using System.Linq;
 
 namespace Microsoft.AspNet.Server.KestrelTests
 {
@@ -23,7 +24,7 @@ namespace Microsoft.AspNet.Server.KestrelTests
         private async Task App(Frame frame)
         {
             frame.ResponseHeaders.Clear();
-            for (; ;)
+            while (true)
             {
                 var buffer = new byte[8192];
                 var count = await frame.RequestBody.ReadAsync(buffer, 0, buffer.Length);
@@ -60,17 +61,17 @@ namespace Microsoft.AspNet.Server.KestrelTests
 
         private async Task AppChunked(Frame frame)
         {
+            foreach (var h in frame.RequestHeaders)
+            {
+                Console.WriteLine($"{h.Key}: {h.Value}");
+            }
+            Console.WriteLine($"");
+
             frame.ResponseHeaders.Clear();
             var data = new MemoryStream();
-            for (; ;)
+            while(true)
             {
-                var buffer = new byte[8192];
+                await frame.RequestBody.CopyToAsync(data);
-                var count = await frame.RequestBody.ReadAsync(buffer, 0, buffer.Length);
-                if (count == 0)
-                {
-                    break;
-                }
-                data.Write(buffer, 0, count);
             }
             var bytes = data.ToArray();
             frame.ResponseHeaders["Content-Length"] = new[] { bytes.Length.ToString() };
@@ -115,7 +116,7 @@ namespace Microsoft.AspNet.Server.KestrelTests
             socket.Send(Encoding.ASCII.GetBytes("POST / HTTP/1.0\r\n\r\nHello World"));
             socket.Shutdown(SocketShutdown.Send);
             var buffer = new byte[8192];
-            for (; ;)
+            for (;;)
             {
                 var length = socket.Receive(buffer);
                 if (length == 0) { break; }

test/Microsoft.AspNet.Server.KestrelTests/MemoryPoolBlock2Tests.cs

@@ -10,7 +10,7 @@ namespace Microsoft.AspNet.Server.KestrelTests
     public class MemoryPoolBlock2Tests
     {
         [Fact]
-        public void IndexOfAnyWorks()
+        public void SeekWorks()
         {
             using (var pool = new MemoryPool2())
             {
@@ -22,7 +22,16 @@ namespace Microsoft.AspNet.Server.KestrelTests
                 var iterator = block.GetIterator();
                 foreach (var ch in Enumerable.Range(0, 256).Select(x => (char)x))
                 {
-                    var hit = iterator.IndexOf(ch);
+                    var hit = iterator;
+                    hit.Seek(ch);
+                    Assert.Equal(ch, iterator.GetLength(hit));
+
+                    hit = iterator;
+                    hit.Seek(ch, byte.MaxValue);
+                    Assert.Equal(ch, iterator.GetLength(hit));
+
+                    hit = iterator;
+                    hit.Seek(byte.MaxValue, ch);
                     Assert.Equal(ch, iterator.GetLength(hit));
                 }
             }
@@ -72,5 +81,49 @@ namespace Microsoft.AspNet.Server.KestrelTests
                 }
             }
         }
+
+        [Fact]
+        public void AddDoesNotAdvanceAtEndOfCurrentBlock()
+        {
+            using (var pool = new MemoryPool2())
+            {
+                var block1 = pool.Lease(256);
+                var block2 = block1.Next = pool.Lease(256);
+
+                block1.End += 100;
+                block2.End += 200;
+
+                var iter0 = block1.GetIterator();
+                var iter100 = iter0.Add(100);
+
+                var iter200a = iter0.Add(200);
+                var iter200b = iter100.Add(100);
+
+                var iter300a = iter0.Add(300);
+                var iter300b = iter100.Add(200);
+                var iter300c = iter200a.Add(100);
+
+                var iter300a2 = iter300a.Add(1);
+                var iter300b2 = iter300b.Add(1);
+                var iter300c2 = iter300c.Add(1);
+
+                AssertIterator(iter0, block1, block1.Start);
+                AssertIterator(iter100, block1, block1.End);
+                AssertIterator(iter200a, block2, block2.Start+100);
+                AssertIterator(iter200b, block2, block2.Start + 100);
+                AssertIterator(iter300a, block2, block2.End);
+                AssertIterator(iter300b, block2, block2.End);
+                AssertIterator(iter300c, block2, block2.End);
+                AssertIterator(iter300a2, block2, block2.End);
+                AssertIterator(iter300b2, block2, block2.End);
+                AssertIterator(iter300c2, block2, block2.End);
+            }
+        }
+
+        private void AssertIterator(MemoryPoolBlock2.Iterator iter, MemoryPoolBlock2 block, int index)
+        {
+            Assert.Same(block, iter.Block);
+            Assert.Equal(index, iter.Index);
+        }
     }
 }

test/Microsoft.AspNet.Server.KestrelTests/MemoryPoolExtensions.cs

@@ -0,0 +1,13 @@
+using Microsoft.AspNet.Server.Kestrel.Http;
+
+namespace Microsoft.AspNet.Server.KestrelTests
+{
+    public static class MemoryPoolExtensions
+    {
+        public static MemoryPoolBlock2.Iterator Add(this MemoryPoolBlock2.Iterator iterator, int count)
+        {
+            int actual;
+            return iterator.CopyTo(new byte[count], 0, count,  out actual);
+        } 
+    }
+}

tools/Microsoft.StandardsPolice/StandardsPoliceCompileModule.cs

@@ -71,8 +71,8 @@ namespace Microsoft.StandardsPolice
                     acceptableOrder = true;
                 }
                 if (!acceptableOrder &&
-                    priorUsingDirective.Name.ToString().StartsWith("System.") &&
+                    priorUsingDirective.Name.ToString().StartsWith("System") &&
-                    !usingDirective.Name.ToString().StartsWith("System."))
+                    !usingDirective.Name.ToString().StartsWith("System"))
                 {
                     acceptableOrder = true;
                 }