diff --git a/src/Microsoft.AspNetCore.Server.Kestrel/Http/SocketInput.cs b/src/Microsoft.AspNetCore.Server.Kestrel/Http/SocketInput.cs index 1588e83011..40532e17b4 100644 --- a/src/Microsoft.AspNetCore.Server.Kestrel/Http/SocketInput.cs +++ b/src/Microsoft.AspNetCore.Server.Kestrel/Http/SocketInput.cs @@ -105,7 +105,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Http } else { - _tail.Next = _pinned; + Volatile.Write(ref _tail.Next, _pinned); _tail = _pinned; } diff --git a/src/Microsoft.AspNetCore.Server.Kestrel/Infrastructure/MemoryPoolBlock.cs b/src/Microsoft.AspNetCore.Server.Kestrel/Infrastructure/MemoryPoolBlock.cs index 3badd57fce..a57332a839 100644 --- a/src/Microsoft.AspNetCore.Server.Kestrel/Infrastructure/MemoryPoolBlock.cs +++ b/src/Microsoft.AspNetCore.Server.Kestrel/Infrastructure/MemoryPoolBlock.cs @@ -55,14 +55,14 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Infrastructure /// the Start is guaranteed to be equal to Array.Offset. The value of Start may be assigned anywhere between Data.Offset and /// Data.Offset + Data.Count, and must be equal to or less than End. /// - public int Start { get; set; } + public int Start; /// /// The End represents the offset into Array where the range of "active" bytes ends. At the point when the block is leased /// the End is guaranteed to be equal to Array.Offset. The value of Start may be assigned anywhere between Data.Offset and /// Data.Offset + Data.Count, and must be equal to or less than End. /// - public int End { get; set; } + public volatile int End; /// /// Reference to the next block of data when the overall "active" bytes spans multiple blocks. At the point when the block is @@ -70,7 +70,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Infrastructure /// working memory. The "active" memory is grown when bytes are copied in, End is increased, and Next is assigned. The "active" /// memory is shrunk when bytes are consumed, Start is increased, and blocks are returned to the pool. /// - public MemoryPoolBlock Next { get; set; } + public MemoryPoolBlock Next; ~MemoryPoolBlock() { diff --git a/src/Microsoft.AspNetCore.Server.Kestrel/Infrastructure/MemoryPoolIterator.cs b/src/Microsoft.AspNetCore.Server.Kestrel/Infrastructure/MemoryPoolIterator.cs index 0fcfe7f434..3d42fba3b9 100644 --- a/src/Microsoft.AspNetCore.Server.Kestrel/Infrastructure/MemoryPoolIterator.cs +++ b/src/Microsoft.AspNetCore.Server.Kestrel/Infrastructure/MemoryPoolIterator.cs @@ -4,6 +4,7 @@ using System; using System.Diagnostics; using System.Numerics; +using System.Threading; namespace Microsoft.AspNetCore.Server.Kestrel.Infrastructure { @@ -68,6 +69,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Infrastructure } var index = _index; + var wasLastBlock = block.Next == null; if (index < block.End) { @@ -77,7 +79,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Infrastructure do { - if (block.Next == null) + if (wasLastBlock) { return -1; } @@ -87,6 +89,8 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Infrastructure index = block.Start; } + wasLastBlock = block.Next == null; + if (index < block.End) { _block = block; @@ -102,7 +106,10 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Infrastructure { return; } + + var wasLastBlock = _block.Next == null; var following = _block.End - _index; + if (following >= bytesToSkip) { _index += bytesToSkip; @@ -113,7 +120,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Infrastructure var index = _index; while (true) { - if (block.Next == null) + if (wasLastBlock) { return; } @@ -123,7 +130,10 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Infrastructure block = block.Next; index = block.Start; } + + wasLastBlock = block.Next == null; following = block.End - index; + if (following >= bytesToSkip) { _block = block; @@ -141,6 +151,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Infrastructure return -1; } + var wasLastBlock = _block.Next == null; var index = _index; if (index < block.End) @@ -150,7 +161,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Infrastructure do { - if (block.Next == null) + if (wasLastBlock) { return -1; } @@ -160,6 +171,8 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Infrastructure index = block.Start; } + wasLastBlock = block.Next == null; + if (index < block.End) { return block.Array[index]; @@ -173,11 +186,14 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Infrastructure { return -1; } - else if (_block.End - _index >= sizeof(long)) + + var wasLastBlock = _block.Next == null; + + if (_block.End - _index >= sizeof(long)) { return *(long*)(_block.DataFixedPtr + _index); } - else if (_block.Next == null) + else if (wasLastBlock) { return -1; } @@ -208,6 +224,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Infrastructure var block = _block; var index = _index; + var wasLastBlock = block.Next == null; var following = block.End - index; byte[] array; var byte0 = byte0Vector[0]; @@ -216,16 +233,16 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Infrastructure { while (following == 0) { - var newBlock = block.Next; - if (newBlock == null) + if (wasLastBlock) { _block = block; _index = index; return -1; } - index = newBlock.Start; - following = newBlock.End - index; - block = newBlock; + block = block.Next; + index = block.Start; + wasLastBlock = block.Next == null; + following = block.End - index; } array = block.Array; while (following > 0) @@ -285,6 +302,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Infrastructure var block = _block; var index = _index; + var wasLastBlock = block.Next == null; var following = block.End - index; byte[] array; int byte0Index = int.MaxValue; @@ -296,16 +314,16 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Infrastructure { while (following == 0) { - var newBlock = block.Next; - if (newBlock == null) + if (wasLastBlock) { _block = block; _index = index; return -1; } - index = newBlock.Start; - following = newBlock.End - index; - block = newBlock; + block = block.Next; + index = block.Start; + wasLastBlock = block.Next == null; + following = block.End - index; } array = block.Array; while (following > 0) @@ -389,6 +407,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Infrastructure var block = _block; var index = _index; + var wasLastBlock = block.Next == null; var following = block.End - index; byte[] array; int byte0Index = int.MaxValue; @@ -402,16 +421,16 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Infrastructure { while (following == 0) { - var newBlock = block.Next; - if (newBlock == null) + if (wasLastBlock) { _block = block; _index = index; return -1; } - index = newBlock.Start; - following = newBlock.End - index; - block = newBlock; + block = block.Next; + index = block.Start; + wasLastBlock = block.Next == null; + following = block.End - index; } array = block.Array; while (following > 0) @@ -586,16 +605,13 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Infrastructure { return false; } - else if (_index < _block.End) - { - _block.Array[_index++] = data; - return true; - } var block = _block; var index = _index; while (true) { + var wasLastBlock = block.Next == null; + if (index < block.End) { _block = block; @@ -603,7 +619,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Infrastructure block.Array[index] = data; return true; } - else if (block.Next == null) + else if (wasLastBlock) { return false; } @@ -660,6 +676,11 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Infrastructure var remaining = count; while (true) { + // Determine if we might attempt to copy data from block.Next before + // calculating "following" so we don't risk skipping data that could + // be added after block.End when we decide to copy from block.Next. + // block.End will always be advanced before block.Next is set. + var wasLastBlock = block.Next == null; var following = block.End - index; if (remaining <= following) { @@ -670,7 +691,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Infrastructure } return new MemoryPoolIterator(block, index + remaining); } - else if (block.Next == null) + else if (wasLastBlock) { actual = count - remaining + following; if (array != null) @@ -728,7 +749,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Infrastructure { var nextBlock = pool.Lease(); block.End = blockIndex; - block.Next = nextBlock; + Volatile.Write(ref block.Next, nextBlock); block = nextBlock; blockIndex = block.Data.Offset; @@ -781,7 +802,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Infrastructure { var nextBlock = pool.Lease(); block.End = blockIndex; - block.Next = nextBlock; + Volatile.Write(ref block.Next, nextBlock); block = nextBlock; blockIndex = block.Data.Offset;