huahaofeng
/
aspnetcore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

#553 Use System.Buffers for temporary arrays

This commit is contained in:
Chris R 2016-03-28 13:57:08 -07:00
parent 1b71748150
commit bd60507dcd
11 changed files with 211 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/Microsoft.AspNetCore.Http/project.json
View File

@ -21,7 +21,8 @@
"Microsoft.AspNetCore.Http.Abstractions": "1.0.0-*",
"Microsoft.AspNetCore.WebUtilities": "1.0.0-*",
"Microsoft.Extensions.ObjectPool": "1.0.0-*",
"Microsoft.Net.Http.Headers": "1.0.0-*"
"Microsoft.Net.Http.Headers": "1.0.0-*",
"System.Buffers": "4.0.0-*"
},
"frameworks": {
"net451": { },

17
src/Microsoft.AspNetCore.Owin/WebSockets/OwinWebSocketAdapter.cs
View File

@ -2,6 +2,7 @@
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System;
using System.Buffers;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
@ -33,6 +34,7 @@ namespace Microsoft.AspNetCore.Owin
public class OwinWebSocketAdapter : WebSocket
{
private const int _rentedBufferSize = 1024;
private IDictionary<string, object> _websocketContext;
private WebSocketSendAsync _sendAsync;
private WebSocketReceiveAsync _receiveAsync;
@ -126,11 +128,18 @@ namespace Microsoft.AspNetCore.Owin
await CloseOutputAsync(closeStatus, statusDescription, cancellationToken);
}
byte[] buffer = new byte[1024];
while (State == WebSocketState.CloseSent)
var buffer = ArrayPool<byte>.Shared.Rent(_rentedBufferSize);
try
{
// Drain until close received
await ReceiveAsync(new ArraySegment<byte>(buffer), cancellationToken);
while (State == WebSocketState.CloseSent)
{
// Drain until close received
await ReceiveAsync(new ArraySegment<byte>(buffer), cancellationToken);
}
}
finally
{
ArrayPool<byte>.Shared.Return(buffer);
}
}

21
src/Microsoft.AspNetCore.WebUtilities/BufferedReadStream.cs
View File

@ -2,6 +2,7 @@
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System;
using System.Buffers;
using System.IO;
using System.Text;
using System.Threading;
@ -16,11 +17,17 @@ namespace Microsoft.AspNetCore.WebUtilities
private readonly Stream _inner;
private readonly byte[] _buffer;
private readonly ArrayPool<byte> _bytePool;
private int _bufferOffset = 0;
private int _bufferCount = 0;
private bool _disposed;
public BufferedReadStream(Stream inner, int bufferSize)
: this(inner, bufferSize, ArrayPool<byte>.Shared)
{
}
public BufferedReadStream(Stream inner, int bufferSize, ArrayPool<byte> bytePool)
{
if (inner == null)
{
@ -28,7 +35,8 @@ namespace Microsoft.AspNetCore.WebUtilities
}
_inner = inner;
_buffer = new byte[bufferSize];
_bytePool = bytePool;
_buffer = bytePool.Rent(bufferSize);
}
public ArraySegment<byte> BufferedData
@ -128,10 +136,15 @@ namespace Microsoft.AspNetCore.WebUtilities
protected override void Dispose(bool disposing)
{
_disposed = true;
if (disposing)
if (!_disposed)
{
_inner.Dispose();
_disposed = true;
_bytePool.Return(_buffer);
if (disposing)
{
_inner.Dispose();
}
}
}

71
src/Microsoft.AspNetCore.WebUtilities/FileBufferingReadStream.cs
View File

@ -2,6 +2,7 @@
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System;
using System.Buffers;
using System.Diagnostics;
using System.IO;
using System.Threading;
@ -16,12 +17,15 @@ namespace Microsoft.AspNetCore.WebUtilities
/// </summary>
public class FileBufferingReadStream : Stream
{
private const int _maxRentedBufferSize = 1024 * 1024; // 1MB
private readonly Stream _inner;
private readonly ArrayPool<byte> _bytePool;
private readonly int _memoryThreshold;
private string _tempFileDirectory;
private readonly Func<string> _tempFileDirectoryAccessor;
private Stream _buffer = new MemoryStream(); // TODO: We could have a more efficiently expanding buffer stream.
private Stream _buffer;
private byte[] _rentedBuffer;
private bool _inMemory = true;
private bool _completelyBuffered;
@ -32,6 +36,15 @@ namespace Microsoft.AspNetCore.WebUtilities
Stream inner,
int memoryThreshold,
Func<string> tempFileDirectoryAccessor)
: this(inner, memoryThreshold, tempFileDirectoryAccessor, ArrayPool<byte>.Shared)
{
}
public FileBufferingReadStream(
Stream inner,
int memoryThreshold,
Func<string> tempFileDirectoryAccessor,
ArrayPool<byte> bytePool)
{
if (inner == null)
{
@ -43,6 +56,18 @@ namespace Microsoft.AspNetCore.WebUtilities
throw new ArgumentNullException(nameof(tempFileDirectoryAccessor));
}
_bytePool = bytePool;
if (memoryThreshold < _maxRentedBufferSize)
{
_rentedBuffer = bytePool.Rent(memoryThreshold);
_buffer = new MemoryStream(_rentedBuffer);
_buffer.SetLength(0);
}
else
{
_buffer = new MemoryStream();
}
_inner = inner;
_memoryThreshold = memoryThreshold;
_tempFileDirectoryAccessor = tempFileDirectoryAccessor;
@ -50,6 +75,15 @@ namespace Microsoft.AspNetCore.WebUtilities
// TODO: allow for an optional buffer size limit to prevent filling hard disks. 1gb?
public FileBufferingReadStream(Stream inner, int memoryThreshold, string tempFileDirectory)
: this(inner, memoryThreshold, tempFileDirectory, ArrayPool<byte>.Shared)
{
}
public FileBufferingReadStream(
Stream inner,
int memoryThreshold,
string tempFileDirectory,
ArrayPool<byte> bytePool)
{
if (inner == null)
{
@ -61,6 +95,18 @@ namespace Microsoft.AspNetCore.WebUtilities
throw new ArgumentNullException(nameof(tempFileDirectory));
}
_bytePool = bytePool;
if (memoryThreshold < _maxRentedBufferSize)
{
_rentedBuffer = bytePool.Rent(memoryThreshold);
_buffer = new MemoryStream(_rentedBuffer);
_buffer.SetLength(0);
}
else
{
_buffer = new MemoryStream();
}
_inner = inner;
_memoryThreshold = memoryThreshold;
_tempFileDirectory = tempFileDirectory;
@ -145,11 +191,11 @@ namespace Microsoft.AspNetCore.WebUtilities
if (_inMemory && _buffer.Length + read > _memoryThreshold)
{
var oldBuffer = _buffer;
_buffer = CreateTempFile();
_inMemory = false;
oldBuffer.Position = 0;
oldBuffer.CopyTo(_buffer, 1024 * 16);
_buffer = CreateTempFile();
_buffer.Write(_rentedBuffer, 0, (int)_buffer.Length);
_bytePool.Return(_rentedBuffer);
_rentedBuffer = null;
}
if (read > 0)
@ -216,11 +262,11 @@ namespace Microsoft.AspNetCore.WebUtilities
if (_inMemory && _buffer.Length + read > _memoryThreshold)
{
var oldBuffer = _buffer;
_buffer = CreateTempFile();
_inMemory = false;
oldBuffer.Position = 0;
await oldBuffer.CopyToAsync(_buffer, 1024 * 16, cancellationToken);
_buffer = CreateTempFile();
await _buffer.WriteAsync(_rentedBuffer, 0, (int)_buffer.Length, cancellationToken);
_bytePool.Return(_rentedBuffer);
_rentedBuffer = null;
}
if (read > 0)
@ -270,6 +316,11 @@ namespace Microsoft.AspNetCore.WebUtilities
if (!_disposed)
{
_disposed = true;
if (_rentedBuffer != null)
{
_bytePool.Return(_rentedBuffer);
}
if (disposing)
{
_buffer.Dispose();
@ -285,4 +336,4 @@ namespace Microsoft.AspNetCore.WebUtilities
}
}
}
}
}

69
src/Microsoft.AspNetCore.WebUtilities/FormReader.cs
View File

@ -2,6 +2,7 @@
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System;
using System.Buffers;
using System.Collections.Generic;
using System.IO;
using System.Text;
@ -14,25 +15,40 @@ namespace Microsoft.AspNetCore.WebUtilities
/// <summary>
/// Used to read an 'application/x-www-form-urlencoded' form.
/// </summary>
public class FormReader
public class FormReader : IDisposable
{
private const int _rentedCharPoolLength = 8192;
private readonly TextReader _reader;
private readonly char[] _buffer = new char[1024];
private readonly char[] _buffer;
private readonly ArrayPool<char> _charPool;
private readonly StringBuilder _builder = new StringBuilder();
private int _bufferOffset;
private int _bufferCount;
private bool _disposed;
public FormReader(string data)
: this(data, ArrayPool<char>.Shared)
{
}
public FormReader(string data, ArrayPool<char> charPool)
{
if (data == null)
{
throw new ArgumentNullException(nameof(data));
}
_buffer = charPool.Rent(_rentedCharPoolLength);
_charPool = charPool;
_reader = new StringReader(data);
}
public FormReader(Stream stream, Encoding encoding)
: this(stream, encoding, ArrayPool<char>.Shared)
{
}
public FormReader(Stream stream, Encoding encoding, ArrayPool<char> charPool)
{
if (stream == null)
{
@ -44,6 +60,8 @@ namespace Microsoft.AspNetCore.WebUtilities
throw new ArgumentNullException(nameof(encoding));
}
_buffer = charPool.Rent(_rentedCharPoolLength);
_charPool = charPool;
_reader = new StreamReader(stream, encoding, detectEncodingFromByteOrderMarks: true, bufferSize: 1024 * 2, leaveOpen: true);
}
@ -167,17 +185,18 @@ namespace Microsoft.AspNetCore.WebUtilities
/// <returns>The collection containing the parsed HTTP form body.</returns>
public static Dictionary<string, StringValues> ReadForm(string text)
{
var reader = new FormReader(text);
var accumulator = new KeyValueAccumulator();
var pair = reader.ReadNextPair();
while (pair.HasValue)
using (var reader = new FormReader(text))
{
accumulator.Append(pair.Value.Key, pair.Value.Value);
pair = reader.ReadNextPair();
}
var accumulator = new KeyValueAccumulator();
var pair = reader.ReadNextPair();
while (pair.HasValue)
{
accumulator.Append(pair.Value.Key, pair.Value.Value);
pair = reader.ReadNextPair();
}
return accumulator.GetResults();
return accumulator.GetResults();
}
}
/// <summary>
@ -200,17 +219,27 @@ namespace Microsoft.AspNetCore.WebUtilities
/// <returns>The collection containing the parsed HTTP form body.</returns>
public static async Task<Dictionary<string, StringValues>> ReadFormAsync(Stream stream, Encoding encoding, CancellationToken cancellationToken = new CancellationToken())
{
var reader = new FormReader(stream, encoding);
var accumulator = new KeyValueAccumulator();
var pair = await reader.ReadNextPairAsync(cancellationToken);
while (pair.HasValue)
using (var reader = new FormReader(stream, encoding))
{
accumulator.Append(pair.Value.Key, pair.Value.Value);
pair = await reader.ReadNextPairAsync(cancellationToken);
}
var accumulator = new KeyValueAccumulator();
var pair = await reader.ReadNextPairAsync(cancellationToken);
while (pair.HasValue)
{
accumulator.Append(pair.Value.Key, pair.Value.Value);
pair = await reader.ReadNextPairAsync(cancellationToken);
}
return accumulator.GetResults();
return accumulator.GetResults();
}
}
public void Dispose()
{
if (!_disposed)
{
_disposed = true;
_charPool.Return(_buffer);
}
}
}
}

31
src/Microsoft.AspNetCore.WebUtilities/HttpRequestStreamReader.cs
View File

@ -34,40 +34,13 @@ namespace Microsoft.AspNetCore.WebUtilities
private bool _isBlocked;
public HttpRequestStreamReader(Stream stream, Encoding encoding)
: this(stream, encoding, DefaultBufferSize)
: this(stream, encoding, DefaultBufferSize, ArrayPool<byte>.Shared, ArrayPool<char>.Shared)
{
}
public HttpRequestStreamReader(Stream stream, Encoding encoding, int bufferSize)
: this(stream, encoding, bufferSize, ArrayPool<byte>.Shared, ArrayPool<char>.Shared)
{
if (stream == null)
{
throw new ArgumentNullException(nameof(stream));
}
if (!stream.CanRead)
{
throw new ArgumentException(Resources.HttpRequestStreamReader_StreamNotReadable, nameof(stream));
}
if (encoding == null)
{
throw new ArgumentNullException(nameof(encoding));
}
_stream = stream;
_encoding = encoding;
_decoder = encoding.GetDecoder();
if (bufferSize < MinBufferSize)
{
bufferSize = MinBufferSize;
}
_byteBufferSize = bufferSize;
_byteBuffer = new byte[bufferSize];
var maxCharsPerBuffer = encoding.GetMaxCharCount(bufferSize);
_charBuffer = new char[maxCharsPerBuffer];
}
public HttpRequestStreamReader(

30
src/Microsoft.AspNetCore.WebUtilities/HttpResponseStreamWriter.cs
View File

@ -34,39 +34,13 @@ namespace Microsoft.AspNetCore.WebUtilities
private int _charBufferCount;
public HttpResponseStreamWriter(Stream stream, Encoding encoding)
: this(stream, encoding, DefaultBufferSize)
: this(stream, encoding, DefaultBufferSize, ArrayPool<byte>.Shared, ArrayPool<char>.Shared)
{
}
public HttpResponseStreamWriter(Stream stream, Encoding encoding, int bufferSize)
: this(stream, encoding, bufferSize, ArrayPool<byte>.Shared, ArrayPool<char>.Shared)
{
if (stream == null)
{
throw new ArgumentNullException(nameof(stream));
}
if (!stream.CanWrite)
{
throw new ArgumentException(Resources.HttpResponseStreamWriter_StreamNotWritable, nameof(stream));
}
if (encoding == null)
{
throw new ArgumentNullException(nameof(encoding));
}
_stream = stream;
Encoding = encoding;
_charBufferSize = bufferSize;
if (bufferSize < MinBufferSize)
{
bufferSize = MinBufferSize;
}
_encoder = encoding.GetEncoder();
_byteBuffer = new byte[encoding.GetMaxByteCount(bufferSize)];
_charBuffer = new char[bufferSize];
}
public HttpResponseStreamWriter(

40
src/Microsoft.AspNetCore.WebUtilities/MultipartReaderStream.cs
View File

@ -2,6 +2,7 @@
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System;
using System.Buffers;
using System.Diagnostics;
using System.IO;
using System.Threading;
@ -13,6 +14,8 @@ namespace Microsoft.AspNetCore.WebUtilities
{
private readonly MultipartBoundary _boundary;
private readonly BufferedReadStream _innerStream;
private readonly ArrayPool<byte> _bytePool;
private readonly long _innerOffset;
private long _position;
private long _observedLength;
@ -24,6 +27,17 @@ namespace Microsoft.AspNetCore.WebUtilities
/// <param name="stream">The <see cref="BufferedReadStream"/>.</param>
/// <param name="boundary">The boundary pattern to use.</param>
public MultipartReaderStream(BufferedReadStream stream, MultipartBoundary boundary)
: this(stream, boundary, ArrayPool<byte>.Shared)
{
}
/// <summary>
/// Creates a stream that reads until it reaches the given boundary pattern.
/// </summary>
/// <param name="stream">The <see cref="BufferedReadStream"/>.</param>
/// <param name="boundary">The boundary pattern to use.</param>
/// <param name="bytePool">The ArrayPool pool to use for temporary byte arrays.</param>
public MultipartReaderStream(BufferedReadStream stream, MultipartBoundary boundary, ArrayPool<byte> bytePool)
{
if (stream == null)
{
@ -35,6 +49,7 @@ namespace Microsoft.AspNetCore.WebUtilities
throw new ArgumentNullException(nameof(boundary));
}
_bytePool = bytePool;
_innerStream = stream;
_innerOffset = _innerStream.CanSeek ? _innerStream.Position : 0;
_boundary = boundary;
@ -212,14 +227,18 @@ namespace Microsoft.AspNetCore.WebUtilities
read = _innerStream.Read(buffer, offset, Math.Min(count, matchOffset - bufferedData.Offset));
return UpdatePosition(read);
}
Debug.Assert(matchCount == _boundary.BoundaryBytes.Length);
var length = _boundary.BoundaryBytes.Length;
Debug.Assert(matchCount == length);
// "The boundary may be followed by zero or more characters of
// linear whitespace. It is then terminated by either another CRLF"
// or -- for the final boundary.
byte[] boundary = new byte[_boundary.BoundaryBytes.Length];
read = _innerStream.Read(boundary, 0, boundary.Length);
Debug.Assert(read == boundary.Length); // It should have all been buffered
var boundary = _bytePool.Rent(length);
read = _innerStream.Read(boundary, 0, length);
_bytePool.Return(boundary);
Debug.Assert(read == length); // It should have all been buffered
var remainder = _innerStream.ReadLine(lengthLimit: 100); // Whitespace may exceed the buffer.
remainder = remainder.Trim();
if (string.Equals("--", remainder, StringComparison.Ordinal))
@ -227,7 +246,6 @@ namespace Microsoft.AspNetCore.WebUtilities
FinalBoundaryFound = true;
}
Debug.Assert(FinalBoundaryFound || string.Equals(string.Empty, remainder, StringComparison.Ordinal), "Un-expected data found on the boundary line: " + remainder);
_finished = true;
return 0;
}
@ -264,14 +282,18 @@ namespace Microsoft.AspNetCore.WebUtilities
read = _innerStream.Read(buffer, offset, Math.Min(count, matchOffset - bufferedData.Offset));
return UpdatePosition(read);
}
Debug.Assert(matchCount == _boundary.BoundaryBytes.Length);
var length = _boundary.BoundaryBytes.Length;
Debug.Assert(matchCount == length);
// "The boundary may be followed by zero or more characters of
// linear whitespace. It is then terminated by either another CRLF"
// or -- for the final boundary.
byte[] boundary = new byte[_boundary.BoundaryBytes.Length];
read = _innerStream.Read(boundary, 0, boundary.Length);
Debug.Assert(read == boundary.Length); // It should have all been buffered
var boundary = _bytePool.Rent(length);
read = _innerStream.Read(boundary, 0, length);
_bytePool.Return(boundary);
Debug.Assert(read == length); // It should have all been buffered
var remainder = await _innerStream.ReadLineAsync(lengthLimit: 100, cancellationToken: cancellationToken); // Whitespace may exceed the buffer.
remainder = remainder.Trim();
if (string.Equals("--", remainder, StringComparison.Ordinal))

25
src/Microsoft.AspNetCore.WebUtilities/StreamHelperExtensions.cs
View File

@ -1,6 +1,7 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System.Buffers;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
@ -9,14 +10,28 @@ namespace Microsoft.AspNetCore.WebUtilities
{
public static class StreamHelperExtensions
{
public static async Task DrainAsync(this Stream stream, CancellationToken cancellationToken)
private const int _maxReadBufferSize = 1024 * 4;
public static Task DrainAsync(this Stream stream, CancellationToken cancellationToken)
{
return stream.DrainAsync(ArrayPool<byte>.Shared, cancellationToken);
}
public static async Task DrainAsync(this Stream stream, ArrayPool<byte> bytePool, CancellationToken cancellationToken)
{
byte[] buffer = new byte[1024];
cancellationToken.ThrowIfCancellationRequested();
while (await stream.ReadAsync(buffer, 0, buffer.Length, cancellationToken) > 0)
var buffer = bytePool.Rent(_maxReadBufferSize);
try
{
// Not all streams support cancellation directly.
cancellationToken.ThrowIfCancellationRequested();
while (await stream.ReadAsync(buffer, 0, buffer.Length, cancellationToken) > 0)
{
// Not all streams support cancellation directly.
cancellationToken.ThrowIfCancellationRequested();
}
}
finally
{
bytePool.Return(buffer);
}
}
}

11
src/Microsoft.Net.Http.Headers/ContentDispositionHeaderValue.cs
View File

@ -2,6 +2,7 @@
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System;
using System.Buffers;
using System.Collections.Generic;
using System.Diagnostics.Contracts;
using System.Globalization;
@ -578,12 +579,13 @@ namespace Microsoft.Net.Http.Headers
}
var decoded = new StringBuilder();
byte[] unescapedBytes = null;
try
{
var encoding = Encoding.GetEncoding(parts[0]);
var dataString = parts[2];
var unescapedBytes = new byte[dataString.Length];
unescapedBytes = ArrayPool<byte>.Shared.Rent(dataString.Length);
var unescapedBytesCount = 0;
for (var index = 0; index < dataString.Length; index++)
{
@ -615,6 +617,13 @@ namespace Microsoft.Net.Http.Headers
{
return false; // Unknown encoding or bad characters
}
finally
{
if (unescapedBytes != null)
{
ArrayPool<byte>.Shared.Return(unescapedBytes);
}
}
output = decoded.ToString();
return true;

7
src/Microsoft.Net.Http.Headers/project.json
View File

@ -18,7 +18,7 @@
},
"dependencies": { },
"frameworks": {
"netstandard1.0": {
"netstandard1.1": {
"dependencies": {
"System.Collections": "4.0.11-*",
"System.Diagnostics.Contracts": "4.0.1-*",
@ -26,10 +26,11 @@
"System.Linq": "4.1.0-*",
"System.Resources.ResourceManager": "4.0.1-*",
"System.Runtime.Extensions": "4.1.0-*",
"System.Text.Encoding": "4.0.11-*"
"System.Text.Encoding": "4.0.11-*",
"System.Buffers": "4.0.0-*"
},
"imports": [
"dotnet5.1"
"dotnet5.2"
]
}
}