diff --git a/src/Microsoft.AspNet.Server.Kestrel/Http/Listener.cs b/src/Microsoft.AspNet.Server.Kestrel/Http/Listener.cs index 98de2b440f..5bc7ebdeaf 100644 --- a/src/Microsoft.AspNet.Server.Kestrel/Http/Listener.cs +++ b/src/Microsoft.AspNet.Server.Kestrel/Http/Listener.cs @@ -69,7 +69,7 @@ namespace Microsoft.AspNet.Server.Kestrel.Http try { ListenSocket = new UvTcpHandle(); - ListenSocket.Init(Thread.Loop); + ListenSocket.Init(Thread.Loop, Thread.QueueCloseHandle); ListenSocket.Bind(new IPEndPoint(IPAddress.Any, port)); ListenSocket.Listen(10, _connectionCallback, this); tcs.SetResult(0); @@ -85,7 +85,7 @@ namespace Microsoft.AspNet.Server.Kestrel.Http private void OnConnection(UvStreamHandle listenSocket, int status) { var acceptSocket = new UvTcpHandle(); - acceptSocket.Init(Thread.Loop); + acceptSocket.Init(Thread.Loop, Thread.QueueCloseHandle); listenSocket.Accept(acceptSocket); var connection = new Connection(this, acceptSocket); diff --git a/src/Microsoft.AspNet.Server.Kestrel/Infrastructure/KestrelThread.cs b/src/Microsoft.AspNet.Server.Kestrel/Infrastructure/KestrelThread.cs index fae471eec5..5ef6f101a7 100644 --- a/src/Microsoft.AspNet.Server.Kestrel/Infrastructure/KestrelThread.cs +++ b/src/Microsoft.AspNet.Server.Kestrel/Infrastructure/KestrelThread.cs @@ -4,6 +4,7 @@ using Microsoft.AspNet.Server.Kestrel.Networking; using System; using System.Collections.Generic; +using System.Diagnostics; using System.Runtime.ExceptionServices; using System.Threading; using System.Threading.Tasks; @@ -21,6 +22,8 @@ namespace Microsoft.AspNet.Server.Kestrel UvAsyncHandle _post; Queue _workAdding = new Queue(); Queue _workRunning = new Queue(); + Queue _closeHandleAdding = new Queue(); + Queue _closeHandleRunning = new Queue(); object _workSync = new Object(); bool _stopImmediate = false; private ExceptionDispatchInfo _closeError; @@ -31,10 +34,13 @@ namespace Microsoft.AspNet.Server.Kestrel _loop = new UvLoopHandle(); _post = new UvAsyncHandle(); _thread = new Thread(ThreadStart); + QueueCloseHandle = PostCloseHandle; } public UvLoopHandle Loop { get { return _loop; } } + public Action, IntPtr> QueueCloseHandle { get; internal set; } + public Task StartAsync() { var tcs = new TaskCompletionSource(); @@ -92,6 +98,15 @@ namespace Microsoft.AspNet.Server.Kestrel return tcs.Task; } + private void PostCloseHandle(Action callback, IntPtr handle) + { + lock (_workSync) + { + _closeHandleAdding.Enqueue(new CloseHandle { Callback = callback, Handle = handle }); + } + _post.Send(); + } + private void ThreadStart(object parameter) { var tcs = (TaskCompletionSource)parameter; @@ -119,7 +134,7 @@ namespace Microsoft.AspNet.Server.Kestrel _post.Reference(); _post.DangerousClose(); _engine.Libuv.walk( - _loop, + _loop, (ptr, arg) => { var handle = UvMemory.FromIntPtr(ptr); @@ -138,12 +153,19 @@ namespace Microsoft.AspNet.Server.Kestrel private void OnPost() { - var queue = _workAdding; + DoPostWork(); + DoPostCloseHandle(); + } + + private void DoPostWork() + { + Queue queue; lock (_workSync) { + queue = _workAdding; _workAdding = _workRunning; + _workRunning = queue; } - _workRunning = queue; while (queue.Count != 0) { var work = queue.Dequeue(); @@ -156,7 +178,7 @@ namespace Microsoft.AspNet.Server.Kestrel tcs => { ((TaskCompletionSource)tcs).SetResult(0); - }, + }, work.Completion); } } @@ -168,11 +190,33 @@ namespace Microsoft.AspNet.Server.Kestrel } else { - // TODO: unobserved exception? + Trace.WriteLine("KestrelThread.DoPostWork " + ex.ToString()); } } } } + private void DoPostCloseHandle() + { + Queue queue; + lock (_workSync) + { + queue = _closeHandleAdding; + _closeHandleAdding = _closeHandleRunning; + _closeHandleRunning = queue; + } + while (queue.Count != 0) + { + var closeHandle = queue.Dequeue(); + try + { + closeHandle.Callback(closeHandle.Handle); + } + catch (Exception ex) + { + Trace.WriteLine("KestrelThread.DoPostCloseHandle " + ex.ToString()); + } + } + } private struct Work { @@ -180,5 +224,10 @@ namespace Microsoft.AspNet.Server.Kestrel public object State; public TaskCompletionSource Completion; } + private struct CloseHandle + { + public Action Callback; + public IntPtr Handle; + } } } diff --git a/src/Microsoft.AspNet.Server.Kestrel/Networking/Libuv.cs b/src/Microsoft.AspNet.Server.Kestrel/Networking/Libuv.cs index 18e9478f86..c777af7998 100644 --- a/src/Microsoft.AspNet.Server.Kestrel/Networking/Libuv.cs +++ b/src/Microsoft.AspNet.Server.Kestrel/Networking/Libuv.cs @@ -128,6 +128,10 @@ namespace Microsoft.AspNet.Server.Kestrel.Networking handle.Validate(closed: true); _uv_close(handle.InternalGetHandle(), close_cb); } + public void close(IntPtr handle, uv_close_cb close_cb) + { + _uv_close(handle, close_cb); + } [UnmanagedFunctionPointer(CallingConvention.Cdecl)] public delegate void uv_async_cb(IntPtr handle); diff --git a/src/Microsoft.AspNet.Server.Kestrel/Networking/UcAsyncHandle.cs b/src/Microsoft.AspNet.Server.Kestrel/Networking/UcAsyncHandle.cs index 7709ebd50c..31e8f9b054 100644 --- a/src/Microsoft.AspNet.Server.Kestrel/Networking/UcAsyncHandle.cs +++ b/src/Microsoft.AspNet.Server.Kestrel/Networking/UcAsyncHandle.cs @@ -12,7 +12,11 @@ namespace Microsoft.AspNet.Server.Kestrel.Networking public void Init(UvLoopHandle loop, Action callback) { - CreateHandle(loop, loop.Libuv.handle_size(Libuv.HandleType.ASYNC)); + CreateMemory( + loop.Libuv, + loop.ThreadId, + loop.Libuv.handle_size(Libuv.HandleType.ASYNC)); + _callback = callback; _uv.async_init(loop, this, _uv_async_cb); } diff --git a/src/Microsoft.AspNet.Server.Kestrel/Networking/UvHandle.cs b/src/Microsoft.AspNet.Server.Kestrel/Networking/UvHandle.cs index d08a6910d7..fe12f19330 100644 --- a/src/Microsoft.AspNet.Server.Kestrel/Networking/UvHandle.cs +++ b/src/Microsoft.AspNet.Server.Kestrel/Networking/UvHandle.cs @@ -2,20 +2,40 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; +using System.Threading; namespace Microsoft.AspNet.Server.Kestrel.Networking { public abstract class UvHandle : UvMemory { - static Libuv.uv_close_cb _close_cb = DestroyHandle; + static Libuv.uv_close_cb _destroyMemory = DestroyMemory; + Action, IntPtr> _queueCloseHandle; + + unsafe protected void CreateHandle( + Libuv uv, + int threadId, + int size, + Action, IntPtr> queueCloseHandle) + { + _queueCloseHandle = queueCloseHandle; + CreateMemory(uv, threadId, size); + } protected override bool ReleaseHandle() { var memory = handle; if (memory != IntPtr.Zero) { - _uv.close(this, _close_cb); handle = IntPtr.Zero; + + if (Thread.CurrentThread.ManagedThreadId == ThreadId) + { + _uv.close(memory, _destroyMemory); + } + else + { + _queueCloseHandle(memory2 => _uv.close(memory2, _destroyMemory), memory); + } } return true; } diff --git a/src/Microsoft.AspNet.Server.Kestrel/Networking/UvLoopHandle.cs b/src/Microsoft.AspNet.Server.Kestrel/Networking/UvLoopHandle.cs index 9ea5cb3d8a..a13ae2ccce 100644 --- a/src/Microsoft.AspNet.Server.Kestrel/Networking/UvLoopHandle.cs +++ b/src/Microsoft.AspNet.Server.Kestrel/Networking/UvLoopHandle.cs @@ -2,6 +2,7 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; +using System.Threading; namespace Microsoft.AspNet.Server.Kestrel.Networking { @@ -9,7 +10,11 @@ namespace Microsoft.AspNet.Server.Kestrel.Networking { public void Init(Libuv uv) { - CreateHandle(uv, uv.loop_size()); + CreateMemory( + uv, + Thread.CurrentThread.ManagedThreadId, + uv.loop_size()); + _uv.loop_init(this); } @@ -30,7 +35,7 @@ namespace Microsoft.AspNet.Server.Kestrel.Networking { _uv.loop_close(this); handle = IntPtr.Zero; - DestroyHandle(memory); + DestroyMemory(memory); } return true; } diff --git a/src/Microsoft.AspNet.Server.Kestrel/Networking/UvMemory.cs b/src/Microsoft.AspNet.Server.Kestrel/Networking/UvMemory.cs index e43e600ec3..c3b3c6ce28 100644 --- a/src/Microsoft.AspNet.Server.Kestrel/Networking/UvMemory.cs +++ b/src/Microsoft.AspNet.Server.Kestrel/Networking/UvMemory.cs @@ -14,7 +14,7 @@ namespace Microsoft.AspNet.Server.Kestrel.Networking public abstract class UvMemory : SafeHandle { protected Libuv _uv; - int _threadId; + private int _threadId; public UvMemory() : base(IntPtr.Zero, true) { @@ -30,19 +30,36 @@ namespace Microsoft.AspNet.Server.Kestrel.Networking } } - unsafe protected void CreateHandle(Libuv uv, int size) + public int ThreadId + { + get + { + return _threadId; + } + private set + { + _threadId = value; + } + } + + unsafe protected void CreateMemory(Libuv uv, int threadId, int size) { _uv = uv; - _threadId = Thread.CurrentThread.ManagedThreadId; - + ThreadId = threadId; + handle = Marshal.AllocCoTaskMem(size); *(IntPtr*)handle = GCHandle.ToIntPtr(GCHandle.Alloc(this, GCHandleType.Weak)); } - protected void CreateHandle(UvLoopHandle loop, int size) + unsafe protected static void DestroyMemory(IntPtr memory) { - CreateHandle(loop._uv, size); - _threadId = loop._threadId; + var gcHandlePtr = *(IntPtr*)memory; + if (gcHandlePtr != IntPtr.Zero) + { + var gcHandle = GCHandle.FromIntPtr(gcHandlePtr); + gcHandle.Free(); + } + Marshal.FreeCoTaskMem(memory); } internal IntPtr InternalGetHandle() @@ -57,16 +74,6 @@ namespace Microsoft.AspNet.Server.Kestrel.Networking Trace.Assert(_threadId == Thread.CurrentThread.ManagedThreadId, "ThreadId is incorrect"); } - unsafe protected static void DestroyHandle(IntPtr memory) - { - var gcHandlePtr = *(IntPtr*)memory; - if (gcHandlePtr != IntPtr.Zero) - { - GCHandle.FromIntPtr(gcHandlePtr).Free(); - } - Marshal.FreeCoTaskMem(memory); - } - unsafe public static THandle FromIntPtr(IntPtr handle) { GCHandle gcHandle = GCHandle.FromIntPtr(*(IntPtr*)handle); diff --git a/src/Microsoft.AspNet.Server.Kestrel/Networking/UvShutdownReq.cs b/src/Microsoft.AspNet.Server.Kestrel/Networking/UvShutdownReq.cs index 2606285aa0..ef280549c8 100644 --- a/src/Microsoft.AspNet.Server.Kestrel/Networking/UvShutdownReq.cs +++ b/src/Microsoft.AspNet.Server.Kestrel/Networking/UvShutdownReq.cs @@ -17,7 +17,10 @@ namespace Microsoft.AspNet.Server.Kestrel.Networking public void Init(UvLoopHandle loop) { - CreateHandle(loop, loop.Libuv.req_size(Libuv.RequestType.SHUTDOWN)); + CreateMemory( + loop.Libuv, + loop.ThreadId, + loop.Libuv.req_size(Libuv.RequestType.SHUTDOWN)); } public void Shutdown(UvStreamHandle handle, Action callback, object state) diff --git a/src/Microsoft.AspNet.Server.Kestrel/Networking/UvTcpHandle.cs b/src/Microsoft.AspNet.Server.Kestrel/Networking/UvTcpHandle.cs index 96d5ddbbc9..e3f90ad846 100644 --- a/src/Microsoft.AspNet.Server.Kestrel/Networking/UvTcpHandle.cs +++ b/src/Microsoft.AspNet.Server.Kestrel/Networking/UvTcpHandle.cs @@ -10,7 +10,21 @@ namespace Microsoft.AspNet.Server.Kestrel.Networking { public void Init(UvLoopHandle loop) { - CreateHandle(loop, loop.Libuv.handle_size(Libuv.HandleType.TCP)); + CreateMemory( + loop.Libuv, + loop.ThreadId, + loop.Libuv.handle_size(Libuv.HandleType.TCP)); + + _uv.tcp_init(loop, this); + } + + public void Init(UvLoopHandle loop, Action, IntPtr> queueCloseHandle) + { + CreateHandle( + loop.Libuv, + loop.ThreadId, + loop.Libuv.handle_size(Libuv.HandleType.TCP), queueCloseHandle); + _uv.tcp_init(loop, this); } diff --git a/src/Microsoft.AspNet.Server.Kestrel/Networking/UvWriteRequest.cs b/src/Microsoft.AspNet.Server.Kestrel/Networking/UvWriteRequest.cs index dee7b6bb66..60228c4eca 100644 --- a/src/Microsoft.AspNet.Server.Kestrel/Networking/UvWriteRequest.cs +++ b/src/Microsoft.AspNet.Server.Kestrel/Networking/UvWriteRequest.cs @@ -27,7 +27,10 @@ namespace Microsoft.AspNet.Server.Kestrel.Networking { var requestSize = loop.Libuv.req_size(Libuv.RequestType.WRITE); var bufferSize = Marshal.SizeOf(typeof(Libuv.uv_buf_t)) * BUFFER_COUNT; - CreateHandle(loop, requestSize + bufferSize); + CreateMemory( + loop.Libuv, + loop.ThreadId, + requestSize + bufferSize); _bufs = handle + requestSize; } @@ -37,17 +40,23 @@ namespace Microsoft.AspNet.Server.Kestrel.Networking Action callback, object state) { + // add GCHandle to keeps this SafeHandle alive while request processing + _pins.Add(GCHandle.Alloc(this, GCHandleType.Normal)); + var pBuffers = (Libuv.uv_buf_t*)_bufs; var nBuffers = bufs.Count; if (nBuffers > BUFFER_COUNT) { + // create and pin buffer array when it's larger than the pre-allocated one var bufArray = new Libuv.uv_buf_t[nBuffers]; var gcHandle = GCHandle.Alloc(bufArray, GCHandleType.Pinned); _pins.Add(gcHandle); pBuffers = (Libuv.uv_buf_t*)gcHandle.AddrOfPinnedObject(); } + for (var index = 0; index != nBuffers; ++index) { + // create and pin each segment being written var buf = bufs.Array[bufs.Offset + index]; var gcHandle = GCHandle.Alloc(buf.Array, GCHandleType.Pinned); @@ -56,6 +65,7 @@ namespace Microsoft.AspNet.Server.Kestrel.Networking gcHandle.AddrOfPinnedObject() + buf.Offset, buf.Count); } + _callback = callback; _state = state; _uv.write(this, handle, pBuffers, nBuffers, _uv_write_cb); @@ -97,7 +107,7 @@ namespace Microsoft.AspNet.Server.Kestrel.Networking { protected override bool ReleaseHandle() { - DestroyHandle(handle); + DestroyMemory(handle); handle = IntPtr.Zero; return true; }