Refactor tests to fix false-positive tests passes

.travis.yml

@@ -19,7 +19,6 @@ mono:
 os:
   - linux
   - osx
-osx_image: xcode7.1
 branches:
   only:
     - master

test/Microsoft.DotNet.Watcher.Tools.FunctionalTests/AppWithDepsTests.cs

@@ -3,80 +3,51 @@
 
 using System;
 using System.IO;
+using System.Threading.Tasks;
 using Xunit;
 using Xunit.Abstractions;
 
 namespace Microsoft.DotNet.Watcher.Tools.FunctionalTests
 {
-    public class AppWithDepsTests
+    public class AppWithDepsTests : IDisposable
     {
-        private static readonly TimeSpan _defaultTimeout = TimeSpan.FromSeconds(30);
+        private readonly AppWithDeps _app;
-
-        private readonly ITestOutputHelper _logger;
 
         public AppWithDepsTests(ITestOutputHelper logger)
         {
-            _logger = logger;
+            _app = new AppWithDeps(logger);
+            _app.Prepare();
         }
 
-        // Change a file included in compilation
         [Fact]
-        public void ChangeFileInDependency()
+        public async Task ChangeFileInDependency()
         {
-            using (var scenario = new AppWithDepsScenario(_logger))
+            await _app.StartWatcher().OrTimeout();
-            {
-                scenario.Start();
-                using (var wait = new WaitForFileToChange(scenario.StartedFile))
-                {
-                    var fileToChange = Path.Combine(scenario.DependencyFolder, "Foo.cs");
-                    var programCs = File.ReadAllText(fileToChange);
-                    File.WriteAllText(fileToChange, programCs);
 
-                    wait.Wait(_defaultTimeout,
+            var fileToChange = Path.Combine(_app.DependencyFolder, "Foo.cs");
-                        expectedToChange: true,
+            var programCs = File.ReadAllText(fileToChange);
-                        errorMessage: $"Process did not restart because {scenario.StartedFile} was not changed");
+            File.WriteAllText(fileToChange, programCs);
-                }
+
-            }
+            await _app.HasRestarted().OrTimeout();
         }
 
-        private class AppWithDepsScenario : DotNetWatchScenario
+        public void Dispose()
+        {
+            _app.Dispose();
+        }
+
+        private class AppWithDeps : WatchableApp
         {
-            private const string AppWithDeps = "AppWithDeps";
             private const string Dependency = "Dependency";
 
-            public AppWithDepsScenario(ITestOutputHelper logger)
+            public AppWithDeps(ITestOutputHelper logger)
-                : base(logger)
+                : base("AppWithDeps", logger)
             {
-                StatusFile = Path.Combine(Scenario.TempFolder, "status");
-                StartedFile = StatusFile + ".started";
-
-                Scenario.AddTestProjectFolder(AppWithDeps);
                 Scenario.AddTestProjectFolder(Dependency);
 
-                Scenario.Restore(AppWithDeps); // restore3 should be transitive
-
-                AppWithDepsFolder = Path.Combine(Scenario.WorkFolder, AppWithDeps);
                 DependencyFolder = Path.Combine(Scenario.WorkFolder, Dependency);
             }
 
-            public void Start()
-            {
-                // Wait for the process to start
-                using (var wait = new WaitForFileToChange(StatusFile))
-                {
-                    RunDotNetWatch(new[] { "run", StatusFile }, Path.Combine(Scenario.WorkFolder, AppWithDeps));
-
-                    wait.Wait(_defaultTimeout,
-                        expectedToChange: true,
-                        errorMessage: $"File not created: {StatusFile}");
-                }
-
-                Waiters.WaitForFileToBeReadable(StatusFile, _defaultTimeout);
-            }
-
-            public string StatusFile { get; private set; }
-            public string StartedFile { get; private set; }
-            public string AppWithDepsFolder { get; private set; }
             public string DependencyFolder { get; private set; }
         }
     }

test/Microsoft.DotNet.Watcher.Tools.FunctionalTests/AwaitableProcess.cs

@@ -0,0 +1,102 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Diagnostics;
+using System.IO;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+using System.Threading.Tasks.Dataflow;
+using Microsoft.DotNet.Cli.Utils;
+using Microsoft.Extensions.Internal;
+using Xunit.Abstractions;
+
+namespace Microsoft.DotNet.Watcher.Tools.FunctionalTests
+{
+    public class AwaitableProcess : IDisposable
+    {
+        private Process _process;
+        private readonly ProcessSpec _spec;
+        private BufferBlock<string> _source;
+        private ITestOutputHelper _logger;
+        private int _reading;
+
+        public AwaitableProcess(ProcessSpec spec, ITestOutputHelper logger)
+        {
+            _spec = spec;
+            _logger = logger;
+        }
+
+        public void Start()
+        {
+            if (_process != null)
+            {
+                throw new InvalidOperationException("Already started");
+            }
+
+            var psi = new ProcessStartInfo
+            {
+                UseShellExecute = false,
+                FileName = _spec.Executable,
+                WorkingDirectory = _spec.WorkingDirectory,
+                Arguments = ArgumentEscaper.EscapeAndConcatenateArgArrayForProcessStart(_spec.Arguments),
+                RedirectStandardOutput = true,
+                RedirectStandardError = true
+            };
+            _process = Process.Start(psi);
+            _logger.WriteLine($"{DateTime.Now}: process start: '{psi.FileName} {psi.Arguments}'");
+            StartProcessingOutput(_process.StandardOutput);
+            StartProcessingOutput(_process.StandardError);;
+        }
+
+        public Task<string> GetOutputLineAsync(string message)
+            => GetOutputLineAsync(m => message == m);
+
+        public async Task<string> GetOutputLineAsync(Predicate<string> predicate)
+        {
+            while (!_source.Completion.IsCompleted)
+            {
+                while (await _source.OutputAvailableAsync())
+                {
+                    var next = await _source.ReceiveAsync();
+                    _logger.WriteLine($"{DateTime.Now}: recv: '{next}'");
+                    if (predicate(next))
+                    {
+                        return next;
+                    }
+                }
+            }
+
+            return null;
+        }
+
+        private void StartProcessingOutput(StreamReader streamReader)
+        {
+            _source = _source ?? new BufferBlock<string>();
+            Interlocked.Increment(ref _reading);
+            Task.Run(() =>
+            {
+                string line;
+                while ((line = streamReader.ReadLine()) != null)
+                {
+                    _logger.WriteLine($"{DateTime.Now} post: {line}");
+                    _source.Post(line);
+                }
+
+                if (Interlocked.Decrement(ref _reading) <= 0)
+                {
+                    _source.Complete();
+                }
+            }).ConfigureAwait(false);
+        }
+
+        public void Dispose()
+        {
+            if (_process != null && !_process.HasExited)
+            {
+                _process.KillTree();
+            }
+        }
+    }
+}

test/Microsoft.DotNet.Watcher.Tools.FunctionalTests/GlobbingAppTests.cs

@@ -2,227 +2,134 @@
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
-using System.Diagnostics;
 using System.IO;
-using System.Threading;
+using System.Linq;
+using System.Threading.Tasks;
 using Xunit;
 using Xunit.Abstractions;
 
 namespace Microsoft.DotNet.Watcher.Tools.FunctionalTests
 {
-    public class GlobbingAppTests
+    public class GlobbingAppTests : IDisposable
     {
-        private static readonly TimeSpan _defaultTimeout = TimeSpan.FromSeconds(30);
+        private GlobbingApp _app;
-
-        private static readonly TimeSpan _negativeTestWaitTime = TimeSpan.FromSeconds(10);
-
-        private readonly ITestOutputHelper _logger;
-
         public GlobbingAppTests(ITestOutputHelper logger)
         {
-            _logger = logger;
+            _app = new GlobbingApp(logger);
+            _app.Prepare();
+        }
+
+        [Theory]
+        [InlineData(true)]
+        [InlineData(false)]
+        public async Task ChangeCompiledFile(bool usePollingWatcher)
+        {
+            await _app.StartWatcher().OrTimeout();
+
+            var types = await _app.GetCompiledAppDefinedTypes().OrTimeout();
+            Assert.Equal(2, types);
+
+            var fileToChange = Path.Combine(_app.SourceDirectory, "include", "Foo.cs");
+            var programCs = File.ReadAllText(fileToChange);
+            File.WriteAllText(fileToChange, programCs);
+
+            await _app.HasRestarted().OrTimeout();
+            types = await _app.GetCompiledAppDefinedTypes().OrTimeout();
+            Assert.Equal(2, types);
+        }
+
+        [Fact(Skip = "Broken. See https://github.com/aspnet/DotNetTools/issues/212")]
+        public async Task AddCompiledFile()
+        {
+            await _app.StartWatcher().OrTimeout();
+
+            var types = await _app.GetCompiledAppDefinedTypes().OrTimeout();
+            Assert.Equal(2, types);
+
+            var fileToChange = Path.Combine(_app.SourceDirectory, "include", "Bar.cs");
+            File.WriteAllText(fileToChange, "public class Bar {}");
+
+            await _app.HasRestarted().OrTimeout();
+            types = await _app.GetCompiledAppDefinedTypes().OrTimeout();
+            Assert.Equal(3, types);
+        }
+
+        // TODO re-enable when MSBuild is updated. See https://github.com/aspnet/DotNetTools/issues/224
+        [Fact(Skip = "Broken. See https://github.com/Microsoft/msbuild/issues/701")]
+        public async Task DeleteCompiledFile()
+        {
+            await _app.StartWatcher().OrTimeout();
+
+            var types = await _app.GetCompiledAppDefinedTypes().OrTimeout();
+            Assert.Equal(2, types);
+
+            var fileToChange = Path.Combine(_app.SourceDirectory, "include", "Foo.cs");
+            File.Delete(fileToChange);
+
+            await _app.HasRestarted().OrTimeout();
+            types = await _app.GetCompiledAppDefinedTypes().OrTimeout();
+            Assert.Equal(1, types);
+        }
+
+        // TODO re-enable when MSBuild is updated. See https://github.com/aspnet/DotNetTools/issues/224
+        [Fact(Skip = "Broken. See https://github.com/Microsoft/msbuild/issues/701")]
+        public async Task DeleteSourceFolder()
+        {
+            await _app.StartWatcher().OrTimeout();
+
+            var types = await _app.GetCompiledAppDefinedTypes().OrTimeout();
+            Assert.Equal(2, types);
+
+            var folderToDelete = Path.Combine(_app.SourceDirectory, "include");
+            Directory.Delete(folderToDelete, recursive: true);
+
+            await _app.HasRestarted().OrTimeout();
+            types = await _app.GetCompiledAppDefinedTypes().OrTimeout();
+            Assert.Equal(1, types);
         }
 
         [Fact]
-        public void ChangeCompiledFile_PollingWatcher()
+        public async Task RenameCompiledFile()
         {
-            ChangeCompiledFile(usePollingWatcher: true);
+            await _app.StartWatcher().OrTimeout();
+
+            var oldFile = Path.Combine(_app.SourceDirectory, "include", "Foo.cs");
+            var newFile = Path.Combine(_app.SourceDirectory, "include", "Foo_new.cs");
+            File.Move(oldFile, newFile);
+
+            await _app.HasRestarted().OrTimeout();
         }
 
         [Fact]
-        public void ChangeCompiledFile_DotNetWatcher()
+        public async Task ChangeExcludedFile()
         {
-            ChangeCompiledFile(usePollingWatcher: false);
+            await _app.StartWatcher().OrTimeout();
+
+            var changedFile = Path.Combine(_app.SourceDirectory, "exclude", "Baz.cs");
+            File.WriteAllText(changedFile, "");
+
+            var restart = _app.HasRestarted();
+            var finished = await Task.WhenAny(Task.Delay(TimeSpan.FromSeconds(10)), restart);
+            Assert.NotSame(restart, finished);
         }
 
-        // Change a file included in compilation
+        public void Dispose()
-        private void ChangeCompiledFile(bool usePollingWatcher)
         {
-            using (var scenario = new GlobbingAppScenario(_logger))
+            _app.Dispose();
-            using (var wait = new WaitForFileToChange(scenario.StartedFile))
+        }
+
+        private class GlobbingApp : WatchableApp
+        {
+            public GlobbingApp(ITestOutputHelper logger)
+                : base("GlobbingApp", logger)
             {
-                scenario.UsePollingWatcher = usePollingWatcher;
-
-                scenario.Start();
-
-                var fileToChange = Path.Combine(scenario.TestAppFolder, "include", "Foo.cs");
-                var programCs = File.ReadAllText(fileToChange);
-                File.WriteAllText(fileToChange, programCs);
-
-                wait.Wait(_defaultTimeout,
-                    expectedToChange: true,
-                    errorMessage: $"Process did not restart because {scenario.StartedFile} was not changed");
             }
 
-        }
+            public async Task<int> GetCompiledAppDefinedTypes()
-
-        // Add a file to a folder included in compilation
-        [Fact]
-        public void AddCompiledFile()
-        {
-            // Add a file in a folder that's included in compilation
-            using (var scenario = new GlobbingAppScenario(_logger))
-            using (var wait = new WaitForFileToChange(scenario.StartedFile))
             {
-                scenario.Start();
+                var definedTypesMessage = await Process.GetOutputLineAsync(m => m.StartsWith("Defined types = "));
-
+                return int.Parse(definedTypesMessage.Split('=').Last());
-                var fileToChange = Path.Combine(scenario.TestAppFolder, "include", "Bar.cs");
-                File.WriteAllText(fileToChange, "");
-
-                wait.Wait(_defaultTimeout,
-                    expectedToChange: true,
-                    errorMessage: $"Process did not restart because {scenario.StartedFile} was not changed");
             }
         }
-
-        // Delete a file included in compilation
-        [Fact]
-        public void DeleteCompiledFile()
-        {
-            using (var scenario = new GlobbingAppScenario(_logger))
-            using (var wait = new WaitForFileToChange(scenario.StartedFile))
-            {
-                scenario.Start();
-
-                var fileToChange = Path.Combine(scenario.TestAppFolder, "include", "Foo.cs");
-                File.Delete(fileToChange);
-
-                wait.Wait(_defaultTimeout,
-                    expectedToChange: true,
-                    errorMessage: $"Process did not restart because {scenario.StartedFile} was not changed");
-            }
-        }
-
-        // Delete an entire folder
-        [Fact]
-        public void DeleteSourceFolder()
-        {
-            using (var scenario = new GlobbingAppScenario(_logger))
-            using (var wait = new WaitForFileToChange(scenario.StartedFile))
-            {
-                scenario.Start();
-
-                var folderToDelete = Path.Combine(scenario.TestAppFolder, "include");
-                Directory.Delete(folderToDelete, recursive: true);
-
-                wait.Wait(_defaultTimeout,
-                    expectedToChange: true,
-                    errorMessage: $"Process did not restart because {scenario.StartedFile} was not changed");
-            }
-        }
-
-        // Rename a file included in compilation
-        [Fact]
-        public void RenameCompiledFile()
-        {
-            using (var scenario = new GlobbingAppScenario(_logger))
-            using (var wait = new WaitForFileToChange(scenario.StatusFile))
-            {
-                scenario.Start();
-
-                var oldFile = Path.Combine(scenario.TestAppFolder, "include", "Foo.cs");
-                var newFile = Path.Combine(scenario.TestAppFolder, "include", "Foo_new.cs");
-                File.Move(oldFile, newFile);
-
-                wait.Wait(_defaultTimeout,
-                    expectedToChange: true,
-                    errorMessage: $"Process did not restart because {scenario.StartedFile} was not changed");
-            }
-        }
-
-        [Fact]
-        public void ChangeNonCompiledFile_PollingWatcher()
-        {
-            ChangeNonCompiledFile(usePollingWatcher: true);
-        }
-
-        [Fact]
-        public void ChangeNonCompiledFile_DotNetWatcher()
-        {
-            ChangeNonCompiledFile(usePollingWatcher: false);
-        }
-
-        // Add a file that's in a included folder but not matching the globbing pattern
-        private void ChangeNonCompiledFile(bool usePollingWatcher)
-        {
-            using (var scenario = new GlobbingAppScenario(_logger))
-            {
-                scenario.UsePollingWatcher = usePollingWatcher;
-
-                scenario.Start();
-
-                var ids = File.ReadAllLines(scenario.StatusFile);
-                var procId = int.Parse(ids[0]);
-
-                var changedFile = Path.Combine(scenario.TestAppFolder, "include", "not_compiled.css");
-                File.WriteAllText(changedFile, "");
-
-                Console.WriteLine($"Waiting {_negativeTestWaitTime.TotalSeconds} seconds to see if the app restarts");
-                Waiters.WaitForProcessToStop(
-                    procId,
-                    _negativeTestWaitTime,
-                    expectedToStop: false,
-                    errorMessage: "Test app restarted");
-            }
-        }
-
-        // Change a file that's in an excluded folder
-        [Fact]
-        public void ChangeExcludedFile()
-        {
-            using (var scenario = new GlobbingAppScenario(_logger))
-            {
-                scenario.Start();
-
-                var ids = File.ReadAllLines(scenario.StatusFile);
-                var procId = int.Parse(ids[0]);
-
-                var changedFile = Path.Combine(scenario.TestAppFolder, "exclude", "Baz.cs");
-                File.WriteAllText(changedFile, "");
-
-                Console.WriteLine($"Waiting {_negativeTestWaitTime.TotalSeconds} seconds to see if the app restarts");
-                Waiters.WaitForProcessToStop(
-                    procId,
-                    _negativeTestWaitTime,
-                    expectedToStop: false,
-                    errorMessage: "Test app restarted");
-            }
-        }
-
-        private class GlobbingAppScenario : DotNetWatchScenario
-        {
-            private const string TestAppName = "GlobbingApp";
-
-            public GlobbingAppScenario(ITestOutputHelper logger)
-                : base(logger)
-            {
-                StatusFile = Path.Combine(Scenario.TempFolder, "status");
-                StartedFile = StatusFile + ".started";
-
-                Scenario.AddTestProjectFolder(TestAppName);
-                Scenario.Restore(TestAppName);
-
-                TestAppFolder = Path.Combine(Scenario.WorkFolder, TestAppName);
-            }
-
-            public void Start()
-            {
-                // Wait for the process to start
-                using (var wait = new WaitForFileToChange(StartedFile))
-                {
-                    RunDotNetWatch(new[] { "run", StatusFile }, Path.Combine(Scenario.WorkFolder, TestAppName));
-
-                    wait.Wait(_defaultTimeout,
-                        expectedToChange: true,
-                        errorMessage: $"File not created: {StartedFile}");
-                }
-
-                Waiters.WaitForFileToBeReadable(StartedFile, _defaultTimeout);
-            }
-
-            public string StatusFile { get; private set; }
-            public string StartedFile { get; private set; }
-            public string TestAppFolder { get; private set; }
-        }
     }
 }

test/Microsoft.DotNet.Watcher.Tools.FunctionalTests/NoDepsAppTests.cs

@@ -2,132 +2,63 @@
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
-using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
-using System.Threading;
+using System.Threading.Tasks;
 using Xunit;
 using Xunit.Abstractions;
 
 namespace Microsoft.DotNet.Watcher.Tools.FunctionalTests
 {
-    public class NoDepsAppTests
+    public class NoDepsAppTests : IDisposable
     {
-        private static readonly TimeSpan _defaultTimeout = TimeSpan.FromSeconds(30);
+        private readonly WatchableApp _app;
-        private readonly ITestOutputHelper _logger;
 
         public NoDepsAppTests(ITestOutputHelper logger)
         {
-            _logger = logger;
+            _app = new WatchableApp("NoDepsApp", logger);
+            _app.Prepare();
         }
 
         [Fact]
-        public void RestartProcessOnFileChange()
+        public async Task RestartProcessOnFileChange()
         {
-            using (var scenario = new NoDepsAppScenario(_logger))
+            await _app.StartWatcher(new[] { "--no-exit" }).OrTimeout();
-            {
+            var pid = await _app.GetProcessId().OrTimeout();
-                // Wait for the process to start
-                using (var wait = new WaitForFileToChange(scenario.StartedFile))
-                {
-                    scenario.RunDotNetWatch(new[] { "run", scenario.StatusFile, "--no-exit" });
 
-                    wait.Wait(_defaultTimeout,
+            // Then wait for it to restart when we change a file
-                        expectedToChange: true,
+            var fileToChange = Path.Combine(_app.SourceDirectory, "Program.cs");
-                        errorMessage: $"File not created: {scenario.StartedFile}");
+            var programCs = File.ReadAllText(fileToChange);
-                }
+            File.WriteAllText(fileToChange, programCs);
 
-                // Then wait for it to restart when we change a file
+            await _app.HasRestarted().OrTimeout();
-                using (var wait = new WaitForFileToChange(scenario.StartedFile))
+            var pid2 = await _app.GetProcessId().OrTimeout();
-                {
+            Assert.NotEqual(pid, pid2);
-                    var fileToChange = Path.Combine(scenario.TestAppFolder, "Program.cs");
-                    var programCs = File.ReadAllText(fileToChange);
-                    File.WriteAllText(fileToChange, programCs);
 
-                    wait.Wait(_defaultTimeout,
+            // first app should have shut down
-                        expectedToChange: true,
+            Assert.Throws<ArgumentException>(() => Process.GetProcessById(pid));
-                        errorMessage: $"Process did not restart because {scenario.StartedFile} was not changed");
-                }
-
-                // Check that the first child process is no longer running
-                Waiters.WaitForFileToBeReadable(scenario.StatusFile, _defaultTimeout);
-                var ids = File.ReadAllLines(scenario.StatusFile);
-                var firstProcessId = int.Parse(ids[0]);
-                Waiters.WaitForProcessToStop(
-                    firstProcessId,
-                    TimeSpan.FromSeconds(1),
-                    expectedToStop: true,
-                    errorMessage: $"PID: {firstProcessId} is still alive");
-            }
         }
 
         [Fact]
-        public void RestartProcessThatTerminatesAfterFileChange()
+        public async Task RestartProcessThatTerminatesAfterFileChange()
         {
-            using (var scenario = new NoDepsAppScenario(_logger))
+            await _app.StartWatcher().OrTimeout();
-            {
+            var pid = await _app.GetProcessId().OrTimeout();
-                // Wait for the process to start
+            await _app.HasExited().OrTimeout(); // process should exit after run
-                using (var wait = new WaitForFileToChange(scenario.StartedFile))
-                {
-                    scenario.RunDotNetWatch(new[] { "run", scenario.StatusFile });
 
-                    wait.Wait(_defaultTimeout,
+            var fileToChange = Path.Combine(_app.SourceDirectory, "Program.cs");
-                        expectedToChange: true,
+            var programCs = File.ReadAllText(fileToChange);
-                        errorMessage: $"File not created: {scenario.StartedFile}");
+            File.WriteAllText(fileToChange, programCs);
-                }
 
-                // Then wait for the app to exit
+            await _app.HasRestarted().OrTimeout();
-                Waiters.WaitForFileToBeReadable(scenario.StartedFile, _defaultTimeout);
+            var pid2 = await _app.GetProcessId().OrTimeout();
-                var ids = File.ReadAllLines(scenario.StatusFile);
+            Assert.NotEqual(pid, pid2);
-                var procId = int.Parse(ids[0]);
+            await _app.HasExited().OrTimeout(); // process should exit after run
-                Waiters.WaitForProcessToStop(
-                    procId,
-                    _defaultTimeout,
-                    expectedToStop: true,
-                    errorMessage: "Test app did not exit");
-
-                // Then wait for it to restart when we change a file
-                using (var wait = new WaitForFileToChange(scenario.StartedFile))
-                {
-                    // On Unix the file write time is in 1s increments;
-                    // if we don't wait, there's a chance that the polling
-                    // watcher will not detect the change
-                    Thread.Sleep(1000);
-
-                    var fileToChange = Path.Combine(scenario.TestAppFolder, "Program.cs");
-                    var programCs = File.ReadAllText(fileToChange);
-                    File.WriteAllText(fileToChange, programCs);
-
-                    wait.Wait(_defaultTimeout,
-                        expectedToChange: true,
-                        errorMessage: $"Process did not restart because {scenario.StartedFile} was not changed");
-                }
-            }
         }
 
-        private class NoDepsAppScenario : DotNetWatchScenario
+        public void Dispose()
         {
-            private const string TestAppName = "NoDepsApp";
+            _app.Dispose();
-
-            public NoDepsAppScenario(ITestOutputHelper logger)
-                : base(logger)
-            {
-                StatusFile = Path.Combine(Scenario.TempFolder, "status");
-                StartedFile = StatusFile + ".started";
-
-                Scenario.AddTestProjectFolder(TestAppName);
-                Scenario.Restore(TestAppName);
-
-                TestAppFolder = Path.Combine(Scenario.WorkFolder, TestAppName);
-            }
-
-            public string StatusFile { get; private set; }
-            public string StartedFile { get; private set; }
-            public string TestAppFolder { get; private set; }
-
-            public void RunDotNetWatch(IEnumerable<string> args)
-            {
-                RunDotNetWatch(args, Path.Combine(Scenario.WorkFolder, TestAppName));
-            }
         }
     }
 }

test/Microsoft.DotNet.Watcher.Tools.FunctionalTests/Properties/AssemblyInfo.cs

@@ -1,3 +0,0 @@
-using Xunit;
-
-[assembly: CollectionBehavior(CollectionBehavior.CollectionPerAssembly)]

test/Microsoft.DotNet.Watcher.Tools.FunctionalTests/Scenario/DotNetWatchScenario.cs

@@ -1,57 +0,0 @@
-// Copyright (c) .NET Foundation. All rights reserved.
-// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
-
-using System;
-using System.Collections.Generic;
-using System.Diagnostics;
-using System.IO;
-using Microsoft.Extensions.Internal;
-using Xunit.Abstractions;
-
-namespace Microsoft.DotNet.Watcher.Tools.FunctionalTests
-{
-    public class DotNetWatchScenario : IDisposable
-    {
-        protected ProjectToolScenario Scenario { get; }
-        public DotNetWatchScenario()
-            : this(null)
-        {
-        }
-
-        public DotNetWatchScenario(ITestOutputHelper logger)
-        {
-            Scenario = new ProjectToolScenario(logger);
-        }
-
-        public Process WatcherProcess { get; private set; }
-
-        public bool UsePollingWatcher { get; set; }
-
-        protected void RunDotNetWatch(IEnumerable<string> arguments, string workingFolder)
-        {
-            IDictionary<string, string> envVariables = null;
-            if (UsePollingWatcher)
-            {
-                envVariables = new Dictionary<string, string>()
-                {
-                    ["DOTNET_USE_POLLING_FILE_WATCHER"] = "true"
-                };
-            }
-
-            WatcherProcess = Scenario.ExecuteDotnetWatch(arguments, workingFolder, envVariables);
-        }
-
-        public virtual void Dispose()
-        {
-            if (WatcherProcess != null)
-            {
-                if (!WatcherProcess.HasExited)
-                {
-                    WatcherProcess.KillTree();
-                }
-                WatcherProcess.Dispose();
-            }
-            Scenario.Dispose();
-        }
-    }
-}

test/Microsoft.DotNet.Watcher.Tools.FunctionalTests/Scenario/ProjectToolScenario.cs

@@ -3,9 +3,7 @@
 
 using System;
 using System.Collections.Generic;
-using System.Diagnostics;
 using System.IO;
-using System.Linq;
 using System.Reflection;
 using System.Threading;
 using Microsoft.DotNet.Cli.Utils;
@@ -58,12 +56,21 @@ namespace Microsoft.DotNet.Watcher.Tools.FunctionalTests
 
         public void Restore(string project)
         {
-            project = Path.Combine(WorkFolder, project);
-
             _logger?.WriteLine($"Restoring msbuild project in {project}");
+            ExecuteCommand(project, "restore");
+        }
 
-            var restore = Command
+        public void Build(string project)
-                .Create(new Muxer().MuxerPath, new[] { "restore", "/p:SkipInvalidConfigurations=true" })
+        {
+            _logger?.WriteLine($"Building {project}");
+            ExecuteCommand(project, "build");
+        }
+
+        private void ExecuteCommand(string project, params string[] arguments)
+        {
+            project = Path.Combine(WorkFolder, project);
+            var command = Command
+                .Create(new Muxer().MuxerPath, arguments)
                 .WorkingDirectory(project)
                 .CaptureStdErr()
                 .CaptureStdOut()
@@ -71,9 +78,9 @@ namespace Microsoft.DotNet.Watcher.Tools.FunctionalTests
                 .OnOutputLine(l => _logger?.WriteLine(l))
                 .Execute();
 
-            if (restore.ExitCode != 0)
+            if (command.ExitCode != 0)
             {
-                throw new Exception($"Exit code {restore.ExitCode}");
+                throw new InvalidOperationException($"Exit code {command.ExitCode}");
             }
         }
 
@@ -88,7 +95,7 @@ namespace Microsoft.DotNet.Watcher.Tools.FunctionalTests
             File.Copy(nugetConfigFilePath, tempNugetConfigFile);
         }
 
-        public Process ExecuteDotnetWatch(IEnumerable<string> arguments, string workDir, IDictionary<string, string> environmentVariables = null)
+        public IEnumerable<string> GetDotnetWatchArguments()
         {
             // this launches a new .NET Core process using the runtime of the current test app
             // and the version of dotnet-watch that this test app is compiled against
@@ -104,32 +111,7 @@ namespace Microsoft.DotNet.Watcher.Tools.FunctionalTests
 
             args.Add(Path.Combine(AppContext.BaseDirectory, "dotnet-watch.dll"));
 
-            var argsStr = ArgumentEscaper.EscapeAndConcatenateArgArrayForProcessStart(args.Concat(arguments));
+            return args;
-
-            _logger?.WriteLine($"Running dotnet {argsStr} in {workDir}");
-
-            var psi = new ProcessStartInfo(new Muxer().MuxerPath, argsStr)
-            {
-                UseShellExecute = false,
-                WorkingDirectory = workDir
-            };
-
-            if (environmentVariables != null)
-            {
-                foreach (var newEnvVar in environmentVariables)
-                {
-                    var varKey = newEnvVar.Key;
-                    var varValue = newEnvVar.Value;
-#if NET451
-                    psi.EnvironmentVariables[varKey] = varValue;
-
-#else
-                    psi.Environment[varKey] = varValue;
-#endif
-                }
-            }
-
-            return Process.Start(psi);
         }
 
         private static string FindNugetConfig()

test/Microsoft.DotNet.Watcher.Tools.FunctionalTests/Scenario/WatchableApp.cs

@@ -0,0 +1,91 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.IO;
+using System.Linq;
+using System.Runtime.CompilerServices;
+using System.Threading.Tasks;
+using Microsoft.DotNet.Cli.Utils;
+using Xunit.Abstractions;
+
+namespace Microsoft.DotNet.Watcher.Tools.FunctionalTests
+{
+    public class WatchableApp : IDisposable
+    {
+        private const string StartedMessage = "Started";
+        private const string ExitingMessage = "Exiting";
+
+        protected ProjectToolScenario Scenario { get; }
+        private readonly ITestOutputHelper _logger;
+        protected AwaitableProcess Process { get; set; }
+        private string _appName;
+        private bool _prepared;
+
+        public WatchableApp(string appName, ITestOutputHelper logger)
+        {
+            _logger = logger;
+            _appName = appName;
+            Scenario = new ProjectToolScenario(logger);
+            Scenario.AddTestProjectFolder(appName);
+            SourceDirectory = Path.Combine(Scenario.WorkFolder, appName);
+        }
+
+        public string SourceDirectory { get; }
+
+        public Task HasRestarted()
+            => Process.GetOutputLineAsync(StartedMessage);
+
+        public Task HasExited()
+            => Process.GetOutputLineAsync(ExitingMessage);
+
+        public bool UsePollingWatcher { get; set; }
+
+        public Task StartWatcher([CallerMemberName] string name = null)
+            => StartWatcher(Array.Empty<string>(), name);
+
+        public async Task<int> GetProcessId()
+        {
+            var line = await Process.GetOutputLineAsync(l => l.StartsWith("PID ="));
+            var pid = line.Split('=').Last();
+            return int.Parse(pid);
+        }
+
+        public void Prepare()
+        {
+            Scenario.Restore(_appName);
+            Scenario.Build(_appName);
+            _prepared = true;
+        }
+
+        public async Task StartWatcher(string[] arguments, [CallerMemberName] string name = null)
+        {
+            if (!_prepared)
+            {
+                throw new InvalidOperationException("Call .Prepare() first");
+            }
+
+            var args = Scenario
+                .GetDotnetWatchArguments()
+                .Concat(new[] { "run", "--" })
+                .Concat(arguments);
+
+            var spec = new ProcessSpec
+            {
+                Executable = new Muxer().MuxerPath,
+                Arguments = args,
+                WorkingDirectory = SourceDirectory
+            };
+
+            Process = new AwaitableProcess(spec, _logger);
+            Process.Start();
+            await Process.GetOutputLineAsync(StartedMessage);
+        }
+
+        public virtual void Dispose()
+        {
+            Process.Dispose();
+            Scenario.Dispose();
+        }
+    }
+}

test/Microsoft.DotNet.Watcher.Tools.FunctionalTests/TaskExtensions.cs

@@ -0,0 +1,27 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Runtime.CompilerServices;
+using System.Threading.Tasks;
+
+namespace Microsoft.DotNet.Watcher.Tools.FunctionalTests
+{
+    public static class TaskExtensions
+    {
+        public static async Task<T> OrTimeout<T>(this Task<T> task, int timeout = 30, [CallerFilePath] string file = null, [CallerLineNumber] int line = 0)
+        {
+            await OrTimeout((Task)task, timeout, file, line);
+            return task.Result;
+        }
+
+        public static async Task OrTimeout(this Task task, int timeout = 30, [CallerFilePath] string file = null, [CallerLineNumber] int line = 0)
+        {
+            var finished = await Task.WhenAny(task, Task.Delay(TimeSpan.FromSeconds(timeout)));
+            if (!ReferenceEquals(finished, task))
+            {
+                throw new TimeoutException($"Task exceeded max running time of {timeout}s at {file}:{line}");
+            }
+        }
+    }
+}

test/Microsoft.DotNet.Watcher.Tools.FunctionalTests/TestProjects/AppWithDeps/AppWithDeps.csproj

@@ -9,7 +9,7 @@
     <Compile Include="**\*.cs" />
     <EmbeddedResource Include="**\*.resx" />
     <ProjectReference Include="..\Dependency\Dependency.csproj" />
-    <PackageReference Include="Microsoft.NET.Sdk" Version="1.0.0-alpha-20161029-1" PrivateAssets="All" />
+    <PackageReference Include="Microsoft.NET.Sdk" Version="1.0.0-alpha-20161104-2" PrivateAssets="All" />
     <PackageReference Include="Microsoft.NETCore.App" Version="1.0.1" />
   </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />

test/Microsoft.DotNet.Watcher.Tools.FunctionalTests/TestProjects/AppWithDeps/Program.cs

@@ -3,7 +3,6 @@
 
 using System;
 using System.Diagnostics;
-using System.IO;
 using System.Threading;
 
 namespace ConsoleApplication
@@ -14,26 +13,9 @@ namespace ConsoleApplication
 
         public static void Main(string[] args)
         {
-            ConsoleWrite("AppWithDeps started.");
+            Console.WriteLine("Started");
-
+            Console.WriteLine($"PID = " + Process.GetCurrentProcess().Id);
-            File.AppendAllLines(args[0], new string[] { $"{processId}" });
+            Thread.Sleep(Timeout.Infinite);
-
-            File.WriteAllText(args[0] + ".started", "");
-            Block();
-        }
-
-        private static void ConsoleWrite(string text)
-        {
-            Console.WriteLine($"[{processId}] {text}");
-        }
-
-        private static void Block()
-        {
-            while (true)
-            {
-                ConsoleWrite("Blocked...");
-                Thread.Sleep(1000);
-            }
         }
     }
 }

test/Microsoft.DotNet.Watcher.Tools.FunctionalTests/TestProjects/Dependency/Dependency.csproj

@@ -9,7 +9,7 @@
     <Compile Include="**\*.cs" />
     <EmbeddedResource Include="**\*.resx" />
     <PackageReference Include="NETStandard.Library" Version="1.6.0" />
-    <PackageReference Include="Microsoft.NET.Sdk" Version="1.0.0-alpha-20161029-1" PrivateAssets="All" />
+    <PackageReference Include="Microsoft.NET.Sdk" Version="1.0.0-alpha-20161104-2" PrivateAssets="All" />
   </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
 </Project>

test/Microsoft.DotNet.Watcher.Tools.FunctionalTests/TestProjects/GlobbingApp/GlobbingApp.csproj

@@ -8,7 +8,7 @@
   <ItemGroup>
     <Compile Include="Program.cs;include\*.cs" Exclude="exclude\*" />
     <EmbeddedResource Include="**\*.resx"  />
-    <PackageReference Include="Microsoft.NET.Sdk" Version="1.0.0-alpha-20161029-1" PrivateAssets="All" />
+    <PackageReference Include="Microsoft.NET.Sdk" Version="1.0.0-alpha-20161104-2" PrivateAssets="All" />
     <PackageReference Include="Microsoft.NETCore.App" Version="1.0.1" />
   </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />

test/Microsoft.DotNet.Watcher.Tools.FunctionalTests/TestProjects/GlobbingApp/Program.cs

@@ -3,37 +3,20 @@
 
 using System;
 using System.Diagnostics;
-using System.IO;
+using System.Linq;
+using System.Reflection;
 using System.Threading;
 
 namespace ConsoleApplication
 {
     public class Program
     {
-        private static readonly int processId = Process.GetCurrentProcess().Id;
-
         public static void Main(string[] args)
         {
-            ConsoleWrite("GlobbingApp started.");
+            Console.WriteLine("Started");
-
+            Console.WriteLine("PID = " + Process.GetCurrentProcess().Id);
-            File.AppendAllLines(args[0], new string[] { $"{processId}" });
+            Console.WriteLine("Defined types = " + typeof(Program).GetTypeInfo().Assembly.DefinedTypes.Count());
-            
+            Thread.Sleep(Timeout.Infinite);
-            File.WriteAllText(args[0] + ".started", "");
-            Block();
-        }
-
-        private static void ConsoleWrite(string text)
-        {
-            Console.WriteLine($"[{processId}] {text}");
-        }
-
-        private static void Block()
-        {
-            while (true)
-            {
-                ConsoleWrite("Blocked...");
-                Thread.Sleep(1000);
-            }
         }
     }
 }

test/Microsoft.DotNet.Watcher.Tools.FunctionalTests/TestProjects/NoDepsApp/NoDepsApp.csproj

@@ -8,7 +8,7 @@
   <ItemGroup>
     <Compile Include="**\*.cs" />
     <EmbeddedResource Include="**\*.resx" />
-    <PackageReference Include="Microsoft.NET.Sdk" Version="1.0.0-alpha-20161029-1" PrivateAssets="All" />
+    <PackageReference Include="Microsoft.NET.Sdk" Version="1.0.0-alpha-20161104-2" PrivateAssets="All" />
     <PackageReference Include="Microsoft.NETCore.App" Version="1.0.1" />
   </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />

test/Microsoft.DotNet.Watcher.Tools.FunctionalTests/TestProjects/NoDepsApp/Program.cs

@@ -3,41 +3,21 @@
 
 using System;
 using System.Diagnostics;
-using System.IO;
 using System.Threading;
 
 namespace ConsoleApplication
 {
     public class Program
     {
-        private static readonly int processId = Process.GetCurrentProcess().Id;
-
         public static void Main(string[] args)
         {
-            ConsoleWrite("NoDepsApp started.");
+            Console.WriteLine("Started");
-
+            Console.WriteLine($"PID = " + Process.GetCurrentProcess().Id);
-            File.AppendAllLines(args[0], new string[] { $"{processId}" });
+            if (args.Length > 0 && args[0] == "--no-exit")
-            
-            File.WriteAllText(args[0] + ".started", "");
-
-            if (args.Length > 1 && args[1] == "--no-exit")
             {
-                Block();
+                Thread.Sleep(Timeout.Infinite);
-            }
-        }
-
-        private static void ConsoleWrite(string text)
-        {
-            Console.WriteLine($"[{processId}] {text}");
-        }
-
-        private static void Block()
-        {
-            while (true)
-            {
-                ConsoleWrite("Blocked...");
-                Thread.Sleep(1000);
             }
+            Console.WriteLine("Exiting");
         }
     }
 }

test/Microsoft.DotNet.Watcher.Tools.FunctionalTests/WaitForFileToChange.cs

@@ -1,61 +0,0 @@
-// Copyright (c) .NET Foundation. All rights reserved.
-// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
-
-using System;
-using System.IO;
-using System.Threading;
-using Microsoft.DotNet.Watcher.Internal;
-
-namespace Microsoft.DotNet.Watcher.Tools.FunctionalTests
-{
-    public class WaitForFileToChange : IDisposable
-    {
-        private readonly IFileSystemWatcher _watcher;
-        private readonly string _expectedFile;
-
-        private ManualResetEvent _changed = new ManualResetEvent(false);
-
-        public WaitForFileToChange(string file)
-        {
-            _watcher = FileWatcherFactory.CreateWatcher(Path.GetDirectoryName(file), usePollingWatcher: true);
-            _expectedFile = file;
-
-            _watcher.OnFileChange += WatcherEvent;
-            
-            _watcher.EnableRaisingEvents = true;
-        }
-
-        private void WatcherEvent(object sender, string file)
-        {
-            if (file.Equals(_expectedFile, StringComparison.Ordinal))
-            {
-                Waiters.WaitForFileToBeReadable(_expectedFile, TimeSpan.FromSeconds(10));
-                _changed?.Set();
-            }
-        }
-
-        public void Wait(TimeSpan timeout, bool expectedToChange, string errorMessage)
-        {
-            if (_changed != null)
-            {
-                var changed = _changed.WaitOne(timeout);
-                if (changed != expectedToChange)
-                {
-                    throw new Exception(errorMessage);
-                }
-            }
-        }
-
-        public void Dispose()
-        {
-            _watcher.EnableRaisingEvents = false;
-
-            _watcher.OnFileChange -= WatcherEvent;
-            
-            _watcher.Dispose();
-            _changed.Dispose();
-
-            _changed = null;
-        }
-    }
-}

test/Microsoft.DotNet.Watcher.Tools.FunctionalTests/Waiters.cs

@@ -1,84 +0,0 @@
-// Copyright (c) .NET Foundation. All rights reserved.
-// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
-
-using System;
-using System.Diagnostics;
-using System.IO;
-using System.Threading;
-
-namespace Microsoft.DotNet.Watcher.Tools.FunctionalTests
-{
-    public static class Waiters
-    {
-        public static void WaitForFileToBeReadable(string file, TimeSpan timeout)
-        {
-            var watch = new Stopwatch();
-
-            Exception lastException = null;
-
-            watch.Start();
-            while (watch.Elapsed < timeout)
-            {
-                try
-                {
-                    File.ReadAllText(file);
-                    watch.Stop();
-                    return;
-                }
-                catch (Exception e)
-                {
-                    lastException = e;
-                }
-                Thread.Sleep(500);
-            }
-            watch.Stop();
-
-            if (lastException != null)
-            {
-                Console.WriteLine("Last exception:");
-                Console.WriteLine(lastException);
-            }
-
-            throw new InvalidOperationException($"{file} is not readable.");
-        }
-
-        public static void WaitForProcessToStop(int processId, TimeSpan timeout, bool expectedToStop, string errorMessage)
-        {
-            Console.WriteLine($"Waiting for process {processId} to stop...");
-
-            Process process = null;
-
-            try
-            {
-                process = Process.GetProcessById(processId);
-            }
-            catch (Exception e)
-            {
-                // If we expect the process to stop, then it might have stopped already
-                if (!expectedToStop)
-                {
-                    Console.WriteLine($"Could not find process {processId}: {e}");
-                }
-            }
-
-            var watch = new Stopwatch();
-            watch.Start();
-            while (watch.Elapsed < timeout)
-            {
-                if (process == null || process.HasExited)
-                {
-                    Console.WriteLine($"Process {processId} is no longer running");
-                    break;
-                }
-                Thread.Sleep(500);
-            }
-            watch.Stop();
-
-            bool isStopped = process == null || process.HasExited;
-            if (isStopped != expectedToStop)
-            {
-                throw new InvalidOperationException(errorMessage);
-            }
-        }
-    }
-}

test/Microsoft.DotNet.Watcher.Tools.FunctionalTests/test.cmd

@@ -0,0 +1,6 @@
+@echo off
+
+rem For local testing
+
+dotnet build
+..\..\.build\dotnet\dotnet.exe exec --depsfile bin\Debug\netcoreapp1.0\Microsoft.DotNet.Watcher.Tools.FunctionalTests.deps.json --runtimeconfig bin\Debug\netcoreapp1.0\Microsoft.DotNet.Watcher.Tools.FunctionalTests.runtimeconfig.json ..\..\.build\dotnet-test-xunit\2.2.0-preview2-build1029\lib\netcoreapp1.0\dotnet-test-xunit.dll bin\Debug\netcoreapp1.0\Microsoft.DotNet.Watcher.Tools.FunctionalTests.dll %*

test/Microsoft.DotNet.Watcher.Tools.FunctionalTests/test.sh

@@ -0,0 +1,8 @@
+dotnet build
+
+../../.build/dotnet/dotnet exec \
+--depsfile bin/Debug/netcoreapp1.0/Microsoft.DotNet.Watcher.Tools.FunctionalTests.deps.json \
+--runtimeconfig bin/Debug/netcoreapp1.0/Microsoft.DotNet.Watcher.Tools.FunctionalTests.runtimeconfig.json \
+../../.build/dotnet-test-xunit/2.2.0-preview2-build1029/lib/netcoreapp1.0/dotnet-test-xunit.dll \
+bin/Debug/netcoreapp1.0/Microsoft.DotNet.Watcher.Tools.FunctionalTests.dll \
+$@