Always restore w/ Core CLR and build w/ CLR or Mono

- #290
- remove workaround for aspnet/dnx#2566; not needed if building with CLR or Mono
- install Core CLR DNX in `build.sh`
  - must be applied in all repos using Travis before this PR goes in
- add `Dnu()` and `Dnx()` macros
  - remove `K()` macro from `_k-test.shade`; use `Dnx()` instead
- incorporate per-repo changes not yet applied to Universe templates
  - use newer `NuGet.exe`
  - Universe itself does not need the Korebuild package or `dnvm`

Changes partially revert 618a73b e.g. always start build with CLR or Mono DNX

nits:
- reduced DNX switches and `dnvm` calls in `build.cmd`
- base NuGet skip decision on _both_ KoreBuild and Sake's installation status
- make `Quiet` variable work more reliably; small changes often led to compilation problems
  - requires a small change in DNX repo at the same time
- remove some trailing whitespace and tabs
- add a bit more description to `_k.shade`

build-template/build.cmd

@@ -2,14 +2,15 @@
 cd %~dp0
 
 SETLOCAL
-SET CACHED_NUGET=%LocalAppData%\NuGet\NuGet.exe
+SET NUGET_VERSION=latest
+SET CACHED_NUGET=%LocalAppData%\NuGet\nuget.%NUGET_VERSION%.exe
 SET BUILDCMD_KOREBUILD_VERSION=""
 SET BUILDCMD_DNX_VERSION=""
 
 IF EXIST %CACHED_NUGET% goto copynuget
 echo Downloading latest version of NuGet.exe...
 IF NOT EXIST %LocalAppData%\NuGet md %LocalAppData%\NuGet
-@powershell -NoProfile -ExecutionPolicy unrestricted -Command "$ProgressPreference = 'SilentlyContinue'; Invoke-WebRequest 'https://www.nuget.org/nuget.exe' -OutFile '%CACHED_NUGET%'"
+@powershell -NoProfile -ExecutionPolicy unrestricted -Command "$ProgressPreference = 'SilentlyContinue'; Invoke-WebRequest 'https://dist.nuget.org/win-x86-commandline/%NUGET_VERSION%/nuget.exe' -OutFile '%CACHED_NUGET%'"
 
 :copynuget
 IF EXIST .nuget\nuget.exe goto restore
@@ -17,22 +18,23 @@ md .nuget
 copy %CACHED_NUGET% .nuget\nuget.exe > nul
 
 :restore
-IF EXIST packages\KoreBuild goto run
+IF EXIST packages\Sake goto getdnx
 IF %BUILDCMD_KOREBUILD_VERSION%=="" (
-    .nuget\NuGet.exe install KoreBuild -ExcludeVersion -o packages -nocache -pre
+    .nuget\nuget.exe install KoreBuild -ExcludeVersion -o packages -nocache -pre
 ) ELSE (
-    .nuget\NuGet.exe install KoreBuild -version %BUILDCMD_KOREBUILD_VERSION% -ExcludeVersion -o packages -nocache -pre
+    .nuget\nuget.exe install KoreBuild -version %BUILDCMD_KOREBUILD_VERSION% -ExcludeVersion -o packages -nocache -pre
 )
 .nuget\NuGet.exe install Sake -ExcludeVersion -Source https://www.nuget.org/api/v2/ -Out packages
 
-IF "%SKIP_DNX_INSTALL%"=="1" goto run
+:getdnx
-IF %BUILDCMD_DNX_VERSION%=="" (
+IF "%SKIP_DNX_INSTALL%"=="" (
-    CALL packages\KoreBuild\build\dnvm upgrade -runtime CLR -arch x86
+    IF "%BUILDCMD_DNX_VERSION%"=="" (
+        BUILDCMD_DNX_VERSION=latest
+    )
+    CALL packages\KoreBuild\build\dnvm install %BUILDCMD_DNX_VERSION% -runtime CoreCLR -arch x86 -alias default
+    CALL packages\KoreBuild\build\dnvm install default -runtime CLR -arch x86 -alias default
 ) ELSE (
-    CALL packages\KoreBuild\build\dnvm install %BUILDCMD_DNX_VERSION% -runtime CLR -arch x86 -alias default
+    CALL packages\KoreBuild\build\dnvm use default -runtime CLR -arch x86
 )
-CALL packages\KoreBuild\build\dnvm install default -runtime CoreCLR -arch x86
 
-:run
-CALL packages\KoreBuild\build\dnvm use default
 packages\Sake\tools\Sake.exe -I packages\KoreBuild\build -f makefile.shade %*

build-template/build.sh

@@ -10,19 +10,21 @@ else
     fi
 fi
 mkdir -p $cachedir
+nugetVersion=latest
+cachePath=$cachedir/nuget.$nugetVersion.exe
 
-url=https://www.nuget.org/nuget.exe
+url=https://dist.nuget.org/win-x86-commandline/$nugetVersion/nuget.exe
 
-if test ! -f $cachedir/nuget.exe; then
+if test ! -f $cachePath; then
-    wget -O $cachedir/nuget.exe $url 2>/dev/null || curl -o $cachedir/nuget.exe --location $url /dev/null
+    wget -O $cachePath $url 2>/dev/null || curl -o $cachePath --location $url /dev/null
 fi
 
 if test ! -e .nuget; then
     mkdir .nuget
-    cp $cachedir/nuget.exe .nuget/nuget.exe
+    cp $cachePath .nuget/nuget.exe
 fi
 
-if test ! -d packages/KoreBuild; then
+if test ! -d packages/Sake; then
     mono .nuget/nuget.exe install KoreBuild -ExcludeVersion -o packages -nocache -pre
     mono .nuget/nuget.exe install Sake -ExcludeVersion -Source https://www.nuget.org/api/v2/ -Out packages
 fi
@@ -31,8 +33,11 @@ if ! type dnvm > /dev/null 2>&1; then
     source packages/KoreBuild/build/dnvm.sh
 fi
 
-if ! type dnx > /dev/null 2>&1; then
+if ! type dnx > /dev/null 2>&1 || [ -z "$SKIP_DNX_INSTALL" ]; then
-    dnvm upgrade
+    dnvm install latest -runtime coreclr -alias default
+    dnvm install default -runtime mono -alias default
+else
+    dnvm use default -runtime mono
 fi
 
 mono packages/Sake/tools/Sake.exe -I packages/KoreBuild/build -f makefile.shade "$@"

build.cmd

@@ -2,18 +2,22 @@
 cd %~dp0
 
 SETLOCAL
-SET CACHED_NUGET="%LocalAppData%\NuGet\NuGet.exe"
+SET NUGET_VERSION=latest
+SET CACHED_NUGET=%LocalAppData%\NuGet\nuget.%NUGET_VERSION%.exe
 
 IF EXIST %CACHED_NUGET% goto copynuget
 echo Downloading latest version of NuGet.exe...
-IF NOT EXIST "%LocalAppData%\NuGet" md "%LocalAppData%\NuGet"
+IF NOT EXIST %LocalAppData%\NuGet md %LocalAppData%\NuGet
-@powershell -NoProfile -ExecutionPolicy unrestricted -Command "$ProgressPreference = 'SilentlyContinue'; Invoke-WebRequest 'https://www.nuget.org/nuget.exe' -OutFile '%CACHED_NUGET%'"
+@powershell -NoProfile -ExecutionPolicy unrestricted -Command "$ProgressPreference = 'SilentlyContinue'; Invoke-WebRequest 'https://dist.nuget.org/win-x86-commandline/%NUGET_VERSION%/nuget.exe' -OutFile '%CACHED_NUGET%'"
 
 :copynuget
-IF EXIST .nuget\nuget.exe goto build
+IF EXIST .nuget\nuget.exe goto restore
 md .nuget
 copy %CACHED_NUGET% .nuget\nuget.exe > nul
 
-:build
+:restore
+IF EXIST packages\Sake goto run
 .nuget\NuGet.exe install Sake -ExcludeVersion -Source https://www.nuget.org/api/v2/ -Out packages
+
+:run
 packages\Sake\tools\Sake.exe -I build -f makefile.shade %*

build.sh

@@ -10,29 +10,22 @@ else
     fi
 fi
 mkdir -p $cachedir
+nugetVersion=latest
+cachePath=$cachedir/nuget.$nugetVersion.exe
 
-url=https://www.nuget.org/nuget.exe
+url=https://dist.nuget.org/win-x86-commandline/$nugetVersion/nuget.exe
 
-if test ! -f $cachedir/nuget.exe; then
+if test ! -f $cachePath; then
-    wget -O $cachedir/nuget.exe $url 2>/dev/null || curl -o $cachedir/nuget.exe --location $url /dev/null
+    wget -O $cachePath $url 2>/dev/null || curl -o $cachePath --location $url /dev/null
 fi
 
 if test ! -e .nuget; then
     mkdir .nuget
-    cp $cachedir/nuget.exe .nuget/nuget.exe
+    cp $cachePath .nuget/nuget.exe
 fi
 
-if test ! -d packages/KoreBuild; then
+if test ! -d packages/Sake; then
-    mono .nuget/nuget.exe install KoreBuild -ExcludeVersion -o packages -nocache -pre
     mono .nuget/nuget.exe install Sake -ExcludeVersion -Source https://www.nuget.org/api/v2/ -Out packages
 fi
 
-if ! type dnvm > /dev/null 2>&1; then
+mono packages/Sake/tools/Sake.exe -I build -f makefile.shade "$@"
-    source packages/KoreBuild/build/dnvm.sh
-fi
-
-if ! type dnx > /dev/null 2>&1; then
-    dnvm upgrade
-fi
-
-mono packages/Sake/tools/Sake.exe -I packages/KoreBuild/build -f makefile.shade "$@"

build/BuildEnv.shade

@@ -1,5 +1,4 @@
 use namespace="System"
-use namespace="System.IO"
 
 functions
   @{
@@ -34,27 +33,4 @@ functions
             return Environment.GetEnvironmentVariable("KOREBUILD_BUILD_V2") == "1";
         }
     }
-
-    bool IsDnxCoreClr
-    {
-        get
-        {
-            // Check default for invoked commands rather than current environment. Sake always runs in Mono on Linux.
-            var paths = Environment.GetEnvironmentVariable("PATH").Split((char)';');
-            foreach (var path in paths)
-            {
-                if (path.Contains(Path.DirectorySeparatorChar + "dnx-"))
-                {
-                    if (path.Contains("-coreclr-"))
-                    {
-                        return true;
-                    }
-
-                    break;
-                }
-            }
-
-            return false;
-        }
-    }
   }

build/_copy.shade

@@ -3,7 +3,6 @@ use import="Files"
 default include='**/*.*'
 default exclude=''
 default overwrite='${ false }'
-default Quiet='${ false }'
 
 @{
   var copyFiles = Files.BasePath(Path.GetFullPath(sourceDir));

build/_dnu.shade

@@ -0,0 +1,19 @@
+@{/*
+
+dnu
+    Run dnu commands in your project. Executes dnu command.
+
+command=''
+    The `dnu` subcommand to execute.
+dnvmUse=''
+    Optional. The DNX framework to use. Suitable for a `dnvm exec` command.
+*/}
+
+default dnvmUse=''
+var dnvmPath = '${ Path.Combine(Directory.GetCurrentDirectory(), "packages", "KoreBuild", "build", "dnvm") }'
+
+exec program='cmd' commandline='/C dnu ${command}' if='!IsLinux && string.IsNullOrEmpty(dnvmUse)'
+exec program='cmd' commandline='/C "${dnvmPath}" exec ${dnvmUse} dnu ${command}' if='!IsLinux && !string.IsNullOrEmpty(dnvmUse)'
+exec program='dnu' commandline='${command}' if='IsLinux && string.IsNullOrEmpty(dnvmUse)'
+var commandLine = 'bash -c ". \"${dnvmPath}.sh\"; dnvm exec ${dnvmUse} dnu ${command}"'
+exec program='/usr/bin/env' commandline='${ commandLine }' if='IsLinux && !string.IsNullOrEmpty(dnvmUse)'

build/_k-restore.shade

@@ -10,7 +10,6 @@ restoreDir=''
 default currentDir = '${ Directory.GetCurrentDirectory() }'
 default restoreDir = '${ currentDir }'
 
-default restore_options=' ${ E("KOREBUILD_DNU_RESTORE_OPTIONS") }'
+default restore_options=' ${ E("KOREBUILD_DNU_RESTORE_OPTIONS") } --parallel'
 
-exec program='cmd' commandline='/C dnu restore --parallel${ restore_options }' workingdir='${ restoreDir }' if='!IsLinux'
+dnu command='restore${ restore_options }' workingDir='${ restoreDir }' dnvmUse='default -runtime coreclr'
-exec program='dnu' commandline='restore${ restore_options }' workingdir='${ restoreDir }' if='IsLinux'

build/_k-standard-goals.shade

@@ -14,8 +14,6 @@ default TARGET_DIR='${Path.Combine(BASE_DIR, "artifacts")}'
 default BUILD_DIR='${Path.Combine(TARGET_DIR, "build")}'
 default TEST_DIR='${Path.Combine(TARGET_DIR, "test")}'
 default Configuration='${E("Configuration")}'
-default Quiet='${ false }'
-default IncludeProjectsWithoutDnxClr = '${ true }'
 default PACKAGELIST_JSON_FILENAME = 'NuGetPackageVerifier.json'
 default DNX_TOOLS_FEED = 'https://www.myget.org/F/dnxtools/api/v3/index.json'
 default NUGET_FEED = 'https://api.nuget.org/v3/index.json'
@@ -96,21 +94,9 @@ default NUGET_FEED = 'https://api.nuget.org/v3/index.json'
 
 #ci-deep-clean .deep-clean target='clean' if='IsTeamCity'
 
--// Do not move below any of the build-compile targets; must run before them.
-#build-coreclr-on-linux target='compile' if='IsLinux && IsDnxCoreClr'
-  @{
-    // Work around aspnet/dnx#2566. Note this does not play well with IsBuildV2.
-    IncludeProjectsWithoutDnxClr = false;
-    AddToE("KOREBUILD_DNU_BUILD_OPTIONS", "--framework dnxcore50");
-    AddToE("KOREBUILD_DNU_PACK_OPTIONS", "--framework dnxcore50");
-  }
-
 #build-compile target='compile' if='!IsBuildV2 && Directory.Exists("src")'
   @{
-    var projectFiles = Files
+    var projectFiles = Files.Include("src/**/project.json").ToList();
-        .Include("src/**/project.json")
-        .Where(projectFilePath => IncludeProjectsWithoutDnxClr || ProjectIncludesDnxCore(projectFilePath))
-        .ToList();
     if (ShouldRunInParallel)
     {
         Parallel.ForEach(projectFiles, projectFile => DnuPack(projectFile, BUILD_DIR, Configuration));
@@ -296,6 +282,8 @@ default NUGET_FEED = 'https://api.nuget.org/v3/index.json'
   stylecop-run each='var projectFile in Files.Include("src/**/project.json")'
 
 functions @{
+    private static bool Quiet { get; set; }
+
     string E(string key) { return Environment.GetEnvironmentVariable(key); }
     void E(string key, string value) { Environment.SetEnvironmentVariable(key, value); }
     void AddToE(string key, string append)
@@ -311,19 +299,6 @@ functions @{
       }
     }
 
-    public bool ProjectIncludesDnxCore(string projectFilePath)
-    {
-        var projectText = File.ReadAllText(projectFilePath);
-        var project = (JsonObject)Json.Deserialize(projectText);
-        var frameworks = project.ValueAsJsonObject("frameworks");
-        if (frameworks == null)
-        {
-            return false;
-        }
-
-        return frameworks.Keys.Any(framework => framework.StartsWith("dnxcore", StringComparison.OrdinalIgnoreCase));
-    }
-
     IEnumerable<string> GetDirectoriesContaining(string path, string searchPattern)
     {
       var sep = Path.DirectorySeparatorChar;
@@ -395,6 +370,15 @@ functions @{
 macro name='Exec' program='string' commandline='string'
     exec
 
+macro name='Dnu' command='string'
+    dnu
+
+macro name='Dnx' command='string' dnxDir='string'
+    k
+
+macro name='Dnx' command='string' dnxDir='string' dnvmUse='string'
+    k
+
 macro name="UpdateResx" resxFile='string'
     k-generate-resx
 

build/_k-test.shade

@@ -9,7 +9,7 @@ default KOREBUILD_TEST_DNXCORE='${E("KOREBUILD_TEST_DNXCORE")}'
 k-test
     Run unit tests in your project.
 
-projectFile='' 
+projectFile=''
     Required. Path to the test project.json to execute
 
 */}
@@ -19,7 +19,7 @@ projectFile=''
     var project = (JsonObject)Json.Deserialize(projectText);
 
     var commands = project.ValueAsJsonObject("commands");
-            
+
     if (commands != null && commands.Keys.Contains("test"))
     {
         var projectFolder = Path.GetDirectoryName(projectFile);
@@ -42,7 +42,7 @@ projectFile=''
         {
             targetFrameworks = configs.Keys;
         }
-        
+
         // Currently only dnx* targets are supported. See aspnet/Universe#53
         targetFrameworks = targetFrameworks.Where(k => k.StartsWith("dnx", StringComparison.OrdinalIgnoreCase));
 
@@ -52,28 +52,18 @@ projectFile=''
 
             if (!framework.StartsWith("dnxcore", StringComparison.OrdinalIgnoreCase))
             {
-                string runtime;
                 if (IsLinux)
                 {
-                    runtime = "mono";
-
                     // Work around issue with testing in parallel on Mono.
                     testArgs = " -parallel none";
                 }
-                else
-                {
-                    runtime = "clr";
-                }
 
-                K("test" + testArgs, projectFolder, "default -runtime " + runtime);
+                Dnx("test" + testArgs, projectFolder);
             }
             else if (!IsLinux || !string.IsNullOrEmpty(KOREBUILD_TEST_DNXCORE))
             {
-                K("test" + testArgs, projectFolder, "default -runtime coreclr");
+                Dnx("test" + testArgs, projectFolder, "default -runtime coreclr");
             }
         }
     }
 }
-
-macro name='K' command='string' workingdir='string' dnvmUse='string'
-    k

build/_k.shade

@@ -1,18 +1,24 @@
 @{/*
 
-k 
+k
-    Run dnx commands in your project. Executes dnx cmd.
+    Run dnx commands in your project. Executes `dnx` command.
 
 command=''
+    The `dnx` subcommand to execute.
+dnxDir=''
+    Optional. The directory in which to execute the `dnx` command.
 dnvmUse=''
-
+    Optional. The DNX framework to use. Suitable for a `dnvm run` command.
 */}
 
+default currentDir = '${Directory.GetCurrentDirectory()}'
+default dnxDir = '${ currentDir }'
+
 default dnvmUse=''
 var dnvmPath = '${ Path.Combine(Directory.GetCurrentDirectory(), "packages", "KoreBuild", "build", "dnvm") }'
 
-exec program='cmd' commandline='/C dnx ${command}' if='!IsLinux && string.IsNullOrEmpty(dnvmUse)'
+exec program='cmd' commandline='/C dnx ${command}' workingdir='${dnxDir}' if='!IsLinux && string.IsNullOrEmpty(dnvmUse)'
-exec program='cmd' commandline='/C "${dnvmPath}" run ${dnvmUse} ${command}' if='!IsLinux && !string.IsNullOrEmpty(dnvmUse)'
+exec program='cmd' commandline='/C "${dnvmPath}" run ${dnvmUse} ${command}' workingdir='${dnxDir}' if='!IsLinux && !string.IsNullOrEmpty(dnvmUse)'
-exec program='dnx' commandline='${command}' if='IsLinux && string.IsNullOrEmpty(dnvmUse)'
+exec program='dnx' commandline='${command}' workingdir='${dnxDir}' if='IsLinux && string.IsNullOrEmpty(dnvmUse)'
 var commandLine = 'bash -c ". \"${dnvmPath}.sh\"; dnvm run ${dnvmUse} ${command}"'
-exec program='/usr/bin/env' commandline='${ commandLine }' if='IsLinux && !string.IsNullOrEmpty(dnvmUse)'
+exec program='/usr/bin/env' commandline='${ commandLine }' workingdir='${dnxDir}' if='IsLinux && !string.IsNullOrEmpty(dnvmUse)'

build/_kpm-build.shade

@@ -37,14 +37,7 @@ default build_options=' ${E("KOREBUILD_DNU_BUILD_OPTIONS")}'
 
         var projectsArg=projectFile.Replace(";", " ");
         var dnuArgs=string.Format("build{0} {1} --configuration {2}", build_options, projectsArg, configuration);
-        if (!IsLinux)
+        Dnu(dnuArgs);
-        {
-            Exec("cmd", "/C dnu " + dnuArgs);
-        }
-        else
-        {
-            Exec("dnu", dnuArgs);
-        }
     }
     else
     {
@@ -54,13 +47,6 @@ default build_options=' ${E("KOREBUILD_DNU_BUILD_OPTIONS")}'
         DeleteFolder(projectBin);
 
         var dnuArgs=string.Format("build{0} {1} --configuration {2}", build_options, projectFolder, configuration);
-        if (!IsLinux)
+        Dnu(dnuArgs);
-        {
-            Exec("cmd", "/C dnu " + dnuArgs);
-        }
-        else
-        {
-            Exec("dnu", dnuArgs);
-        }
     }
 }

build/_kpm-pack.shade

@@ -40,14 +40,7 @@ default pack_options=' ${E("KOREBUILD_DNU_PACK_OPTIONS")}'
 
         var projectsArg=projectFile.Replace(";", " ");
         var dnuArgs=string.Format("pack{0} {1} --configuration {2}", pack_options, projectsArg, configuration);
-        if (!IsLinux)
+        Dnu(dnuArgs);
-        {
-            Exec("cmd", "/C dnu " + dnuArgs);
-        }
-        else
-        {
-            Exec("dnu", dnuArgs);
-        }
 
         foreach(var projFolder in projectsToPack)
         {
@@ -66,14 +59,7 @@ default pack_options=' ${E("KOREBUILD_DNU_PACK_OPTIONS")}'
         DeleteFolder(projectBin);
 
         var dnuArgs=string.Format("pack{0} {1} --configuration {2}", pack_options, projectFolder, configuration);
-        if (!IsLinux)
+        Dnu(dnuArgs);
-        {
-            Exec("cmd", "/C dnu " + dnuArgs);
-        }
-        else
-        {
-            Exec("dnu", dnuArgs);
-        }
 
         CopyFolder(projectBin, Path.Combine(kpmPackOutputDir, projectName), true);
     }

build/_kpm-publish.shade

@@ -17,5 +17,4 @@ default targetPackagesDir=''
                            .Where(p => !p.EndsWith(".symbols.nupkg"));
 }
 
-exec program='cmd' commandline='/C dnu packages add ${package} ${targetPackagesDir}' if='!IsLinux' each='var package in packages'
+dnu command='/C dnu packages add ${package} ${targetPackagesDir}' each='var package in packages'
-exec program='dnu' commandline='packages add ${package} ${targetPackagesDir}' if='IsLinux' each='var package in packages'

makefile.shade

@@ -14,6 +14,8 @@ use import="BuildEnv"
 
 functions
     @{
+        private static bool Quiet { get; set; }
+
         static string BUILD_BRANCH = Environment.GetEnvironmentVariable("BUILD_BRANCH") ?? "dev";
         static string BASE_DIR = Directory.GetCurrentDirectory();
         static string TARGET_DIR = Path.Combine(BASE_DIR, "artifacts", "build");
@@ -43,6 +45,11 @@ var buildTarget = "compile"
 
 #default .compile
 
+#--quiet
+  @{
+    Quiet = true;
+  }
+
 #pull
 #compile .pull
 #install .pull