From 618a73b399b363a5c202c1d093280b1941bc6457 Mon Sep 17 00:00:00 2001 From: Doug Bunting Date: Mon, 5 Oct 2015 23:02:50 -0700 Subject: [PATCH] Support building with Core CLR (on Linux) - hits all of the checkboxes in #272 Specify test runtime explicitly and support testing w/ DNX Core on Linux - turn on DNX Core on Linux testing with `--test-dnxcore` target - or `KOREBUILD_TEST_DNXCORE` environment variable Work around aspnet/dnx#2566 - builds fail with DNX Core on Linux otherwise Do not attempt to install nodejs if not on Windows Use user's `default` DNVM when `%SKIP_DNX_INSTALL%` defined on Windows - enables builds with DNX Core - should be the default for new repos; older repos can opt in Use consistent case for all `--quiet` environment variables - also name these variables consistently; start w/ `KOREBUILD_` - environment variables are case-sensitive on Linux and mixed case was annoying - also get rid of leading space in these environment variable values Check `IsLinux` and not `IsMono` - `IsMono` is always `true` on Linux since Sake always runs in Mono there - but `IsLinux` is the right question nits: - remove tabs from a few files - remove compilation warnings in local `makefile.shade` --- build-template/build.cmd | 10 ++--- build/BuildEnv.shade | 37 ++++++++++++++---- build/_bower.shade | 3 +- build/_git.shade | 4 +- build/_grunt.shade | 3 +- build/_k-restore.shade | 6 +-- build/_k-standard-goals.shade | 70 +++++++++++++++++++++++++++++------ build/_k-test.shade | 22 +++++++++-- build/_k.shade | 11 +++--- build/_kpm-build.shade | 6 +-- build/_kpm-pack.shade | 6 +-- build/_kpm-publish.shade | 4 +- build/_node-install.shade | 4 +- build/_rimraf.shade | 5 ++- makefile.shade | 8 ++-- 15 files changed, 145 insertions(+), 54 deletions(-) diff --git a/build-template/build.cmd b/build-template/build.cmd index 4f34e4bc21..fdda9efab3 100644 --- a/build-template/build.cmd +++ b/build-template/build.cmd @@ -19,20 +19,20 @@ copy %CACHED_NUGET% .nuget\nuget.exe > nul :restore IF EXIST packages\KoreBuild goto run IF %BUILDCMD_KOREBUILD_VERSION%=="" ( - .nuget\NuGet.exe install KoreBuild -ExcludeVersion -o packages -nocache -pre + .nuget\NuGet.exe install KoreBuild -ExcludeVersion -o packages -nocache -pre ) ELSE ( - .nuget\NuGet.exe install KoreBuild -version %BUILDCMD_KOREBUILD_VERSION% -ExcludeVersion -o packages -nocache -pre + .nuget\NuGet.exe install KoreBuild -version %BUILDCMD_KOREBUILD_VERSION% -ExcludeVersion -o packages -nocache -pre ) .nuget\NuGet.exe install Sake -ExcludeVersion -Source https://www.nuget.org/api/v2/ -Out packages IF "%SKIP_DNX_INSTALL%"=="1" goto run IF %BUILDCMD_DNX_VERSION%=="" ( - CALL packages\KoreBuild\build\dnvm upgrade -runtime CLR -arch x86 + CALL packages\KoreBuild\build\dnvm upgrade -runtime CLR -arch x86 ) ELSE ( - CALL packages\KoreBuild\build\dnvm install %BUILDCMD_DNX_VERSION% -runtime CLR -arch x86 -alias default + CALL packages\KoreBuild\build\dnvm install %BUILDCMD_DNX_VERSION% -runtime CLR -arch x86 -alias default ) CALL packages\KoreBuild\build\dnvm install default -runtime CoreCLR -arch x86 :run -CALL packages\KoreBuild\build\dnvm use default -runtime CLR -arch x86 +CALL packages\KoreBuild\build\dnvm use default packages\Sake\tools\Sake.exe -I packages\KoreBuild\build -f makefile.shade %* diff --git a/build/BuildEnv.shade b/build/BuildEnv.shade index 81d7d716c5..e8066f9847 100644 --- a/build/BuildEnv.shade +++ b/build/BuildEnv.shade @@ -1,5 +1,5 @@ - use namespace="System" +use namespace="System.IO" functions @{ @@ -7,7 +7,7 @@ functions { var start = new DateTime(2015, 1, 1); var now = DateTime.UtcNow; - + string version = "0"; // If the computer date is set before the start date, then the version is 0 if (now >= start) @@ -21,17 +21,40 @@ functions string BuildNumber { - get + get { return "t" + DateTime.UtcNow.ToString("yyMMddHHmmss"); } } - + bool IsBuildV2 { - get - { - return Environment.GetEnvironmentVariable("KOREBUILD_BUILD_V2") == "1"; + get + { + return Environment.GetEnvironmentVariable("KOREBUILD_BUILD_V2") == "1"; + } + } + + bool IsDnxCoreClr + { + get + { + // Check default for invoked commands rather than current environment. Sake always runs in Mono on Linux. + var paths = Environment.GetEnvironmentVariable("PATH").Split((char)';'); + foreach (var path in paths) + { + if (path.Contains(Path.DirectorySeparatorChar + "dnx-")) + { + if (path.Contains("-coreclr-")) + { + return true; + } + + break; + } + } + + return false; } } } \ No newline at end of file diff --git a/build/_bower.shade b/build/_bower.shade index 85e0956447..1057204326 100644 --- a/build/_bower.shade +++ b/build/_bower.shade @@ -10,7 +10,8 @@ var bowerCmd = '${ bowerGloballyInstalled ? "bower" : bowerLibrary }' - bowerCommand = bowerCommand + " --config.interactive=false"; - // Install bower locally if not already installed either globally or locally; creates bowerLibrary file if run -npm npmCommand='install ${E("npm_install_options")} --prefix "${nodeDir}" bower' if='!(bowerGloballyInstalled || bowerInstalled)' once='installBower' +var installCommand = 'install ${E("KOREBUILD_NPM_INSTALL_OPTIONS")} --prefix "${nodeDir}" bower' +npm npmCommand='${installCommand}' if='!(bowerGloballyInstalled || bowerInstalled)' once='installBower' - // Run bower exec program='cmd' commandline='/C ${bowerCmd} ${bowerCommand}' workingdir='${bowerDir}' if='bowerGloballyInstalled && !IsLinux' diff --git a/build/_git.shade b/build/_git.shade index 6d34b212bf..eb05ca81be 100644 --- a/build/_git.shade +++ b/build/_git.shade @@ -2,6 +2,6 @@ default gitFolder='' -// Use cmd to invoke git so that people who have git as a 'cmd' -exec program='cmd' commandline='/C git ${gitCommand}' workingdir='${gitFolder}' if='!IsMono' -exec program='git' commandline='${gitCommand}' workingdir='${gitFolder}' if='IsMono' +exec program='cmd' commandline='/C git ${gitCommand}' workingdir='${gitFolder}' if='!IsLinux' +exec program='git' commandline='${gitCommand}' workingdir='${gitFolder}' if='IsLinux' diff --git a/build/_grunt.shade b/build/_grunt.shade index b3cb7439a8..7dcd815dba 100644 --- a/build/_grunt.shade +++ b/build/_grunt.shade @@ -7,7 +7,8 @@ default gruntCliGloballyInstalled = '${ !gruntCliInstalled && TestCommand("grunt var gruntCmd = '${ gruntCliGloballyInstalled ? "grunt" : gruntCliLibrary }' - // Install grunt-cli locally if not already installed either globally or locally; creates gruntCliLibrary file if run -npm npmCommand='install ${E("npm_install_options")} --prefix "${nodeDir}" grunt-cli' if='!(gruntCliGloballyInstalled || gruntCliInstalled)' once='installGruntCli' +var installCommand = 'install ${E("KOREBUILD_NPM_INSTALL_OPTIONS")} --prefix "${nodeDir}" grunt-cli' +npm npmCommand='${installCommand}' if='!(gruntCliGloballyInstalled || gruntCliInstalled)' once='installGruntCli' -// Run grunt-cli exec program='cmd' commandline='/C ${gruntCmd}' workingdir='${gruntDir}' if='gruntCliGloballyInstalled && !IsLinux' diff --git a/build/_k-restore.shade b/build/_k-restore.shade index de5d385de1..55be2b03a6 100644 --- a/build/_k-restore.shade +++ b/build/_k-restore.shade @@ -10,7 +10,7 @@ restoreDir='' default currentDir = '${ Directory.GetCurrentDirectory() }' default restoreDir = '${ currentDir }' -default restore_options='${ E("NUGET3_restore_options") }' +default restore_options=' ${ E("KOREBUILD_DNU_RESTORE_OPTIONS") }' -exec program='cmd' commandline='/C dnu restore --parallel ${ restore_options }' workingdir='${ restoreDir }' if='!IsMono' -exec program='dnu' commandline='restore ${ restore_options }' workingdir='${ restoreDir }' if='IsMono' \ No newline at end of file +exec program='cmd' commandline='/C dnu restore --parallel${ restore_options }' workingdir='${ restoreDir }' if='!IsLinux' +exec program='dnu' commandline='restore${ restore_options }' workingdir='${ restoreDir }' if='IsLinux' diff --git a/build/_k-standard-goals.shade b/build/_k-standard-goals.shade index d3eabaa3c4..87ca1be662 100644 --- a/build/_k-standard-goals.shade +++ b/build/_k-standard-goals.shade @@ -3,9 +3,10 @@ use namespace="System" use namespace="System.Globalization" use namespace="System.IO" use namespace="System.Linq" -use import="Files" use import="BuildEnv" use import="Environment" +use import="Files" +use import="Json" use-teamcity default BASE_DIR='${Directory.GetCurrentDirectory()}' @@ -14,6 +15,7 @@ default BUILD_DIR='${Path.Combine(TARGET_DIR, "build")}' default TEST_DIR='${Path.Combine(TARGET_DIR, "test")}' default Configuration='${E("Configuration")}' default Quiet='${ false }' +default IncludeProjectsWithoutDnxClr = '${ true }' default PACKAGELIST_JSON_FILENAME = 'NuGetPackageVerifier.json' default DNX_TOOLS_FEED = 'https://www.myget.org/F/dnxtools/api/v3/index.json' default NUGET_FEED = 'https://api.nuget.org/v3/index.json' @@ -43,12 +45,12 @@ default NUGET_FEED = 'https://api.nuget.org/v3/index.json' #restore-npm-modules -// Find all dirs that contain a package.json file var npmDirs = '${GetDirectoriesContaining(Directory.GetCurrentDirectory(), "package.json")}' - npm npmCommand='install ${E("npm_install_options")}' each='var npmDir in npmDirs' + npm npmCommand='install ${E("KOREBUILD_NPM_INSTALL_OPTIONS")}' each='var npmDir in npmDirs' #restore-bower-components -// Find all dirs that contain a bower.json file var bowerDirs = '${GetDirectoriesContaining(Directory.GetCurrentDirectory(), "bower.json")}' - bower each='var bowerDir in bowerDirs' bowerCommand='install ${E("bower_install_options")}' + bower each='var bowerDir in bowerDirs' bowerCommand='install ${E("KOREBUILD_BOWER_INSTALL_OPTIONS")}' #run-grunt .restore-npm-modules .restore-bower-components target='initialize' -// Find all dirs that contain a gruntfile.js file @@ -94,9 +96,21 @@ default NUGET_FEED = 'https://api.nuget.org/v3/index.json' #ci-deep-clean .deep-clean target='clean' if='IsTeamCity' +-// Do not move below any of the build-compile targets; must run before them. +#build-coreclr-on-linux target='compile' if='IsLinux && IsDnxCoreClr' + @{ + // Work around aspnet/dnx#2566. Note this does not play well with IsBuildV2. + IncludeProjectsWithoutDnxClr = false; + AddToE("KOREBUILD_DNU_BUILD_OPTIONS", "--framework dnxcore50"); + AddToE("KOREBUILD_DNU_PACK_OPTIONS", "--framework dnxcore50"); + } + #build-compile target='compile' if='!IsBuildV2 && Directory.Exists("src")' @{ - var projectFiles = Files.Include("src/**/project.json").ToList(); + var projectFiles = Files + .Include("src/**/project.json") + .Where(projectFilePath => IncludeProjectsWithoutDnxClr || ProjectIncludesDnxCore(projectFilePath)) + .ToList(); if (ShouldRunInParallel) { Parallel.ForEach(projectFiles, projectFile => DnuPack(projectFile, BUILD_DIR, Configuration)); @@ -151,7 +165,7 @@ default NUGET_FEED = 'https://api.nuget.org/v3/index.json' } } -#native-compile target='compile' if='!IsMono && Directory.Exists(Path.Combine(BASE_DIR, "src"))' +#native-compile target='compile' if='!IsLinux && Directory.Exists(Path.Combine(BASE_DIR, "src"))' var programFilesX86 = '${Environment.GetFolderPath(Environment.SpecialFolder.ProgramFilesX86)}' var nativeProjects ='${Files.Include(Path.Combine(BASE_DIR, "src", "**", "*.vcxproj"))}' @@ -259,18 +273,23 @@ default NUGET_FEED = 'https://api.nuget.org/v3/index.json' #--quiet @{ - E("NUGET3_pack_options"," --quiet"); - E("NUGET3_build_options"," --quiet"); - E("NUGET3_restore_options"," --quiet"); - E("bower_install_options","--quiet"); - E("npm_install_options","--quiet"); + AddToE("KOREBUILD_BOWER_INSTALL_OPTIONS", "--quiet"); + AddToE("KOREBUILD_DNU_BUILD_OPTIONS", "--quiet"); + AddToE("KOREBUILD_DNU_PACK_OPTIONS", "--quiet"); + AddToE("KOREBUILD_DNU_RESTORE_OPTIONS", "--quiet"); + AddToE("KOREBUILD_NPM_INSTALL_OPTIONS", "--quiet"); Quiet = true; } #--parallel - @{ + @{ E("KOREBUILD_PARALLEL", "1"); - } + } + +#--test-dnxcore + @{ + E("KOREBUILD_TEST_DNXCORE", "1"); + } #stylecop if='Directory.Exists("src")' stylecop-setup @@ -279,6 +298,32 @@ default NUGET_FEED = 'https://api.nuget.org/v3/index.json' functions @{ string E(string key) { return Environment.GetEnvironmentVariable(key); } void E(string key, string value) { Environment.SetEnvironmentVariable(key, value); } + void AddToE(string key, string append) + { + var original = E(key); + if (string.IsNullOrEmpty(original)) + { + E(key, append); + } + else + { + E(key, original + " " + append); + } + } + + public bool ProjectIncludesDnxCore(string projectFilePath) + { + var projectText = File.ReadAllText(projectFilePath); + var project = (JsonObject)Json.Deserialize(projectText); + var frameworks = project.ValueAsJsonObject("frameworks"); + if (frameworks == null) + { + return false; + } + + return frameworks.Keys.Any(framework => framework.StartsWith("dnxcore", StringComparison.OrdinalIgnoreCase)); + } + IEnumerable GetDirectoriesContaining(string path, string searchPattern) { var sep = Path.DirectorySeparatorChar; @@ -290,6 +335,7 @@ functions @{ .Select(p => Path.GetDirectoryName(p)) .Distinct(); } + bool TestCommand(string program, string commandline) { // Tests whether a given command succeeds at the command line. diff --git a/build/_k-test.shade b/build/_k-test.shade index 765dd13d02..c40485000f 100644 --- a/build/_k-test.shade +++ b/build/_k-test.shade @@ -2,6 +2,7 @@ use import="Json" use import="Environment" default NO_PARALLEL_TEST_PROJECTS='${E("NO_PARALLEL_TEST_PROJECTS")}' +default KOREBUILD_TEST_DNXCORE='${E("KOREBUILD_TEST_DNXCORE")}' @{/* @@ -47,15 +48,28 @@ projectFile='' foreach (var framework in targetFrameworks) { - var testArgs = (IsMono || noParallelTestProjects.Contains(projectName)) ? " -parallel none" : ""; + var testArgs = noParallelTestProjects.Contains(projectName) ? " -parallel none" : ""; if (!framework.StartsWith("dnxcore", StringComparison.OrdinalIgnoreCase)) { - K(("test" + testArgs), projectFolder, ""); + string runtime; + if (IsLinux) + { + runtime = "mono"; + + // Work around issue with testing in parallel on Mono. + testArgs = " -parallel none"; + } + else + { + runtime = "clr"; + } + + K("test" + testArgs, projectFolder, "default -runtime " + runtime); } - else if (!IsMono) + else if (!IsLinux || !string.IsNullOrEmpty(KOREBUILD_TEST_DNXCORE)) { - K(("test" + testArgs), projectFolder, "default -runtime CoreCLR"); + K("test" + testArgs, projectFolder, "default -runtime coreclr"); } } } diff --git a/build/_k.shade b/build/_k.shade index c83df51cae..2b4a254f9b 100644 --- a/build/_k.shade +++ b/build/_k.shade @@ -9,9 +9,10 @@ dnvmUse='' */} default dnvmUse='' +var dnvmPath = '${ Path.Combine(Directory.GetCurrentDirectory(), "packages", "KoreBuild", "build", "dnvm") }' -var dnvmPath='${Directory.GetCurrentDirectory()}\packages\KoreBuild\build\dnvm' - -exec program='cmd' commandline='/C dnx ${command}' if='!IsMono && string.IsNullOrEmpty(dnvmUse)' -exec program='cmd' commandline='/C "${dnvmPath}" run ${dnvmUse} ${command}' if='!IsMono && !string.IsNullOrEmpty(dnvmUse)' -exec program='dnx' commandline='${command}' if='IsMono' +exec program='cmd' commandline='/C dnx ${command}' if='!IsLinux && string.IsNullOrEmpty(dnvmUse)' +exec program='cmd' commandline='/C "${dnvmPath}" run ${dnvmUse} ${command}' if='!IsLinux && !string.IsNullOrEmpty(dnvmUse)' +exec program='dnx' commandline='${command}' if='IsLinux && string.IsNullOrEmpty(dnvmUse)' +var commandLine = 'bash -c ". \"${dnvmPath}.sh\"; dnvm run ${dnvmUse} ${command}"' +exec program='/usr/bin/env' commandline='${ commandLine }' if='IsLinux && !string.IsNullOrEmpty(dnvmUse)' diff --git a/build/_kpm-build.shade b/build/_kpm-build.shade index a0130af8e2..48619e9ac4 100644 --- a/build/_kpm-build.shade +++ b/build/_kpm-build.shade @@ -11,7 +11,7 @@ configuration='' */} default configuration = 'Debug' -default build_options='${E("NUGET3_build_options")}' +default build_options=' ${E("KOREBUILD_DNU_BUILD_OPTIONS")}' @{ if (IsBuildV2) @@ -37,7 +37,7 @@ default build_options='${E("NUGET3_build_options")}' var projectsArg=projectFile.Replace(";", " "); var dnuArgs=string.Format("build{0} {1} --configuration {2}", build_options, projectsArg, configuration); - if (!IsMono) + if (!IsLinux) { Exec("cmd", "/C dnu " + dnuArgs); } @@ -54,7 +54,7 @@ default build_options='${E("NUGET3_build_options")}' DeleteFolder(projectBin); var dnuArgs=string.Format("build{0} {1} --configuration {2}", build_options, projectFolder, configuration); - if (!IsMono) + if (!IsLinux) { Exec("cmd", "/C dnu " + dnuArgs); } diff --git a/build/_kpm-pack.shade b/build/_kpm-pack.shade index 0a438cdbef..c20ceac813 100644 --- a/build/_kpm-pack.shade +++ b/build/_kpm-pack.shade @@ -14,7 +14,7 @@ configuration='' */} default configuration = 'Debug' -default pack_options='${E("NUGET3_pack_options")}' +default pack_options=' ${E("KOREBUILD_DNU_PACK_OPTIONS")}' @{ if (IsBuildV2) @@ -40,7 +40,7 @@ default pack_options='${E("NUGET3_pack_options")}' var projectsArg=projectFile.Replace(";", " "); var dnuArgs=string.Format("pack{0} {1} --configuration {2}", pack_options, projectsArg, configuration); - if (!IsMono) + if (!IsLinux) { Exec("cmd", "/C dnu " + dnuArgs); } @@ -66,7 +66,7 @@ default pack_options='${E("NUGET3_pack_options")}' DeleteFolder(projectBin); var dnuArgs=string.Format("pack{0} {1} --configuration {2}", pack_options, projectFolder, configuration); - if (!IsMono) + if (!IsLinux) { Exec("cmd", "/C dnu " + dnuArgs); } diff --git a/build/_kpm-publish.shade b/build/_kpm-publish.shade index 1ade43509d..10193ac874 100644 --- a/build/_kpm-publish.shade +++ b/build/_kpm-publish.shade @@ -17,5 +17,5 @@ default targetPackagesDir='' .Where(p => !p.EndsWith(".symbols.nupkg")); } -exec program='cmd' commandline='/C dnu packages add ${package} ${targetPackagesDir}' if='!IsMono' each='var package in packages' -exec program='dnu' commandline='packages add ${package} ${targetPackagesDir}' if='IsMono' each='var package in packages' +exec program='cmd' commandline='/C dnu packages add ${package} ${targetPackagesDir}' if='!IsLinux' each='var package in packages' +exec program='dnu' commandline='packages add ${package} ${targetPackagesDir}' if='IsLinux' each='var package in packages' diff --git a/build/_node-install.shade b/build/_node-install.shade index 656d0dd0e4..52fd7a438c 100644 --- a/build/_node-install.shade +++ b/build/_node-install.shade @@ -21,7 +21,9 @@ var npmInstallZipPath = '${Path.Combine(nodeDir, npmZip)}' var nodeInstalled = '${ File.Exists(nodeInstallExePath) }' default nodeGloballyInstalled = '${ !nodeInstalled && TestCommand("node", "--version") }' -var doInstall = '${ !(nodeGloballyInstalled || nodeInstalled) }' + +-// Command simply fails on Linux if nodejs is not available. +var doInstall = '${ !IsLinux && !(nodeGloballyInstalled || nodeInstalled) }' @{ if (doInstall) { diff --git a/build/_rimraf.shade b/build/_rimraf.shade index 3210117fda..e5216eed92 100644 --- a/build/_rimraf.shade +++ b/build/_rimraf.shade @@ -2,13 +2,14 @@ default currentDir = '${Directory.GetCurrentDirectory()}' default workingDir = '${ currentDir }' default nodeDir = '${Path.Combine(currentDir, "bin", "nodejs")}' default rimrafLibrary = '${ Path.Combine(nodeDir, "node_modules", "rimraf", "bin.js") }' -var rimrafInstalled = '${ File.Exists(Path.Combine(rimrafLibrary)) }' +var rimrafInstalled = '${ File.Exists(rimrafLibrary) }' default rimrafGloballyInstalled = '${ !rimrafInstalled && TestCommand("rimraf", "::") }' var rimrafCmd = '${ rimrafGloballyInstalled ? "rimraf" : rimrafLibrary }' - // Install rimraf locally if not already installed either globally or locally; creates rimrafLibrary file if run -npm npmCommand='install ${E("npm_install_options")} --prefix "${nodeDir}" rimraf' if='!(rimrafGloballyInstalled || rimrafInstalled)' once='installRimraf' +var installCommand = 'install ${E("KOREBUILD_NPM_INSTALL_OPTIONS")} --prefix "${nodeDir}" rimraf' +npm npmCommand='${installCommand}' if='!(rimrafGloballyInstalled || rimrafInstalled)' once='installRimraf' - // Run rimraf exec program='cmd' commandline='/C ${rimrafCmd} "${rimrafDir}"' workingdir='${workingDir}' if='rimrafGloballyInstalled && !IsLinux' diff --git a/makefile.shade b/makefile.shade index dc65e98830..0fc4e4fc5d 100644 --- a/makefile.shade +++ b/makefile.shade @@ -1,5 +1,4 @@ -var PROJECT='AspNet' var VERSION='0.2.1' use-teamcity @@ -302,7 +301,7 @@ var buildTarget = "compile" { GitCommand(repo, string.Format("describe --tags > ..\\{0}", versionFile)); } - catch(Exception e) + catch { Log.Warn(string.Format("{0} repo not tagged. Skipping....", repo)); continue; @@ -546,7 +545,10 @@ functions req.Method = "HEAD"; try { - using (req.GetResponse()); + using (req.GetResponse()) + { + // intentionally empty + } } catch (WebException ex) {