aspnetcore/makefile.shade

775 lines
26 KiB
Plaintext

var VERSION='0.2.1'
use-teamcity
use namespace='System'
use namespace='System.IO'
use namespace='System.Collections.Generic'
use namespace='System.Net'
use namespace='System.Linq'
use namespace='System.Text'
use namespace='System.Text.RegularExpressions'
use namespace='System.Threading.Tasks'
use import="BuildEnv"
functions
@{
private static bool Quiet { get; set; }
static string BUILD_BRANCH = Environment.GetEnvironmentVariable("BUILD_BRANCH") ?? "dev";
static string BASE_DIR = Directory.GetCurrentDirectory();
static string TARGET_DIR = Path.Combine(BASE_DIR, "artifacts", "build");
static bool SKIP_NO_CREDENTIALS = Environment.GetEnvironmentVariable("UNIVERSE_SKIP_NO_CREDENTIALS") == "1";
// Doesn't build on Mono since their contracts don't match
string[] excludeReposOnMono = new[] { "Helios" };
IEnumerable<string> repositories = GetRepositoriesToBuild();
static bool useHttps = UseHttps(BASE_DIR);
static string gitHubUriPrefix = useHttps ? "https://github.com/aspnet/" : "git@github.com:aspnet/";
}
var buildTarget = "compile"
@{
var kBuildVersion = Environment.GetEnvironmentVariable("DNX_BUILD_VERSION");
if (!string.IsNullOrEmpty(kBuildVersion))
{
VERSION += "-" + kBuildVersion;
}
else
{
VERSION += "-" + BuildNumber;
}
}
#default .compile
#--quiet
@{
Quiet = true;
}
#pull
#compile .pull
#install .pull
#pack
directory create='${TARGET_DIR}'
nuget-pack nuspecFile='${Path.Combine(BASE_DIR, "KoreBuild.nuspec")}' packageVersion='${VERSION}' outputDir='${TARGET_DIR}'
nuget-pack nuspecFile='${Path.Combine(BASE_DIR, "KoreBuild-dotnet", "KoreBuild-dotnet.nuspec")}' packageVersion='${VERSION}' outputDir='${TARGET_DIR}'
#pack-install .pack
nuget-local-publish sourcePackagesDir='${TARGET_DIR}'
#git-pull target='pull'
@{
Parallel.ForEach(repositories, repo =>
{
CloneOrUpdate(repo);
});
}
#verify-all .pull .change-default-build-target-to-verify .build-all
#build-and-install-all .change-default-build-target-for-coherence-build .build-all
#smoke-test-mono .pull .fix-project-json .change-default-build-target-to-verify .build-all
#fix-project-json
@{
repositories = repositories.Except(excludeReposOnMono).ToArray();
}
-// Fix project.json to remove .Net portable references
for each='var repo in repositories'
for each='var file in Files.Include(repo + "/**" + "/project.json")'
update-file updateFile='${file}'
@{
updateText = updateText.Replace(".NETPortable,Version=4.6,Profile=Profile151", "foo");
updateText = updateText.Replace(".NETPortable,Version=v4.6,Profile=Profile151", "foo");
}
#change-default-build-target-to-verify
@{
buildTarget = "verify";
}
#change-default-build-target-for-coherence-build
- buildTarget = Environment.GetEnvironmentVariable("KOREBUILD_BUILD_TARGETS") ?? "compile nuget-install";
#init
@{
var templatePath = Path.Combine(BASE_DIR, "build-template");
var templateFiles = Files.Include(templatePath + Path.DirectorySeparatorChar + "*.*").Select(Path.GetFileName).ToList();
foreach(var repo in repositories)
{
foreach (string fileName in templateFiles)
{
var targetFile = Path.Combine(Directory.GetCurrentDirectory(), repo, fileName);
var sourceFile = Path.Combine(Directory.GetCurrentDirectory(), templatePath, fileName);
// Don't update the makefile
if (fileName.Equals("makefile.shade", StringComparison.OrdinalIgnoreCase) && File.Exists(targetFile))
{
continue;
}
// Don't update extra configuration files
if (fileName.Equals("NuGet.master.config", StringComparison.OrdinalIgnoreCase) ||
fileName.Equals("NuGet.release.config", StringComparison.OrdinalIgnoreCase))
{
continue;
}
if (!File.Exists(targetFile) ||
(File.ReadAllText(sourceFile) != File.ReadAllText(targetFile)))
{
Log.Info("Updating " + fileName + " to " + repo);
File.Copy(sourceFile, targetFile, true);
}
}
}
}
#update-release
-// Merge dev branch to release
@{
Parallel.ForEach(GetAllRepos(), CloneOrUpdate);
Log.Info("************************************* Checking repos for diffs *************************");
foreach (var repo in GetAllRepos())
{
Log.Info("Checking repo: " + repo);
// Check if the repo previously had a release branch
try
{
GitCommand(repo, "rev-parse --verify --quiet origin/release");
}
catch
{
Log.Info("Repository " + repo + " does not have a release branch.");
continue;
}
try
{
GitCommand(repo, "log -1 --exit-code origin/dev..origin/release");
}
catch
{
Log.Warn("Unmerged changes in repository " + repo);
GitCommand(repo, "log origin/dev..origin/release");
throw;
}
}
Log.Info("No conflicts in repos, continuing with creating release branch.");
foreach (var repo in GetAllRepos())
{
GitCommand(repo, "checkout origin/dev -B release");
// Update NuGet.Config
var nugetConfigPath = Path.Combine(repo, "NuGet.config");
if (File.Exists(nugetConfigPath))
{
var original = File.ReadAllText(nugetConfigPath);
var modified = original
.Replace("https://www.myget.org/F/aspnetcidev", "https://www.myget.org/F/aspnetcirelease")
.Replace("https://www.myget.org/F/azureadwebstacknightly", "https://www.myget.org/F/azureadwebstackrelease");
if (!string.Equals(original, modified, StringComparison.Ordinal))
{
File.WriteAllText(nugetConfigPath, modified);
GitCommand(repo, "add NuGet.config");
GitCommand(repo, "commit -m \"Updating to release NuGet.config.\"");
}
}
GitCommand(repo, "push origin release:release");
GitCommand(repo, "checkout origin/dev -B dev");
GitCommand(repo, "merge release -s ours");
GitCommand(repo, "push origin dev:dev");
}
}
#pull-all
-Parallel.ForEach(GetAllRepos(), CloneOrUpdate);
#update-master .pull-all
-// Merge release branch to master
-// Pin versions of packages in project.json and updated project.lock.json
-// More information https://github.com/aspnet/Universe/wiki/%23pin-version-:-Pinning-package-version-for-a-particular-release-in-project.json
@{
var COHERENCE_FEED = Environment.GetEnvironmentVariable("COHERENCE_FEED");
if (string.IsNullOrEmpty(COHERENCE_FEED))
{
throw new Exception("COHERENCE_FEED not specified. Usually this is Packages-NoTimestamp directory of Coherence-Signed.");
}
var KOREBUILD_VERSION = Environment.GetEnvironmentVariable("KOREBUILD_VERSION");
if (string.IsNullOrEmpty(KOREBUILD_VERSION))
{
throw new Exception("KOREBUILD_VERSION environment variable is not specified.");
}
var NUGET_VERSION = Environment.GetEnvironmentVariable("NUGET_VERSION");
if (string.IsNullOrEmpty(NUGET_VERSION))
{
throw new Exception("NUGET_VERSION not specified. This is most recent stable build of NuGet from http://dist.nuget.org/index.html. e.g. v3.2");
}
var excludeReposForJson = new[]
{
"Coherence",
"Coherence-Signed",
"dnvm",
"Entropy",
"Setup",
"libuv-build",
};
Exec("cmd", "/C dnu restore", @"tools\PinVersion");
foreach (var repo in GetAllRepos())
{
GitCommand(repo, "checkout origin/release -B master");
if (File.Exists(Path.Combine(repo, "NuGet.config")))
{
File.Copy(Path.Combine("build-template", "NuGet.master.config"),
Path.Combine(repo, "NuGet.config"),
overwrite: true);
GitCommand(repo, "add NuGet.*");
GitCommand(repo, "commit -m \"Updating NuGet.config\"");
}
}
var reposToPin = GetAllRepos().Except(excludeReposForJson);
foreach (var repo in reposToPin)
{
var repoPath = Path.Combine(Directory.GetCurrentDirectory(), repo);
Exec("dnx",
string.Format(@"run ""{0}"" ""{1}"" ""{2}""",
Path.Combine(Directory.GetCurrentDirectory(), repo),
COHERENCE_FEED,
KOREBUILD_VERSION),
@"tools\PinVersion");
GitCommand(repo, "commit -am \"Updating json files to pin versions and build.cmd to pin KoreBuild and DNX\"");
}
foreach (var repo in GetAllRepos())
{
GitCommand(repo, "push origin +master:master");
}
CallTarget("update-prerelease-tags");
}
#update-prerelease-tags
-// Update tags on each repo to have the latest release tag
@{
var PRERELEASETAG = Environment.GetEnvironmentVariable("PRERELEASETAG");
if (string.IsNullOrEmpty(PRERELEASETAG))
{
throw new Exception("PRERELEASETAG tag not defined");
}
var versionFile = "version.txt";
foreach (var repo in GetAllRepos())
{
GitCommand(repo, "pull --tags");
try
{
GitCommand(repo, string.Format("describe --tags > ..\\{0}", versionFile));
}
catch
{
Log.Warn(string.Format("{0} repo not tagged. Skipping....", repo));
continue;
}
var version = File.ReadAllText(versionFile);
File.Delete(versionFile);
Log.Info(string.Format("Current version on repo {0} is {1}", repo, version));
var majorVersion = version.Split(new string[]{"-"}, StringSplitOptions.None)[0];
var newVersion = majorVersion + string.Format("-{0}", PRERELEASETAG);
Log.Info(string.Format("New version for repo is {0}", newVersion));
GitCommand(repo, string.Format("tag -f -a {0} -m \"Tag for new release {0}\"", newVersion));
GitCommand(repo, "push origin --tags +" + newVersion);
}
}
#only-compile .build-all
-Log.Warn("only-compile target is deprecated. Use build-all");
#build-all target='compile'
@{
var failed = new Dictionary<string, Exception>();
var skipped = new List<string>();
var ciVolatileShare = Environment.GetEnvironmentVariable("CI_VOLATILE_SHARE");
var nugetExe = Environment.GetEnvironmentVariable("PUSH_NUGET_EXE");
foreach(var repo in repositories)
{
var blockName = string.Format("Building {0}", repo);
if (IsTeamCity)
{
Log.Info(string.Format("##teamcity[blockOpened name='{0}']", FormatForTeamCity(blockName)));
}
try
{
Log.Info(blockName);
if (SKIP_NO_CREDENTIALS)
{
if (!Directory.Exists(repo))
{
// The directory was not cloned because the credentials were not specified so don't try to build it
skipped.Add(repo);
Log.Warn(string.Format("The repo {0} does not exist locally and it will not be built.", repo));
continue;
}
}
if (IsMono)
{
Exec(Path.Combine(repo, "build.sh"), buildTarget, repo);
}
else
{
Exec("build.cmd", buildTarget, repo);
}
if (!string.IsNullOrEmpty(ciVolatileShare))
{
Log.Info("Publishing packages to " + ciVolatileShare);
if (string.IsNullOrEmpty(nugetExe))
{
Log.Warn("PUSH_NUGET_EXE not specified.");
}
else
{
NuGetPackagesAdd(Path.Combine(repo, "artifacts", "build"), ciVolatileShare);
}
}
Log.Info(string.Format("Build {0} succeeded", repo));
}
catch(Exception ex)
{
Log.Warn(string.Format("Build {0} failed: {1}", repo, ex.Message));
failed[repo] = ex;
}
finally
{
if (IsTeamCity)
{
Log.Info(string.Format("##teamcity[blockClosed name='{0}']", FormatForTeamCity(blockName)));
}
}
}
foreach(var repo in repositories)
{
Exception ex;
if (failed.TryGetValue(repo, out ex))
{
Log.Warn(string.Format("Build {0} failed: {1}", repo, ex.Message));
if (IsTeamCity)
{
Log.Warn(string.Format("##teamcity[message text='{0}' errorDetails='{1}' status='ERROR']",
FormatForTeamCity(ex.Message),
FormatForTeamCity(ex.StackTrace)));
}
}
else if (skipped.Contains(repo))
{
Log.Warn(string.Format("Build {0} skipped", repo));
}
else
{
Log.Info(string.Format("Build {0} succeeded", repo));
}
}
if (failed.Any())
{
throw new Exception("One or more repos failed to build");
}
}
#only-install target='install'
@{
foreach(var repo in repositories)
{
if (IsMono)
{
Exec(Path.Combine(repo, "build.sh"), "install", repo);
}
else
{
Exec("build.cmd", "install", repo);
}
}
}
#run-snapshot-manager
@{
Exec(@".nuget\nuget.exe", @"restore -out packages tools\TCDependencyManager\packages.config", "");
var programFiles = Environment.GetFolderPath(Environment.SpecialFolder.ProgramFilesX86);
var msbuildPath = Path.Combine(programFiles, "MSBuild", "12.0", "Bin", "MsBuild.exe");
Exec(msbuildPath, "TCDependencyManager.csproj", @"tools\TCDependencyManager");
Exec(@"tools\TCDependencyManager\bin\Debug\TCDependencyManager.exe", "", "");
}
#git-status description='Show status of repos known by Universe'
@{
foreach(var repo in repositories)
{
GitStatus(repo);
}
}
#git-clean description='REMOVE ALL CHANGES to the working directory'
@{
Console.WriteLine("This runs `git clean -xfd` in all non-Universe repos.");
Console.WriteLine("This should REMOVE ALL CHANGES to the working directory.");
Console.Write("***** Are you sure? ***** (Y or anything else)? ");
if (Console.ReadLine() != "Y")
{
throw new Exception("git-clean cancelled");
}
foreach(var repo in repositories)
{
GitClean(repo);
}
}
macro name='ExecClr' program='string' commandline='string'
exec-clr
macro name='GitPull' gitUri='string' gitBranch='string' gitFolder='string'
git-pull
macro name='GitClone' gitUri='string' gitBranch='string'
git-clone
macro name='GitConfig' gitOptionName='string' gitOptionValue='string' gitFolder='string'
git-config
macro name='GitStatus' gitFolder='string'
git gitCommand='status'
macro name='GitClean' gitFolder='string'
git gitCommand='clean -xdf'
macro name='GitCommand' gitFolder='string' gitCommand='string'
git
macro name='Exec' program='string' commandline='string' workingdir='string'
exec
macro name='NuGetPackagesAdd' sourcePackagesDir='string' targetPackagesDir='string'
nuget-packages-add
functions
@{
static bool UseHttps(string directory)
{
var filename = Path.Combine(directory, ".git", "config");
if (!File.Exists(filename))
{
Console.WriteLine("Unable to find '{0}' file", filename);
return false;
}
var url = ReadOriginUrl(filename);
return IsHttpsUrl(url);
}
// Perform equivalent of `git config remote.origin.url` but directly
// read config file to get value.
static string ReadOriginUrl(string filename)
{
// Subsection portion of configuration name is case-sensitive; rest
// of name is case-insensitive.
var beginOriginSection = new Regex(@"^\[(?i:remote) ""origin""\]\s*$");
var beginSection = new Regex(@"^\[");
var urlLine = new Regex(@"^\s+url = (\S+)\s*$", RegexOptions.IgnoreCase);
var inRemoteOriginSection = false;
foreach (var line in File.ReadAllLines(filename))
{
if (beginOriginSection.IsMatch(line))
{
inRemoteOriginSection = true;
continue;
}
if (inRemoteOriginSection)
{
if (beginSection.IsMatch(line))
{
// Read through the section without finding URL line.
break;
}
var match = urlLine.Match(line);
if (match.Success && match.Groups.Count == 2 && match.Groups[1].Success)
{
return match.Groups[1].Value;
}
}
}
Console.WriteLine("Unable to parse '{0}' file", filename);
return null;
}
static bool IsHttpsUrl(string url)
{
if (string.IsNullOrEmpty(url))
{
return false;
}
return url.StartsWith("https://", System.StringComparison.OrdinalIgnoreCase);
}
static bool IsAccessible(string key)
{
var req = WebRequest.CreateHttp("https://github.com/aspnet/" + key);
req.Method = "HEAD";
try
{
using (req.GetResponse())
{
// intentionally empty
}
}
catch (WebException ex)
{
if (ex.Response != null &&
((HttpWebResponse)ex.Response).StatusCode == HttpStatusCode.NotFound)
{
return false;
}
// Ignore any other exception. They should surface as part of git clone with well-known messages.
}
return true;
}
IEnumerable<string> GetAllRepos()
{
var nonDefaultRepos = new[]
{
"DNX",
"Coherence",
"Coherence-Signed",
"dnvm",
"Setup",
};
return Enumerable.Concat(nonDefaultRepos, repositories);
}
bool ShouldSkipCopyingNugetConfig(string repo)
{
var skipList = new string[]{"Coherence-Signed"};
return skipList.Any(r => r == repo);
}
void CloneOrUpdate(string repo)
{
var repoUrl = gitHubUriPrefix + repo + ".git";
if(Directory.Exists(repo))
{
GitPull(repoUrl, BUILD_BRANCH, repo);
}
else
{
if (useHttps &&
!IsAccessible(repo))
{
if (SKIP_NO_CREDENTIALS)
{
// This is used on the XPlat CI. Must return because otherwise git will wait for user
// input and hang the build
Log.Warn(string.Format("The repo at '{0}' is not accesible and it will not be cloned.", repoUrl));
return;
}
Log.Warn(string.Format("The repo at '{0}' is not accessible. If you do not have access to this repository, skip the git prompt" +
" for credentials to skip cloning this repository. To avoid this prompt, re-clone the Universe repository over ssh.",
repoUrl));
}
try
{
GitClone(repoUrl, BUILD_BRANCH);
}
catch (Exception ex)
{
Log.Warn(string.Format("Unable to clone repository at '{0}': {1}", repoUrl, ex.Message));
return;
}
}
}
string FormatForTeamCity(string input)
{
return input.Replace("|", "||")
.Replace("'", "|'")
.Replace("\r", "|r")
.Replace("\n", "|n")
.Replace("]", "|]");
}
// Currently there are two packages that need to be special cased.
// TODO: Revisit as this is changed in the near future.
string ProcessExcludesForPackages(string jsonEntry, bool stable=false)
{
var newEntry = jsonEntry.Replace("Microsoft.Net.Http.Client\": \"1.0.0-*\"", "Microsoft.Net.Http.Client\": \"1.0.0-beta3\"");
newEntry = newEntry.Replace("Microsoft.AspNet.HttpFeature\": { \"version\": \"1.0.0-*\"", "Microsoft.AspNet.HttpFeature\": { \"version\": \"1.0.0-beta2\"");
return newEntry;
}
// Create a search replace expression based on branch, prerelease tag and buildNumber
string BuildVersionExpression(string branch, string preRelease = "", string buildNumber = "")
{
var stableBranches = new[] {"master"};
var builder = new StringBuilder();
// Add pre release version tag
if(!String.IsNullOrEmpty(preRelease))
{
builder.Append("-");
builder.Append(preRelease);
}
// If buildnumber is provided, append.
// This for CORE CLR packages
if(!String.IsNullOrEmpty(buildNumber))
{
builder.Append("-");
builder.Append(buildNumber);
}
else if(!stableBranches.Contains(branch))
{
builder.Append("-*");
}
return builder.ToString();
}
// Verify if this is indeed the project.json or project.lock.json under project
bool ShouldIncludeProjectJson(string jsonPath)
{
// Should be under src, test or samples folders
if(!jsonPath.StartsWith("src") && !jsonPath.StartsWith("test") && !jsonPath.StartsWith("samples"))
{
return false;
}
// Within the specified folders, few projects generate under build folders. They can be excluded
if(jsonPath.Contains("bin") || jsonPath.Contains("Debug")
|| jsonPath.Contains("artifacts") || jsonPath.Contains("node_modules") || !jsonPath.Contains(".json"))
{
return false;
}
return true;
}
static IEnumerable<string> GetRepositoriesToBuild()
{
IEnumerable<string> reposToBuild = new HashSet<string>(StringComparer.OrdinalIgnoreCase) {
// The repos list is topologically sorted based on build order
"PlatformAbstractions",
"Common",
"JsonPatch",
"FileSystem",
"Configuration",
"DependencyInjection",
"EventNotification",
"Options",
"Logging",
"dnx-watch",
"HtmlAbstractions",
"UserSecrets",
"DataProtection",
"HttpAbstractions",
"Testing",
"aspnet.xunit",
"Microsoft.Data.Sqlite",
"Caching",
"Razor",
"RazorTooling",
"Hosting",
"EntityFramework",
"WebListener",
"KestrelHttpServer",
"IISIntegration",
"ServerTests",
"Session",
"CORS",
"Routing",
"StaticFiles",
"Diagnostics",
"Security",
"Antiforgery",
"WebSockets",
"Localization",
"BasicMiddleware",
"Proxy",
"Mvc",
"Identity",
"Scaffolding",
"SignalR-Server",
"SignalR-SQLServer",
"SignalR-Redis",
"SignalR-ServiceBus",
"BrowserLink",
"Entropy",
"MusicStore",
};
var repositoryInclude = Environment.GetEnvironmentVariable("KOREBUILD_REPOSITORY_INCLUDE");
if (!string.IsNullOrEmpty(repositoryInclude))
{
reposToBuild = new HashSet<string>(
repositoryInclude.Split(new string[] {","}, StringSplitOptions.RemoveEmptyEntries),
StringComparer.OrdinalIgnoreCase);
}
var repositoryExclude = Environment.GetEnvironmentVariable("KOREBUILD_REPOSITORY_EXCLUDE");
if (!string.IsNullOrEmpty(repositoryExclude))
{
var reposToExclude = repositoryExclude.Split(new string[] {","}, StringSplitOptions.RemoveEmptyEntries);
reposToBuild = reposToBuild.Except(reposToExclude);
}
return reposToBuild.ToList();
}
}