aspnetcore/makefile.shade

702 lines
23 KiB
Plaintext

var PROJECT='AspNet'
var VERSION='0.2.1'
use namespace='System'
use namespace='System.IO'
use namespace='System.Collections.Generic'
use namespace='System.Net'
use namespace='System.Linq'
use namespace='System.Text'
use namespace='System.Text.RegularExpressions'
use namespace='System.Threading.Tasks'
use import="BuildEnv"
functions
@{
static string BUILD_BRANCH = Environment.GetEnvironmentVariable("BUILD_BRANCH") ?? "dev";
static string BASE_DIR = Directory.GetCurrentDirectory();
static string TARGET_DIR = Path.Combine(BASE_DIR, "artifacts", "build");
static bool SKIP_NO_CREDENTIALS = Environment.GetEnvironmentVariable("UNIVERSE_SKIP_NO_CREDENTIALS") == "1";
string[] repos = new[] {
// The repos list is topologically sorted based on build order
"Common",
"Configuration",
"Caching",
"DataProtection",
"DependencyInjection",
"EventNotification",
"Options",
"Logging",
"Testing",
"Diagnostics",
"Microsoft.Data.Sqlite",
"EntityFramework",
"FileSystem",
"HttpAbstractions",
"Hosting",
"Helios",
"Antiforgery",
"Identity",
"Razor",
"RazorTooling",
"Routing",
"Mvc",
"Scaffolding",
"Security",
"SignalR-Server",
"SignalR-SQLServer",
"SignalR-Redis",
"StaticFiles",
"WebListener",
"KestrelHttpServer",
"WebSockets",
"Session",
"UserSecrets",
"CORS",
"Entropy",
};
// Doesn't build on Mono since their contracts don't match
string[] excludeReposOnMono = new[] { "Helios" };
static bool useHttps = UseHttps(BASE_DIR);
static string gitHubUriPrefix = useHttps ? "https://github.com/aspnet/" : "git@github.com:aspnet/";
}
var buildTarget = "compile"
@{
var kBuildVersion = Environment.GetEnvironmentVariable("DNX_BUILD_VERSION");
if (!string.IsNullOrEmpty(kBuildVersion))
{
VERSION += "-" + kBuildVersion;
}
else
{
VERSION += "-" + BuildNumber;
}
}
#default .compile
#pull
#compile .pull
#install .pull
#pack
directory create='${TARGET_DIR}'
nuget-pack nuspecFile='${Path.Combine(BASE_DIR, "KoreBuild.nuspec")}' packageVersion='${VERSION}' outputDir='${TARGET_DIR}'
#pack-install .pack
nuget-local-publish sourcePackagesDir='${TARGET_DIR}'
#git-pull target='pull'
@{
Parallel.ForEach(repos, repo =>
{
CloneOrUpdate(repo);
});
}
#verify-all .pull .change-default-build-target .only-compile
#smoke-test-mono .pull .fix-project-json .change-default-build-target .only-compile
#fix-project-json
@{
repos = repos.Except(excludeReposOnMono).ToArray();
}
-// Fix project.json to remove .Net portable references
for each='var repo in repos'
for each='var file in Files.Include(repo + "/**" + "/project.json")'
update-file updateFile='${file}'
@{
updateText = updateText.Replace(".NETPortable,Version=4.6,Profile=Profile151", "foo");
updateText = updateText.Replace(".NETPortable,Version=v4.6,Profile=Profile151", "foo");
}
#change-default-build-target
@{
buildTarget = "verify";
}
#init
@{
var templatePath = Path.Combine(BASE_DIR, "build-template");
var templateFiles = Files.Include(templatePath + Path.DirectorySeparatorChar + "*.*").Select(Path.GetFileName).ToList();
foreach(var repo in repos)
{
foreach (string fileName in templateFiles)
{
var targetFile = Path.Combine(Directory.GetCurrentDirectory(), repo, fileName);
var sourceFile = Path.Combine(Directory.GetCurrentDirectory(), templatePath, fileName);
// Don't update the makefile
if (fileName.Equals("makefile.shade", StringComparison.OrdinalIgnoreCase) && File.Exists(targetFile))
{
continue;
}
// Don't update extra configuration files
if (fileName.Equals("NuGet.master.config", StringComparison.OrdinalIgnoreCase) ||
fileName.Equals("NuGet.release.config", StringComparison.OrdinalIgnoreCase))
{
continue;
}
if (!File.Exists(targetFile) ||
(File.ReadAllText(sourceFile) != File.ReadAllText(targetFile)))
{
Log.Info("Updating " + fileName + " to " + repo);
File.Copy(sourceFile, targetFile, true);
}
}
}
}
#check-update-release
@{
Parallel.ForEach(GetAllRepos(), CloneOrUpdate);
Log.Info("************************************* Checking repos for diffs *************************");
foreach (var repo in GetAllRepos())
{
Log.Info("Checking repo: " + repo);
GitCommand(repo, "log origin/dev..origin/release");
}
Log.Info("If there are no conflicts in the repos, continue with the 'update-release' target");
}
#update-release
-// Merge dev branch to release
@{
foreach (var repo in GetAllRepos())
{
CloneOrUpdate(repo);
GitCommand(repo, "checkout origin/dev -B release");
if(!ShouldSkipCopyingNugetConfig(repo))
{
File.Copy(Path.Combine("build-template", "NuGet.release.config"),
Path.Combine(repo, "NuGet.config"),
overwrite: true);
GitCommand(repo, "commit -am \"Updating to release NuGet.config\"");
}
GitCommand(repo, "push origin release:release");
GitCommand(repo, "checkout origin/dev -B dev");
GitCommand(repo, "merge release -s ours");
GitCommand(repo, "push origin dev:dev");
}
}
#pull-all
-Parallel.ForEach(GetAllRepos(), CloneOrUpdate);
#update-master
-// Merge release branch to master
@{
foreach (var repo in GetAllRepos())
{
CloneOrUpdate(repo);
GitCommand(repo, "checkout origin/release -B master");
if(File.Exists(Path.Combine(repo, "NuGet.config")))
{
File.Copy(Path.Combine("build-template", "NuGet.master.config"),
Path.Combine(repo, "NuGet.config"),
overwrite: true);
GitCommand(repo, "commit -am \"Updating NuGet.config\"");
}
GitCommand(repo, "push origin master:master -f");
}
}
#pin-version
-// Pin versions of packages in project.json and updated project.lock.json
-// More information https://github.com/aspnet/Universe/wiki/%23pin-version-:-Pinning-package-version-for-a-particular-release-in-project.json
@{
var coherenceFeed = Environment.GetEnvironmentVariable("COHERENCE_FEED");
if (string.IsNullOrEmpty(coherenceFeed))
{
throw new Exception("COHERENCE_FEED not specified. Usually this is Packages-NoTimestamp directory of Coherence-Signed.");
}
var excludeReposForJson = new[]
{
"Coherence",
"Coherence-Signed",
"dnvm",
"Entropy"
};
var repos = GetAllRepos().Except(excludeReposForJson);
Parallel.ForEach(repos, CloneOrUpdate);
foreach (var repo in repos)
{
GitCommand(repo, "checkout master");
if (File.Exists(Path.Combine(repo, "build.cmd")))
{
File.Copy(Path.Combine("build-template", "build.cmd"),
Path.Combine(repo, "build.cmd"),
overwrite: true);
}
var repoPath = Path.Combine(Directory.GetCurrentDirectory(), repo);
Exec("dnx", string.Format(@".\tools\PinVersion run ""{0}"" ""{1}""", repo, coherenceFeed), string.Empty);
// Build projects to produce project.lock.json
Exec("build.cmd", "compile", repo);
// Replace lock=true in project.lock.json
foreach(var file in Directory.GetFiles(repo, "project.lock.json", SearchOption.AllDirectories))
{
var contents = File.ReadAllText(file);
var newContents = contents.Replace("\"locked\": false", "\"locked\": true");
File.WriteAllText(file, newContents);
}
// Add the project.json and project.lock.json files
foreach(var file in Directory.GetFiles(repo, "project*", SearchOption.AllDirectories))
{
var gitFilePath = file.Substring(file.IndexOf(repo) + repo.Length + 1);
// Check if this needs to be included
if(ShouldIncludeProjectJson(gitFilePath))
{
GitCommand(repo, string.Format("add -f {0}", gitFilePath));
}
}
GitCommand(repo, "commit -am \"Updating json files to pin versions and build.cmd to pin KoreBuild and DNX\"");
GitCommand(repo, "push origin master:master -f");
}
}
#update-prerelease-tags
-// Update tags on each repo to have the latest release tag
@{
var newPreRelease = Environment.GetEnvironmentVariable("PRERELEASETAG");
if(string.IsNullOrEmpty(newPreRelease))
{
Log.Warn("No prerelease tag defined");
return;
}
var versionFile = "version.txt";
foreach (var repo in GetAllRepos())
{
CloneOrUpdate(repo);
GitCommand(repo, "checkout master");
GitCommand(repo, "pull --tags");
try
{
GitCommand(repo, string.Format("describe --tags > ..\\{0}", versionFile));
}
catch(Exception e)
{
Log.Warn(string.Format("{0} repo not tagged. Skipping....", repo));
continue;
}
var version = File.ReadAllText(versionFile);
File.Delete(versionFile);
Log.Info(string.Format("Current version on repo {0} is {1}", repo, version));
var majorVersion = version.Split(new string[]{"-"}, StringSplitOptions.None)[0];
var newVersion = majorVersion + string.Format("-{0}", newPreRelease);
Log.Info(string.Format("New version for repo is {0}", newVersion));
GitCommand(repo, string.Format("tag -f -a {0} -m \"Tag for new release {0}\"", newVersion));
GitCommand(repo, "push -f --tags ");
}
}
#only-compile target='compile'
@{
var failed = new Dictionary<string, Exception>();
var skipped = new List<string>();
foreach(var repo in repos)
{
var blockName = string.Format("Building {0}", repo);
if (IsTeamCity)
{
Log.Info(string.Format("##teamcity[blockOpened name='{0}']", FormatForTeamCity(blockName)));
}
try
{
Log.Info(blockName);
if (SKIP_NO_CREDENTIALS)
{
if (!Directory.Exists(repo))
{
// The directory was not cloned because the credentials were not specified so don't try to build it
skipped.Add(repo);
Log.Warn(string.Format("The repo {0} does not exist locally and it will not be built.", repo));
continue;
}
}
if (IsMono)
{
Exec("build.sh", buildTarget, repo);
}
else
{
Exec("build.cmd", buildTarget, repo);
}
Log.Info(string.Format("Build {0} succeeded", repo));
}
catch(Exception ex)
{
Log.Warn(string.Format("Build {0} failed: {1}", repo, ex.Message));
failed[repo] = ex;
}
finally
{
if (IsTeamCity)
{
Log.Info(string.Format("##teamcity[blockClosed name='{0}']", FormatForTeamCity(blockName)));
}
}
}
foreach(var repo in repos)
{
Exception ex;
if (failed.TryGetValue(repo, out ex))
{
Log.Warn(string.Format("Build {0} failed: {1}", repo, ex.Message));
if (IsTeamCity)
{
Log.Warn(string.Format("##teamcity[message text='{0}' errorDetails='{1}' status='ERROR']",
FormatForTeamCity(ex.Message),
FormatForTeamCity(ex.StackTrace)));
}
}
else if (skipped.Contains(repo))
{
Log.Warn(string.Format("Build {0} skipped", repo));
}
else
{
Log.Info(string.Format("Build {0} succeeded", repo));
}
}
if (failed.Any())
{
throw new Exception("One or more repos failed to build");
}
}
#only-install target='install'
@{
foreach(var repo in repos)
{
if (IsMono)
{
Exec("build.sh", "install", repo);
}
else
{
Exec("build.cmd", "install", repo);
}
}
}
#run-snapshot-manager
@{
Exec(@".nuget\nuget.exe", @"restore -out packages tools\TCDependencyManager\packages.config", "");
var programFiles = Environment.GetFolderPath(Environment.SpecialFolder.ProgramFilesX86);
var msbuildPath = Path.Combine(programFiles, "MSBuild", "12.0", "Bin", "MsBuild.exe");
Exec(msbuildPath, "TCDependencyManager.csproj", @"tools\TCDependencyManager");
Exec(@"tools\TCDependencyManager\bin\Debug\TCDependencyManager.exe", "", "");
}
#git-status description='Show status of repos known by Universe'
@{
foreach(var repo in repos)
{
GitStatus(repo);
}
}
#git-clean description='REMOVE ALL CHANGES to the working directory'
@{
Console.WriteLine("This runs `git clean -xfd` in all non-Universe repos.");
Console.WriteLine("This should REMOVE ALL CHANGES to the working directory.");
Console.Write("***** Are you sure? ***** (Y or anything else)? ");
if (Console.ReadLine() != "Y")
{
throw new Exception("git-clean cancelled");
}
foreach(var repo in repos)
{
GitClean(repo);
}
}
macro name='GitPull' gitUri='string' gitBranch='string' gitFolder='string'
git-pull
macro name='GitClone' gitUri='string' gitBranch='string'
git-clone
macro name='GitConfig' gitOptionName='string' gitOptionValue='string' gitFolder='string'
git-config
macro name='GitStatus' gitFolder='string'
git gitCommand='status'
macro name='GitClean' gitFolder='string'
git gitCommand='clean -xdf'
macro name='GitCommand' gitFolder='string' gitCommand='string'
git
macro name='Exec' program='string' commandline='string' workingdir='string'
exec
functions
@{
static bool UseHttps(string directory)
{
var filename = Path.Combine(directory, ".git", "config");
if (!File.Exists(filename))
{
Console.WriteLine("Unable to find '{0}' file", filename);
return false;
}
var url = ReadOriginUrl(filename);
return IsHttpsUrl(url);
}
// Perform equivalent of `git config remote.origin.url` but directly
// read config file to get value.
static string ReadOriginUrl(string filename)
{
// Subsection portion of configuration name is case-sensitive; rest
// of name is case-insensitive.
var beginOriginSection = new Regex(@"^\[(?i:remote) ""origin""\]\s*$");
var beginSection = new Regex(@"^\[");
var urlLine = new Regex(@"^\s+url = (\S+)\s*$", RegexOptions.IgnoreCase);
var inRemoteOriginSection = false;
foreach (var line in File.ReadAllLines(filename))
{
if (beginOriginSection.IsMatch(line))
{
inRemoteOriginSection = true;
continue;
}
if (inRemoteOriginSection)
{
if (beginSection.IsMatch(line))
{
// Read through the section without finding URL line.
break;
}
var match = urlLine.Match(line);
if (match.Success && match.Groups.Count == 2 && match.Groups[1].Success)
{
return match.Groups[1].Value;
}
}
}
Console.WriteLine("Unable to parse '{0}' file", filename);
return null;
}
static bool IsHttpsUrl(string url)
{
if (string.IsNullOrEmpty(url))
{
return false;
}
return url.StartsWith("https://", System.StringComparison.OrdinalIgnoreCase);
}
static bool IsAccessible(string key)
{
var req = WebRequest.CreateHttp("https://github.com/aspnet/" + key);
req.Method = "HEAD";
try
{
using (req.GetResponse());
}
catch (WebException ex)
{
if (ex.Response != null &&
((HttpWebResponse)ex.Response).StatusCode == HttpStatusCode.NotFound)
{
return false;
}
// Ignore any other exception. They should surface as part of git clone with well-known messages.
}
return true;
}
IEnumerable<string> GetAllRepos()
{
var nonDefaultRepos = new[]
{
"XRE",
"MusicStore",
"Coherence",
"Coherence-Signed",
"dnvm",
};
return Enumerable.Concat(nonDefaultRepos, repos);
}
bool ShouldSkipCopyingNugetConfig(string repo)
{
var skipList = new string[]{"Coherence-Signed"};
return skipList.Any(r => r == repo);
}
void CloneOrUpdate(string repo)
{
var repoUrl = gitHubUriPrefix + repo + ".git";
if(Directory.Exists(repo))
{
GitPull(repoUrl, BUILD_BRANCH, repo);
}
else
{
if (useHttps &&
!IsAccessible(repo))
{
if (SKIP_NO_CREDENTIALS)
{
// This is used on the XPlat CI. Must return because otherwise git will wait for user
// input and hang the build
Log.Warn(string.Format("The repo at '{0}' is not accesible and it will not be cloned.", repoUrl));
return;
}
Log.Warn(string.Format("The repo at '{0}' is not accessible. If you do not have access to this repository, skip the git prompt" +
" for credentials to skip cloning this repository. To avoid this prompt, re-clone the Universe repository over ssh.",
repoUrl));
}
try
{
GitClone(repoUrl, BUILD_BRANCH);
}
catch (Exception ex)
{
Log.Warn(string.Format("Unable to clone repository at '{0}': {1}", repoUrl, ex.Message));
return;
}
// Configuring Git bug tracking settings hangs from time to time on Win7/2008 when running as part of the CI build
if (!IsTeamCity)
{
GitConfig("bugtraq.url", "http://github.com/aspnet/" + repo + "/issues/%BUGID%", repo);
GitConfig("bugtraq.logregex", @"#(\d+)", repo);
}
}
}
string FormatForTeamCity(string input)
{
return input.Replace("|", "||")
.Replace("'", "|'")
.Replace("\r", "|r")
.Replace("\n", "|n")
.Replace("]", "|]");
}
// Currently there are two packages that need to be special cased.
// TODO: Revisit as this is changed in the near future.
string ProcessExcludesForPackages(string jsonEntry, bool stable=false)
{
var newEntry = jsonEntry.Replace("Microsoft.Net.Http.Client\": \"1.0.0-*\"", "Microsoft.Net.Http.Client\": \"1.0.0-beta3\"");
newEntry = newEntry.Replace("Microsoft.AspNet.HttpFeature\": { \"version\": \"1.0.0-*\"", "Microsoft.AspNet.HttpFeature\": { \"version\": \"1.0.0-beta2\"");
return newEntry;
}
// Create a search replace expression based on branch, prerelease tag and buildNumber
string BuildVersionExpression(string branch, string preRelease = "", string buildNumber = "")
{
var stableBranches = new[] {"master"};
var builder = new StringBuilder();
// Add pre release version tag
if(!String.IsNullOrEmpty(preRelease))
{
builder.Append("-");
builder.Append(preRelease);
}
// If buildnumber is provided, append.
// This for CORE CLR packages
if(!String.IsNullOrEmpty(buildNumber))
{
builder.Append("-");
builder.Append(buildNumber);
}
else if(!stableBranches.Contains(branch))
{
builder.Append("-*");
}
return builder.ToString();
}
// Verify if this is indeed the project.json or project.lock.json under project
bool ShouldIncludeProjectJson(string jsonPath)
{
// Should be under src, test or samples folders
if(!jsonPath.StartsWith("src") && !jsonPath.StartsWith("test") && !jsonPath.StartsWith("samples"))
{
return false;
}
// Within the specified folders, few projects generate under build folders. They can be excluded
if(jsonPath.Contains("bin") || jsonPath.Contains("Debug")
|| jsonPath.Contains("artifacts") || jsonPath.Contains("node_modules") || !jsonPath.Contains(".json"))
{
return false;
}
return true;
}
}