1090 lines
38 KiB
Plaintext
1090 lines
38 KiB
Plaintext
|
|
var VERSION='0.2.1'
|
|
|
|
use-ci-loggers
|
|
use namespace='System'
|
|
use namespace='System.IO'
|
|
use namespace='System.Collections.Concurrent'
|
|
use namespace='System.Collections.Generic'
|
|
use namespace='System.Net'
|
|
use namespace='System.Linq'
|
|
use namespace='System.Text'
|
|
use namespace='System.Text.RegularExpressions'
|
|
use namespace='System.Threading.Tasks'
|
|
use import="BuildEnv"
|
|
use import="Files"
|
|
use import="Json"
|
|
|
|
functions
|
|
@{
|
|
const string UniverseCommitsFileName = "commits-universe";
|
|
|
|
private static bool Quiet { get; set; }
|
|
|
|
static string BUILD_BRANCH = Environment.GetEnvironmentVariable("BUILD_BRANCH") ?? "dev";
|
|
static string BASE_DIR = Directory.GetCurrentDirectory();
|
|
static string TARGET_DIR = Path.Combine(BASE_DIR, "artifacts", "build");
|
|
static string COHERENCE_CACHE_DIR = Environment.GetEnvironmentVariable("COHERENCE_CACHE_DIR") ?? Path.Combine(BASE_DIR, ".coherence-cache");
|
|
static string DROPS_SHARE = Environment.GetEnvironmentVariable("ASPNETCI_DROPS_SHARE") ?? @"\\aspnetci\drops";
|
|
static bool SKIP_NO_CREDENTIALS = Environment.GetEnvironmentVariable("UNIVERSE_SKIP_NO_CREDENTIALS") == "1";
|
|
|
|
// Doesn't build on Mono since their contracts don't match
|
|
string[] repositories = GetRepositoriesToBuild().ToArray();
|
|
string[] noSrcRepositoryExclude = GetNoSrcDeleteRepositories().ToArray();
|
|
|
|
static bool useHttps = UseHttps(BASE_DIR);
|
|
static string gitHubUriPrefix = useHttps ? "https://github.com/aspnet/" : "git@github.com:aspnet/";
|
|
}
|
|
|
|
var buildTarget = "compile"
|
|
|
|
@{
|
|
var kBuildVersion = Environment.GetEnvironmentVariable("DNX_BUILD_VERSION");
|
|
if (!string.IsNullOrEmpty(kBuildVersion))
|
|
{
|
|
VERSION += "-" + kBuildVersion;
|
|
}
|
|
else
|
|
{
|
|
VERSION += "-" + BuildNumber;
|
|
}
|
|
}
|
|
|
|
#default .compile
|
|
|
|
#--quiet
|
|
@{
|
|
Quiet = true;
|
|
}
|
|
|
|
#pull
|
|
#compile .pull
|
|
#install .pull
|
|
|
|
#pack
|
|
directory create='${TARGET_DIR}'
|
|
|
|
#pack-install .pack
|
|
nuget-local-publish sourcePackagesDir='${TARGET_DIR}'
|
|
|
|
#git-pull target='pull'
|
|
@{
|
|
Parallel.ForEach(repositories, repo =>
|
|
{
|
|
CloneOrUpdate(repo);
|
|
});
|
|
}
|
|
|
|
#sync-commits
|
|
@{
|
|
var commitsToSync = GetCommitsToSync();
|
|
|
|
Parallel.ForEach(repositories, repo =>
|
|
{
|
|
if (commitsToSync.ContainsKey(repo))
|
|
{
|
|
var syncHash = commitsToSync[repo];
|
|
Console.WriteLine(string.Format("Syncing {0} to {1}", repo, syncHash));
|
|
Git("reset --hard " + syncHash, repo);
|
|
}
|
|
});
|
|
}
|
|
|
|
#verify-all .pull .change-default-build-target-to-verify .build-all
|
|
|
|
#ci-test .cache-coherence .pull .sync-commits .remove-src-folders .change-default-build-target-to-verify .build-all
|
|
|
|
#ci-build
|
|
@{
|
|
var ciVolatileShare = Environment.GetEnvironmentVariable("CI_VOLATILE_SHARE");
|
|
var nugetExe = Path.Combine(".build", "nuget.exe");
|
|
var universeArtifacts = "artifacts";
|
|
var universeBuild = Path.Combine(universeArtifacts, "build");
|
|
Directory.CreateDirectory(universeBuild);
|
|
buildTarget = Environment.GetEnvironmentVariable("KOREBUILD_BUILD_TARGETS") ?? "--quiet compile nuget-install";
|
|
var blockLogger = Log as IBlockLogger;
|
|
|
|
if (blockLogger != null)
|
|
{
|
|
blockLogger.StartBlock("Cloning repos");
|
|
}
|
|
|
|
Parallel.ForEach(repositories, repo =>
|
|
{
|
|
if (Directory.Exists(repo))
|
|
{
|
|
GitCommand("", string.Format("pull --ff-only --quiet --depth 1 git@github.com:aspnet/{1}.git {0}:{0}", BUILD_BRANCH, repo));
|
|
}
|
|
else
|
|
{
|
|
GitCommand("", string.Format("clone --quiet --depth 1 --branch {0} git@github.com:aspnet/{1}.git", BUILD_BRANCH, repo));
|
|
}
|
|
});
|
|
|
|
if (blockLogger != null)
|
|
{
|
|
blockLogger.EndBlock("Cloning repos");
|
|
}
|
|
|
|
var batchedRepos = GetBuildGraph();
|
|
Log.Info("Building repositories in batches: ");
|
|
foreach (var repos in batchedRepos)
|
|
{
|
|
Log.Info(string.Format("{0} - {1}", repos.Key, string.Join(", ", repos)));
|
|
}
|
|
|
|
var commits = new ConcurrentDictionary<string, string>();
|
|
|
|
foreach (var batch in batchedRepos)
|
|
{
|
|
Parallel.ForEach(batch.ToArray(), new ParallelOptions { MaxDegreeOfParallelism = 4 }, repo =>
|
|
{
|
|
|
|
var blockName = string.Format("Building {0}", repo);
|
|
|
|
if (blockLogger != null)
|
|
{
|
|
blockLogger.StartBlock(blockName);
|
|
}
|
|
|
|
if (!IsLinux)
|
|
{
|
|
Exec("cmd", "/C xcopy /S/Q/I/Y .build " + Path.Combine(repo, ".build"), "");
|
|
}
|
|
|
|
Exec(CreateBuildWithFlowId(repo), buildTarget, repo);
|
|
var repoArtifacts = Path.Combine(repo, "artifacts");
|
|
var repoBuild = Path.Combine(repoArtifacts, "build");
|
|
if (Directory.Exists(repoBuild))
|
|
{
|
|
foreach (var source in Directory.EnumerateFiles(repoBuild, "*.nupkg"))
|
|
{
|
|
File.Copy(source, Path.Combine(universeBuild, Path.GetFileName(source)), overwrite: true);
|
|
}
|
|
|
|
var commitFile = Path.Combine(repoArtifacts, "commit");
|
|
if (!File.Exists(commitFile))
|
|
{
|
|
throw new FileNotFoundException("Couldn't find the commit file for " + repo + ": " + commitFile);
|
|
}
|
|
|
|
commits.TryAdd(repo, File.ReadAllLines(commitFile)[0]);
|
|
|
|
if (!string.IsNullOrEmpty(ciVolatileShare))
|
|
{
|
|
Log.Info("Publishing packages to " + ciVolatileShare);
|
|
if (string.IsNullOrEmpty(nugetExe))
|
|
{
|
|
Log.Warn("PUSH_NUGET_EXE not specified.");
|
|
}
|
|
else
|
|
{
|
|
NuGetPackagesAdd(repoBuild, ciVolatileShare);
|
|
}
|
|
}
|
|
}
|
|
|
|
Log.Info(string.Format("Build {0} succeeded", repo));
|
|
|
|
if (blockLogger != null)
|
|
{
|
|
blockLogger.EndBlock(blockName);
|
|
}
|
|
});
|
|
}
|
|
|
|
var commitsAsString = string.Join("\n", commits.Select(c => c.Key + ":" + c.Value));
|
|
File.WriteAllText(Path.Combine(universeArtifacts, "commits"), commitsAsString);
|
|
}
|
|
|
|
#remove-src-folders
|
|
@{
|
|
foreach (var repo in GetRepositoriesToBuild())
|
|
{
|
|
if (!noSrcRepositoryExclude.Contains(repo))
|
|
{
|
|
RemoveSrcFolder(repo);
|
|
}
|
|
else
|
|
{
|
|
Console.WriteLine("Keeping the sources for " + repo + " because it's explicitly excluded");
|
|
}
|
|
}
|
|
}
|
|
|
|
#cache-coherence
|
|
@{
|
|
if (!string.IsNullOrWhiteSpace(DROPS_SHARE))
|
|
{
|
|
var coherenceShare = Path.Combine(DROPS_SHARE, "Coherence", BUILD_BRANCH);
|
|
var latestBuild = Directory.EnumerateDirectories(coherenceShare)
|
|
.Select(d => Path.GetFileName(d))
|
|
.Where(d => { int _; return int.TryParse(d, out _); })
|
|
.OrderByDescending(d => d)
|
|
.First();
|
|
var targetCacheDir = Path.Combine(COHERENCE_CACHE_DIR, latestBuild);
|
|
var targetCacheBuildDir = Path.Combine(targetCacheDir, "build");
|
|
|
|
Log.Info("Latest Coherence build is " + latestBuild);
|
|
|
|
if (!Directory.Exists(targetCacheDir))
|
|
{
|
|
var latestBuildShare = Path.Combine(coherenceShare, latestBuild, "build");
|
|
|
|
Log.Info("Caching Coherence build " + latestBuild + " at " + targetCacheDir);
|
|
CopyFolder(latestBuildShare, targetCacheBuildDir, overwrite: true);
|
|
File.Copy(Path.Combine(coherenceShare, latestBuild, UniverseCommitsFileName), Path.Combine(targetCacheDir, UniverseCommitsFileName));
|
|
}
|
|
else
|
|
{
|
|
Log.Info("Coherence build " + latestBuild + " already cached at " + targetCacheDir);
|
|
}
|
|
|
|
Environment.SetEnvironmentVariable("NUGET_VOLATILE_FEED_ASPNETVNEXT", targetCacheBuildDir);
|
|
Environment.SetEnvironmentVariable("NUGET_VOLATILE_FEED_EXTERNAL", Path.Combine(DROPS_SHARE, "latest-packages", "external", BUILD_BRANCH));
|
|
Environment.SetEnvironmentVariable("UNIVERSE_COMMITS_FILE", Path.Combine(targetCacheDir, "commits-universe"));
|
|
}
|
|
}
|
|
|
|
#change-default-build-target-to-verify
|
|
- buildTarget = "verify";
|
|
|
|
#change-default-build-target-for-coherence-build
|
|
- buildTarget = Environment.GetEnvironmentVariable("KOREBUILD_BUILD_TARGETS") ?? "compile nuget-install";
|
|
|
|
#init
|
|
@{
|
|
var templatePath = Path.Combine(BASE_DIR, "build-template");
|
|
var templateFiles = Files.Include(templatePath + Path.DirectorySeparatorChar + "*.*").Select(Path.GetFileName).ToList();
|
|
|
|
foreach(var repo in repositories)
|
|
{
|
|
foreach (string fileName in templateFiles)
|
|
{
|
|
var targetFile = Path.Combine(Directory.GetCurrentDirectory(), repo, fileName);
|
|
var sourceFile = Path.Combine(Directory.GetCurrentDirectory(), templatePath, fileName);
|
|
|
|
// Don't update the makefile
|
|
if (fileName.Equals("makefile.shade", StringComparison.OrdinalIgnoreCase) && File.Exists(targetFile))
|
|
{
|
|
continue;
|
|
}
|
|
|
|
// Don't update extra configuration files
|
|
if (fileName.Equals("NuGet.master.config", StringComparison.OrdinalIgnoreCase) ||
|
|
fileName.Equals("NuGet.release.config", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (!File.Exists(targetFile) ||
|
|
(File.ReadAllText(sourceFile) != File.ReadAllText(targetFile)))
|
|
{
|
|
Log.Info("Updating " + fileName + " to " + repo);
|
|
File.Copy(sourceFile, targetFile, true);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
#update-release
|
|
-// Merge dev branch to release
|
|
@{
|
|
Parallel.ForEach(GetAllRepos(), CloneOrUpdate);
|
|
|
|
Log.Info("************************************* Checking repos for diffs *************************");
|
|
|
|
foreach (var repo in GetAllRepos())
|
|
{
|
|
Log.Info("Checking repo: " + repo);
|
|
// Check if the repo previously had a release branch
|
|
try
|
|
{
|
|
GitCommand(repo, "rev-parse --verify --quiet origin/release");
|
|
}
|
|
catch
|
|
{
|
|
Log.Info("Repository " + repo + " does not have a release branch.");
|
|
continue;
|
|
}
|
|
|
|
try
|
|
{
|
|
GitCommand(repo, "log -1 --exit-code origin/dev..origin/release");
|
|
}
|
|
catch
|
|
{
|
|
Log.Warn("Unmerged changes in repository " + repo);
|
|
GitCommand(repo, "log origin/dev..origin/release");
|
|
throw;
|
|
}
|
|
}
|
|
|
|
Log.Info("No conflicts in repos, continuing with creating release branch.");
|
|
|
|
foreach (var repo in GetAllRepos())
|
|
{
|
|
GitCommand(repo, "checkout origin/dev -B release");
|
|
|
|
// Update NuGet.Config
|
|
var nugetConfigPath = Path.Combine(repo, "NuGet.config");
|
|
if (File.Exists(nugetConfigPath))
|
|
{
|
|
var original = File.ReadAllText(nugetConfigPath);
|
|
var modified = original
|
|
.Replace("https://www.myget.org/F/aspnetcidev", "https://www.myget.org/F/aspnetcirelease")
|
|
.Replace("https://www.myget.org/F/azureadwebstacknightly", "https://www.myget.org/F/azureadwebstackrelease");
|
|
|
|
if (!string.Equals(original, modified, StringComparison.Ordinal))
|
|
{
|
|
File.WriteAllText(nugetConfigPath, modified);
|
|
GitCommand(repo, "add NuGet.config");
|
|
GitCommand(repo, "commit -m \"Updating to release NuGet.config.\"");
|
|
}
|
|
}
|
|
|
|
GitCommand(repo, "push origin release:release");
|
|
GitCommand(repo, "checkout origin/dev -B dev");
|
|
GitCommand(repo, "merge release -s ours");
|
|
GitCommand(repo, "push origin dev:dev");
|
|
}
|
|
}
|
|
|
|
#pull-all
|
|
-Parallel.ForEach(GetAllRepos(), CloneOrUpdate);
|
|
|
|
#update-master .pull-all
|
|
-// Merge release branch to master
|
|
-// Pin versions of packages in project.json and updated project.lock.json
|
|
-// More information https://github.com/aspnet/Universe/wiki/%23pin-version-:-Pinning-package-version-for-a-particular-release-in-project.json
|
|
@{
|
|
|
|
var COHERENCE_FEED = Environment.GetEnvironmentVariable("COHERENCE_FEED");
|
|
if (string.IsNullOrEmpty(COHERENCE_FEED))
|
|
{
|
|
throw new Exception("COHERENCE_FEED not specified. Usually this is Packages-NoTimestamp directory of Coherence-Signed.");
|
|
}
|
|
|
|
var KOREBUILD_VERSION = Environment.GetEnvironmentVariable("KOREBUILD_VERSION");
|
|
if (string.IsNullOrEmpty(KOREBUILD_VERSION))
|
|
{
|
|
throw new Exception("KOREBUILD_VERSION environment variable is not specified.");
|
|
}
|
|
|
|
var NUGET_VERSION = Environment.GetEnvironmentVariable("NUGET_VERSION");
|
|
if (string.IsNullOrEmpty(NUGET_VERSION))
|
|
{
|
|
throw new Exception("NUGET_VERSION not specified. This is most recent stable build of NuGet from http://dist.nuget.org/index.html. e.g. v3.2");
|
|
}
|
|
|
|
var excludeReposForJson = new[]
|
|
{
|
|
"Coherence",
|
|
"Coherence-Signed",
|
|
"dnvm",
|
|
"Entropy",
|
|
"Setup",
|
|
"libuv-build",
|
|
};
|
|
|
|
Exec("cmd", "/C dnu restore", @"tools\PinVersion");
|
|
|
|
foreach (var repo in GetAllRepos())
|
|
{
|
|
GitCommand(repo, "checkout origin/release -B master");
|
|
|
|
if (File.Exists(Path.Combine(repo, "NuGet.config")))
|
|
{
|
|
File.Copy(Path.Combine("build-template", "NuGet.master.config"),
|
|
Path.Combine(repo, "NuGet.config"),
|
|
overwrite: true);
|
|
GitCommand(repo, "add NuGet.*");
|
|
GitCommand(repo, "commit -m \"Updating NuGet.config\"");
|
|
}
|
|
}
|
|
|
|
var reposToPin = GetAllRepos().Except(excludeReposForJson);
|
|
|
|
foreach (var repo in reposToPin)
|
|
{
|
|
var repoPath = Path.Combine(Directory.GetCurrentDirectory(), repo);
|
|
|
|
Exec("dnx",
|
|
string.Format(@"run ""{0}"" ""{1}"" ""{2}""",
|
|
Path.Combine(Directory.GetCurrentDirectory(), repo),
|
|
COHERENCE_FEED,
|
|
KOREBUILD_VERSION),
|
|
@"tools\PinVersion");
|
|
|
|
GitCommand(repo, "commit -am \"Updating json files to pin versions and build.cmd to pin KoreBuild and DNX\"");
|
|
}
|
|
|
|
foreach (var repo in GetAllRepos())
|
|
{
|
|
GitCommand(repo, "push origin +master:master");
|
|
}
|
|
|
|
CallTarget("update-prerelease-tags");
|
|
}
|
|
|
|
#update-prerelease-tags
|
|
-// Update tags on each repo to have the latest release tag
|
|
@{
|
|
var PRERELEASETAG = Environment.GetEnvironmentVariable("PRERELEASETAG");
|
|
if (string.IsNullOrEmpty(PRERELEASETAG))
|
|
{
|
|
throw new Exception("PRERELEASETAG tag not defined");
|
|
}
|
|
|
|
var versionFile = "version.txt";
|
|
|
|
foreach (var repo in GetAllRepos())
|
|
{
|
|
GitCommand(repo, "pull --tags");
|
|
|
|
try
|
|
{
|
|
GitCommand(repo, string.Format("describe --tags > ..\\{0}", versionFile));
|
|
}
|
|
catch
|
|
{
|
|
Log.Warn(string.Format("{0} repo not tagged. Skipping....", repo));
|
|
continue;
|
|
}
|
|
|
|
var version = File.ReadAllText(versionFile);
|
|
File.Delete(versionFile);
|
|
|
|
Log.Info(string.Format("Current version on repo {0} is {1}", repo, version));
|
|
|
|
var majorVersion = version.Split(new string[]{"-"}, StringSplitOptions.None)[0];
|
|
|
|
var newVersion = majorVersion + string.Format("-{0}", PRERELEASETAG);
|
|
|
|
Log.Info(string.Format("New version for repo is {0}", newVersion));
|
|
|
|
GitCommand(repo, string.Format("tag -f -a {0} -m \"Tag for new release {0}\"", newVersion));
|
|
|
|
GitCommand(repo, "push origin --tags +" + newVersion);
|
|
}
|
|
}
|
|
|
|
#only-compile .build-all
|
|
-Log.Warn("only-compile target is deprecated. Use build-all");
|
|
|
|
#build-all target='compile'
|
|
@{
|
|
var failed = new Dictionary<string, Exception>();
|
|
var skipped = new List<string>();
|
|
var ciVolatileShare = Environment.GetEnvironmentVariable("CI_VOLATILE_SHARE");
|
|
var nugetExe = Environment.GetEnvironmentVariable("PUSH_NUGET_EXE");
|
|
var universeArtifacts = Path.Combine("artifacts", "build");
|
|
Directory.CreateDirectory(universeArtifacts);
|
|
|
|
foreach(var repo in repositories)
|
|
{
|
|
var blockName = string.Format("Building {0}", repo);
|
|
if (IsTeamCity)
|
|
{
|
|
Log.Info(string.Format("##teamcity[blockOpened name='{0}']", FormatForTeamCity(blockName)));
|
|
}
|
|
try
|
|
{
|
|
Log.Info(blockName);
|
|
|
|
if (SKIP_NO_CREDENTIALS)
|
|
{
|
|
if (!Directory.Exists(repo))
|
|
{
|
|
// The directory was not cloned because the credentials were not specified so don't try to build it
|
|
skipped.Add(repo);
|
|
Log.Warn(string.Format("The repo {0} does not exist locally and it will not be built.", repo));
|
|
continue;
|
|
}
|
|
}
|
|
|
|
if (IsMono)
|
|
{
|
|
Exec(Path.Combine(repo, "build.sh"), buildTarget, repo);
|
|
}
|
|
else
|
|
{
|
|
Exec("build.cmd", buildTarget, repo);
|
|
}
|
|
|
|
var repoArtifacts = Path.Combine(repo, "artifacts", "build");
|
|
if (Directory.Exists(repoArtifacts))
|
|
{
|
|
foreach (var source in Directory.EnumerateFiles(repoArtifacts, "*.nupkg"))
|
|
{
|
|
File.Copy(source, Path.Combine(universeArtifacts, Path.GetFileName(source)), overwrite: true);
|
|
}
|
|
|
|
if (!string.IsNullOrEmpty(ciVolatileShare))
|
|
{
|
|
Log.Info("Publishing packages to " + ciVolatileShare);
|
|
if (string.IsNullOrEmpty(nugetExe))
|
|
{
|
|
Log.Warn("PUSH_NUGET_EXE not specified.");
|
|
}
|
|
else
|
|
{
|
|
NuGetPackagesAdd(repoArtifacts, ciVolatileShare);
|
|
}
|
|
}
|
|
}
|
|
|
|
Log.Info(string.Format("Build {0} succeeded", repo));
|
|
}
|
|
catch(Exception ex)
|
|
{
|
|
Log.Warn(string.Format("Build {0} failed: {1}", repo, ex.Message));
|
|
failed[repo] = ex;
|
|
}
|
|
finally
|
|
{
|
|
if (IsTeamCity)
|
|
{
|
|
Log.Info(string.Format("##teamcity[blockClosed name='{0}']", FormatForTeamCity(blockName)));
|
|
}
|
|
}
|
|
}
|
|
|
|
foreach(var repo in repositories)
|
|
{
|
|
Exception ex;
|
|
if (failed.TryGetValue(repo, out ex))
|
|
{
|
|
Log.Warn(string.Format("Build {0} failed: {1}", repo, ex.Message));
|
|
if (IsTeamCity)
|
|
{
|
|
Log.Warn(string.Format("##teamcity[message text='{0}' errorDetails='{1}' status='ERROR']",
|
|
FormatForTeamCity(ex.Message),
|
|
FormatForTeamCity(ex.StackTrace)));
|
|
}
|
|
|
|
}
|
|
else if (skipped.Contains(repo))
|
|
{
|
|
Log.Warn(string.Format("Build {0} skipped", repo));
|
|
}
|
|
else
|
|
{
|
|
Log.Info(string.Format("Build {0} succeeded", repo));
|
|
}
|
|
}
|
|
|
|
if (failed.Any())
|
|
{
|
|
throw new Exception("One or more repos failed to build");
|
|
}
|
|
}
|
|
|
|
#only-install target='install'
|
|
@{
|
|
foreach(var repo in repositories)
|
|
{
|
|
if (IsMono)
|
|
{
|
|
Exec(Path.Combine(repo, "build.sh"), "install", repo);
|
|
}
|
|
else
|
|
{
|
|
Exec("build.cmd", "install", repo);
|
|
}
|
|
}
|
|
}
|
|
|
|
#run-snapshot-manager
|
|
@{
|
|
Exec(@".nuget\nuget.exe", @"restore -out packages tools\TCDependencyManager\packages.config", "");
|
|
var programFiles = Environment.GetFolderPath(Environment.SpecialFolder.ProgramFilesX86);
|
|
var msbuildPath = Path.Combine(programFiles, "MSBuild", "12.0", "Bin", "MsBuild.exe");
|
|
Exec(msbuildPath, "TCDependencyManager.csproj", @"tools\TCDependencyManager");
|
|
Exec(@"tools\TCDependencyManager\bin\Debug\TCDependencyManager.exe", "", "");
|
|
}
|
|
|
|
#git-status description='Show status of repos known by Universe'
|
|
@{
|
|
foreach(var repo in repositories)
|
|
{
|
|
GitStatus(repo);
|
|
}
|
|
}
|
|
|
|
#git-clean description='REMOVE ALL CHANGES to the working directory'
|
|
@{
|
|
Console.WriteLine("This runs `git clean -xfd` in all non-Universe repos.");
|
|
Console.WriteLine("This should REMOVE ALL CHANGES to the working directory.");
|
|
Console.Write("***** Are you sure? ***** (Y or anything else)? ");
|
|
if (Console.ReadLine() != "Y")
|
|
{
|
|
throw new Exception("git-clean cancelled");
|
|
}
|
|
foreach(var repo in repositories)
|
|
{
|
|
GitClean(repo);
|
|
}
|
|
}
|
|
|
|
macro name='ExecClr' program='string' commandline='string'
|
|
exec-clr
|
|
|
|
macro name='Git' gitCommand='string' gitFolder='string'
|
|
git
|
|
|
|
macro name='GitPull' gitUri='string' gitBranch='string' gitFolder='string'
|
|
git-pull
|
|
|
|
macro name='GitClone' gitUri='string' gitBranch='string'
|
|
git-clone
|
|
|
|
macro name='GitConfig' gitOptionName='string' gitOptionValue='string' gitFolder='string'
|
|
git-config
|
|
|
|
macro name='GitStatus' gitFolder='string'
|
|
git gitCommand='status'
|
|
|
|
macro name='GitClean' gitFolder='string'
|
|
git gitCommand='clean -xdf'
|
|
|
|
macro name='GitCommand' gitFolder='string' gitCommand='string'
|
|
git
|
|
|
|
macro name='Exec' program='string' commandline='string' workingdir='string'
|
|
exec
|
|
|
|
macro name='NuGetPackagesAdd' sourcePackagesDir='string' targetPackagesDir='string'
|
|
nuget-packages-add
|
|
|
|
macro name="CopyFolder" sourceDir='string' outputDir='string' overwrite='bool'
|
|
copy
|
|
|
|
functions
|
|
@{
|
|
static IDictionary<string, string> GetCommitsToSync()
|
|
{
|
|
var commits = new Dictionary<string, string>();
|
|
|
|
var commitsFile = Environment.GetEnvironmentVariable("UNIVERSE_COMMITS_FILE");
|
|
if (string.IsNullOrEmpty(commitsFile))
|
|
{
|
|
return commits;
|
|
}
|
|
|
|
Console.WriteLine("Using commits file: " + commitsFile);
|
|
var lines = File.ReadAllLines(commitsFile);
|
|
foreach(var line in lines)
|
|
{
|
|
var parts = line.Split(new string[] {":"}, StringSplitOptions.RemoveEmptyEntries);
|
|
commits.Add(parts[0], parts[1]);
|
|
}
|
|
|
|
return commits;
|
|
}
|
|
|
|
static bool UseHttps(string directory)
|
|
{
|
|
var filename = Path.Combine(directory, ".git", "config");
|
|
if (!File.Exists(filename))
|
|
{
|
|
Console.WriteLine("Unable to find '{0}' file", filename);
|
|
return false;
|
|
}
|
|
|
|
var url = ReadOriginUrl(filename);
|
|
return IsHttpsUrl(url);
|
|
}
|
|
|
|
// Perform equivalent of `git config remote.origin.url` but directly
|
|
// read config file to get value.
|
|
static string ReadOriginUrl(string filename)
|
|
{
|
|
// Subsection portion of configuration name is case-sensitive; rest
|
|
// of name is case-insensitive.
|
|
var beginOriginSection = new Regex(@"^\[(?i:remote) ""origin""\]\s*$");
|
|
var beginSection = new Regex(@"^\[");
|
|
var urlLine = new Regex(@"^\s+url = (\S+)\s*$", RegexOptions.IgnoreCase);
|
|
|
|
var inRemoteOriginSection = false;
|
|
foreach (var line in File.ReadAllLines(filename))
|
|
{
|
|
if (beginOriginSection.IsMatch(line))
|
|
{
|
|
inRemoteOriginSection = true;
|
|
continue;
|
|
}
|
|
|
|
if (inRemoteOriginSection)
|
|
{
|
|
if (beginSection.IsMatch(line))
|
|
{
|
|
// Read through the section without finding URL line.
|
|
break;
|
|
}
|
|
|
|
var match = urlLine.Match(line);
|
|
if (match.Success && match.Groups.Count == 2 && match.Groups[1].Success)
|
|
{
|
|
return match.Groups[1].Value;
|
|
}
|
|
}
|
|
}
|
|
|
|
Console.WriteLine("Unable to parse '{0}' file", filename);
|
|
return null;
|
|
}
|
|
|
|
static bool IsHttpsUrl(string url)
|
|
{
|
|
if (string.IsNullOrEmpty(url))
|
|
{
|
|
return false;
|
|
}
|
|
|
|
return url.StartsWith("https://", System.StringComparison.OrdinalIgnoreCase);
|
|
}
|
|
|
|
static bool IsAccessible(string key)
|
|
{
|
|
var req = WebRequest.CreateHttp("https://github.com/aspnet/" + key);
|
|
req.Method = "HEAD";
|
|
try
|
|
{
|
|
using (req.GetResponse())
|
|
{
|
|
// intentionally empty
|
|
}
|
|
}
|
|
catch (WebException ex)
|
|
{
|
|
if (ex.Response != null &&
|
|
((HttpWebResponse)ex.Response).StatusCode == HttpStatusCode.NotFound)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
// Ignore any other exception. They should surface as part of git clone with well-known messages.
|
|
}
|
|
return true;
|
|
}
|
|
|
|
IEnumerable<string> GetAllRepos()
|
|
{
|
|
var nonDefaultRepos = new[]
|
|
{
|
|
"DNX",
|
|
"Coherence",
|
|
"Coherence-Signed",
|
|
"dnvm",
|
|
"Setup",
|
|
};
|
|
|
|
return Enumerable.Concat(nonDefaultRepos, repositories);
|
|
}
|
|
|
|
void RemoveSrcFolder(string repo)
|
|
{
|
|
var srcDir = Path.Combine(repo, "src");
|
|
if (Directory.Exists(srcDir))
|
|
{
|
|
Console.WriteLine("Deleting " + srcDir);
|
|
Directory.Delete(srcDir, recursive: true);
|
|
}
|
|
}
|
|
|
|
void CloneOrUpdate(string repo)
|
|
{
|
|
var repoUrl = gitHubUriPrefix + repo + ".git";
|
|
|
|
if(Directory.Exists(repo))
|
|
{
|
|
GitPull(repoUrl, BUILD_BRANCH, repo);
|
|
}
|
|
else
|
|
{
|
|
if (useHttps &&
|
|
!IsAccessible(repo))
|
|
{
|
|
if (SKIP_NO_CREDENTIALS)
|
|
{
|
|
// This is used on the XPlat CI. Must return because otherwise git will wait for user
|
|
// input and hang the build
|
|
Log.Warn(string.Format("The repo at '{0}' is not accesible and it will not be cloned.", repoUrl));
|
|
return;
|
|
}
|
|
|
|
Log.Warn(string.Format("The repo at '{0}' is not accessible. If you do not have access to this repository, skip the git prompt" +
|
|
" for credentials to skip cloning this repository. To avoid this prompt, re-clone the Universe repository over ssh.",
|
|
repoUrl));
|
|
}
|
|
|
|
try
|
|
{
|
|
GitClone(repoUrl, BUILD_BRANCH);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Log.Warn(string.Format("Unable to clone repository at '{0}': {1}", repoUrl, ex.Message));
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
|
|
string FormatForTeamCity(string input)
|
|
{
|
|
return input.Replace("|", "||")
|
|
.Replace("'", "|'")
|
|
.Replace("\r", "|r")
|
|
.Replace("\n", "|n")
|
|
.Replace("]", "|]");
|
|
}
|
|
|
|
// Create a search replace expression based on branch, prerelease tag and buildNumber
|
|
string BuildVersionExpression(string branch, string preRelease = "", string buildNumber = "")
|
|
{
|
|
var stableBranches = new[] {"master"};
|
|
var builder = new StringBuilder();
|
|
|
|
// Add pre release version tag
|
|
if(!String.IsNullOrEmpty(preRelease))
|
|
{
|
|
builder.Append("-");
|
|
builder.Append(preRelease);
|
|
}
|
|
|
|
// If buildnumber is provided, append.
|
|
// This for CORE CLR packages
|
|
if(!String.IsNullOrEmpty(buildNumber))
|
|
{
|
|
builder.Append("-");
|
|
builder.Append(buildNumber);
|
|
}
|
|
else if(!stableBranches.Contains(branch))
|
|
{
|
|
builder.Append("-*");
|
|
}
|
|
|
|
return builder.ToString();
|
|
}
|
|
|
|
static IEnumerable<string> GetRepositoriesToBuild()
|
|
{
|
|
IEnumerable<string> reposToBuild = new HashSet<string>(StringComparer.OrdinalIgnoreCase) {
|
|
// The repos list is topologically sorted based on build order
|
|
"PlatformAbstractions",
|
|
"Common",
|
|
"JsonPatch",
|
|
"FileSystem",
|
|
"Configuration",
|
|
"DependencyInjection",
|
|
"EventNotification",
|
|
"Options",
|
|
"Logging",
|
|
"dotnet-watch",
|
|
"HtmlAbstractions",
|
|
"UserSecrets",
|
|
"DataProtection",
|
|
"HttpAbstractions",
|
|
"Testing",
|
|
"Microsoft.Data.Sqlite",
|
|
"Caching",
|
|
"Razor",
|
|
"RazorTooling",
|
|
"Hosting",
|
|
"EntityFramework",
|
|
"WebListener",
|
|
"KestrelHttpServer",
|
|
"IISIntegration",
|
|
"ServerTests",
|
|
"Session",
|
|
"CORS",
|
|
"Routing",
|
|
"StaticFiles",
|
|
"Diagnostics",
|
|
"Security",
|
|
"Antiforgery",
|
|
"WebSockets",
|
|
"Localization",
|
|
"BasicMiddleware",
|
|
"Proxy",
|
|
"Mvc",
|
|
"Identity",
|
|
"Scaffolding",
|
|
"SignalR-Server",
|
|
"SignalR-SQLServer",
|
|
"SignalR-Redis",
|
|
"SignalR-ServiceBus",
|
|
"BrowserLink",
|
|
"Entropy",
|
|
"MusicStore",
|
|
};
|
|
|
|
var repositoryInclude = Environment.GetEnvironmentVariable("KOREBUILD_REPOSITORY_INCLUDE");
|
|
if (!string.IsNullOrEmpty(repositoryInclude))
|
|
{
|
|
reposToBuild = new HashSet<string>(
|
|
repositoryInclude.Split(new string[] {","}, StringSplitOptions.RemoveEmptyEntries),
|
|
StringComparer.OrdinalIgnoreCase);
|
|
}
|
|
|
|
var repositoryExclude = Environment.GetEnvironmentVariable("KOREBUILD_REPOSITORY_EXCLUDE");
|
|
if (!string.IsNullOrEmpty(repositoryExclude))
|
|
{
|
|
var reposToExclude = repositoryExclude.Split(new string[] {","}, StringSplitOptions.RemoveEmptyEntries);
|
|
reposToBuild = reposToBuild.Except(reposToExclude);
|
|
}
|
|
|
|
return reposToBuild.ToList();
|
|
}
|
|
|
|
static IEnumerable<string> GetNoSrcDeleteRepositories()
|
|
{
|
|
var repositoryExclude = Environment.GetEnvironmentVariable("KOREBUILD_NOSRC_EXCLUDE");
|
|
if (string.IsNullOrEmpty(repositoryExclude))
|
|
{
|
|
return new string[0];
|
|
}
|
|
|
|
return repositoryExclude.Split(new string[] {","}, StringSplitOptions.RemoveEmptyEntries);
|
|
}
|
|
|
|
static IList<IGrouping<int, string>> GetBuildGraph()
|
|
{
|
|
var repositoryLookup = new List<RepositoryInfo>();
|
|
foreach (var repo in GetRepositoriesToBuild())
|
|
{
|
|
var info = new RepositoryInfo { Name = repo };
|
|
var srcDir = Path.Combine(repo, "src");
|
|
|
|
if (Directory.Exists(srcDir))
|
|
{
|
|
foreach (var directory in Directory.EnumerateDirectories(srcDir))
|
|
{
|
|
info.RepositoryNames.Add(Path.GetFileName(directory));
|
|
var projectJson = Path.Combine(directory, "project.json");
|
|
if (File.Exists(projectJson))
|
|
{
|
|
GetDependencies(projectJson, info.DependencyNames);
|
|
}
|
|
}
|
|
}
|
|
|
|
var otherDirs = new[] { "test", "samples" };
|
|
for (var i = 0; i < otherDirs.Length; i++)
|
|
{
|
|
var otherDir = Path.Combine(repo, otherDirs[i]);
|
|
if (Directory.Exists(otherDir))
|
|
{
|
|
foreach (var directory in Directory.EnumerateDirectories(otherDir))
|
|
{
|
|
var projectJson = Path.Combine(directory, "project.json");
|
|
if (File.Exists(projectJson))
|
|
{
|
|
GetDependencies(projectJson, info.DependencyNames);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
info.DependencyNames.ExceptWith(info.RepositoryNames);
|
|
repositoryLookup.Add(info);
|
|
}
|
|
|
|
foreach (var info in repositoryLookup)
|
|
{
|
|
foreach (var item in info.DependencyNames)
|
|
{
|
|
var dependency = repositoryLookup.Find(r => r.RepositoryNames.Contains(item));
|
|
if (dependency != null)
|
|
{
|
|
info.Dependencies.Add(dependency);
|
|
}
|
|
}
|
|
}
|
|
|
|
return repositoryLookup.GroupBy(r => r.Order, r => r.Name).OrderBy(r => r.Key).ToArray();
|
|
}
|
|
|
|
static void GetDependencies(string projectJsonPath, HashSet<string> dependencies)
|
|
{
|
|
var project = (JsonObject)Json.Deserialize(File.ReadAllText(projectJsonPath));
|
|
var dependenciesNode = project.ValueAsJsonObject("dependencies");
|
|
AddKeys(dependencies, dependenciesNode);
|
|
|
|
var frameworkNodes = project.ValueAsJsonObject("frameworks");
|
|
if (frameworkNodes != null)
|
|
{
|
|
foreach (var framework in frameworkNodes.Keys)
|
|
{
|
|
dependenciesNode = frameworkNodes.ValueAsJsonObject(framework).ValueAsJsonObject("dependencies");
|
|
AddKeys(dependencies, dependenciesNode);
|
|
}
|
|
}
|
|
|
|
AddKeys(dependencies, project.ValueAsJsonObject("tools"));
|
|
}
|
|
|
|
static void AddKeys(HashSet<string> target, JsonObject source)
|
|
{
|
|
if (source != null)
|
|
{
|
|
foreach (var key in source.Keys)
|
|
{
|
|
target.Add(key);
|
|
}
|
|
}
|
|
}
|
|
|
|
string CreateBuildWithFlowId(string repo)
|
|
{
|
|
string output;
|
|
if (IsLinux)
|
|
{
|
|
output = Path.Combine(repo, "build-with-flow-id.sh");
|
|
File.WriteAllText(
|
|
output,
|
|
string.Format("export KOREBUILD_FLOWID=KOREBUILD_{0}\nbuild.sh \"$@\"", repo));
|
|
Exec("chmod", string.Format("--reference={0} {1}", Path.Combine(repo, "build.sh"), output), "");
|
|
}
|
|
else
|
|
{
|
|
output = Path.Combine(repo, "build-with-flow-id.cmd");
|
|
File.WriteAllText(
|
|
output,
|
|
string.Format("set KOREBUILD_FLOWID=KOREBUILD_{0}& call build.cmd %*", repo));
|
|
}
|
|
|
|
return Path.GetFullPath(output);
|
|
}
|
|
|
|
private class RepositoryInfo
|
|
{
|
|
public string Name;
|
|
|
|
public HashSet<string> RepositoryNames = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
|
|
|
public HashSet<string> DependencyNames = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
|
|
|
public HashSet<RepositoryInfo> Dependencies = new HashSet<RepositoryInfo>();
|
|
|
|
public int Order
|
|
{
|
|
get
|
|
{
|
|
if (Dependencies.Count > 0)
|
|
{
|
|
return 1 + Dependencies.Max(d => d.Order);
|
|
}
|
|
|
|
return 1;
|
|
}
|
|
}
|
|
|
|
public override string ToString()
|
|
{
|
|
return Name;
|
|
}
|
|
}
|
|
}
|