1122 lines
40 KiB
Plaintext
1122 lines
40 KiB
Plaintext
|
|
var VERSION='0.2.1'
|
|
|
|
use-ci-loggers
|
|
use-release-management
|
|
use namespace='System'
|
|
use namespace='System.IO'
|
|
use namespace='System.Collections.Concurrent'
|
|
use namespace='System.Collections.Generic'
|
|
use namespace='System.Net'
|
|
use namespace='System.Linq'
|
|
use namespace='System.Text'
|
|
use namespace='System.Text.RegularExpressions'
|
|
use namespace='System.Threading.Tasks'
|
|
use import="BuildEnv"
|
|
use import="EnvironmentParameters"
|
|
use import="Files"
|
|
use import="Json"
|
|
|
|
functions
|
|
@{
|
|
const string DefaultBuildBranch = "dev";
|
|
const string DefaultCoherenceCacheDir = ".coherence-cache";
|
|
const string UniverseCommitsFileName = "commits-universe";
|
|
const string CoherenceCacheVersionFileName = "coherence-version";
|
|
|
|
private static bool Quiet { get; set; }
|
|
|
|
static string baseDir = Directory.GetCurrentDirectory();
|
|
static string targetDir = Path.Combine(baseDir, "artifacts", "build");
|
|
|
|
static string buildBranch = GetEnvironmentParameter("BUILD_BRANCH") ?? DefaultBuildBranch;
|
|
static string coherencePath = GetEnvironmentParameter("COHERENCE_PATH");
|
|
static string dropsShare = GetEnvironmentParameter("ASPNETCI_DROPS_SHARE", DefaultDropsShare);
|
|
static string kBuildVersion = GetEnvironmentParameter("DNX_BUILD_VERSION");
|
|
static string ciVolatileShare = GetEnvironmentParameter("CI_VOLATILE_SHARE");
|
|
static string koreBuildTargets = GetEnvironmentParameter("KOREBUILD_BUILD_TARGETS");
|
|
static string nugetVersion = GetEnvironmentParameter("NUGET_VERSION");
|
|
static string nugetExe = GetEnvironmentParameter("PUSH_NUGET_EXE");
|
|
static string universeCommitsFile = GetEnvironmentParameter("UNIVERSE_COMMITS_FILE");
|
|
static bool skipNoCredentials = GetEnvironmentParameter("UNIVERSE_SKIP_NO_CREDENTIALS", value => value == "1");
|
|
static string repositoryInclude = GetEnvironmentParameter("KOREBUILD_REPOSITORY_INCLUDE");
|
|
static string repositoryExclude = GetEnvironmentParameter("KOREBUILD_REPOSITORY_EXCLUDE");
|
|
|
|
static string gateBranch = GetEnvironmentParameter("KOREBUILD_GATE_BRANCH");
|
|
private static bool UseGateBranch = !string.IsNullOrEmpty(gateBranch);
|
|
|
|
// Doesn't build on Mono since their contracts don't match
|
|
string[] repositories = GetRepositoriesToBuild().ToArray();
|
|
string[] noSrcRepositoryExclude = GetNoSrcDeleteRepositories().ToArray();
|
|
|
|
static bool useHttps = UseHttps(baseDir);
|
|
static string gitHubUriPrefix = useHttps ? "https://github.com/aspnet/" : "git@github.com:aspnet/";
|
|
}
|
|
|
|
var buildTarget = "compile"
|
|
|
|
@{
|
|
if (!string.IsNullOrEmpty(kBuildVersion))
|
|
{
|
|
VERSION += "-" + kBuildVersion;
|
|
}
|
|
else
|
|
{
|
|
VERSION += "-" + BuildNumber;
|
|
}
|
|
}
|
|
|
|
#default .compile
|
|
|
|
#--quiet
|
|
@{
|
|
Quiet = true;
|
|
}
|
|
|
|
#pull
|
|
#compile .pull
|
|
#install .pull
|
|
|
|
#pack
|
|
directory create='${targetDir}'
|
|
|
|
#pack-install .pack
|
|
nuget-local-publish sourcePackagesDir='${targetDir}'
|
|
|
|
#git-pull target='pull'
|
|
@{
|
|
Parallel.ForEach(repositories, repo =>
|
|
{
|
|
CloneOrUpdate(repo);
|
|
});
|
|
}
|
|
|
|
#update
|
|
@{
|
|
Parallel.ForEach(repositories, repo =>
|
|
{
|
|
Git("pull --ff-only", repo);
|
|
});
|
|
}
|
|
|
|
#sync-commits
|
|
@{
|
|
if (string.IsNullOrEmpty(universeCommitsFile))
|
|
{
|
|
throw new Exception("UNIVERSE_COMMITS_FILE not specified.");
|
|
}
|
|
|
|
var commitsToSync = GetCommitsToSync(universeCommitsFile);
|
|
|
|
Parallel.ForEach(repositories, repo =>
|
|
{
|
|
if (commitsToSync.ContainsKey(repo) && Directory.Exists(repo))
|
|
{
|
|
var syncHash = commitsToSync[repo];
|
|
Console.WriteLine(string.Format("Syncing {0} to {1}", repo, syncHash));
|
|
Git("reset --hard " + syncHash, repo);
|
|
}
|
|
});
|
|
}
|
|
|
|
#verify-all .pull .change-default-build-target-to-verify .build-all
|
|
|
|
#ci-test .pull .use-local-coherence .sync-commits .remove-src-folders .change-default-build-target-to-verify .build-all
|
|
|
|
#ci-pull
|
|
@{
|
|
var threads = int.Parse(Environment.GetEnvironmentVariable("UNIVERSE_THREADS") ?? "4");
|
|
|
|
var blockLogger = Log as IBlockLogger;
|
|
if (blockLogger != null)
|
|
{
|
|
blockLogger.StartBlock("Cloning repos");
|
|
}
|
|
|
|
Parallel.ForEach(repositories, new ParallelOptions { MaxDegreeOfParallelism = threads }, repo =>
|
|
{
|
|
var useBuildBranch = true;
|
|
if (Directory.Exists(repo))
|
|
{
|
|
Directory.Delete(repo, recursive: true);
|
|
}
|
|
|
|
if (UseGateBranch)
|
|
{
|
|
try
|
|
{
|
|
GitCommand("", string.Format("clone --quiet --depth 1 --branch {0} git@github.com:aspnet/{1}.git", gateBranch, repo));
|
|
useBuildBranch = false;
|
|
Log.Warn(string.Format(
|
|
"{0} has a branch '{1}' which is overriding use of {2}. {1} must be deleted before {2} will be used.",
|
|
repo,
|
|
gateBranch,
|
|
buildBranch));
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Log.Info(string.Format("{0} doesn't exist, falling back to {1}.", gateBranch, buildBranch));
|
|
}
|
|
}
|
|
|
|
if (useBuildBranch)
|
|
{
|
|
GitCommand("", string.Format("clone --quiet --depth 1 --branch {0} git@github.com:aspnet/{1}.git", buildBranch, repo));
|
|
}
|
|
});
|
|
|
|
if (blockLogger != null)
|
|
{
|
|
blockLogger.EndBlock("Cloning repos");
|
|
}
|
|
}
|
|
|
|
#show-build-graph
|
|
@{
|
|
var batchedRepos = GetBuildGraph();
|
|
Log.Info("Building repositories in batches: ");
|
|
foreach (var repos in batchedRepos)
|
|
{
|
|
Log.Info(string.Format("{0} - {1}", repos.Key, string.Join(", ", repos)));
|
|
}
|
|
}
|
|
|
|
#ci-build
|
|
@{
|
|
var nugetExe = Path.Combine(Directory.GetCurrentDirectory(), ".build", "nuget.exe");
|
|
var universeArtifacts = Path.Combine(Directory.GetCurrentDirectory(), "artifacts");
|
|
var universeBuild = Path.Combine(universeArtifacts, "build");
|
|
var packagesPublishDir = Path.Combine(Directory.GetCurrentDirectory(), ".nuget", "publishDir");
|
|
// Historically we've always produced coherent builds using this target. Unless specified otherwise,
|
|
// we'll continue to produce coherent builds here.
|
|
var createCoherentBuild = Environment.GetEnvironmentVariable("CREATE_COHERENT_BUILD") != "false";
|
|
|
|
// Snapshot the .build folder
|
|
if (!IsLinux)
|
|
{
|
|
Exec("cmd", "/C xcopy /S/Q/I/Y .build " + Path.Combine(universeArtifacts, ".build"), "");
|
|
}
|
|
else
|
|
{
|
|
CopyFolder(".build", Path.Combine(universeArtifacts, ".build"), true);
|
|
}
|
|
|
|
var blockLogger = Log as IBlockLogger;
|
|
var commits = new ConcurrentDictionary<string, string>();
|
|
var threads = int.Parse(Environment.GetEnvironmentVariable("UNIVERSE_THREADS") ?? "4");
|
|
buildTarget = koreBuildTargets ?? "--quiet compile nuget-verify nuget-install";
|
|
|
|
var batchedRepos = GetBuildGraph();
|
|
Log.Info("Building repositories in batches: ");
|
|
foreach (var repos in batchedRepos)
|
|
{
|
|
Log.Info(string.Format("{0} - {1}", repos.Key, string.Join(", ", repos)));
|
|
}
|
|
|
|
Directory.CreateDirectory(universeBuild);
|
|
if (createCoherentBuild)
|
|
{
|
|
if (!IsLinux)
|
|
{
|
|
// Publish to a directory and use that as a package source for later builds. This doesn't work in Linux due to
|
|
// https://github.com/NuGet/Home/issues/2383
|
|
|
|
Directory.CreateDirectory(packagesPublishDir);
|
|
Environment.SetEnvironmentVariable("NUGET_VOLATILE_FEED_ARTIFACTS", packagesPublishDir);
|
|
Environment.SetEnvironmentVariable("PACKAGES_PUBLISH_DIR", packagesPublishDir);
|
|
}
|
|
else
|
|
{
|
|
Environment.SetEnvironmentVariable("NUGET_VOLATILE_FEED_ARTIFACTS", universeBuild);
|
|
}
|
|
}
|
|
|
|
if (Environment.GetEnvironmentVariable("UNIVERSE_PUSH_TO_VOLATILE") == "true")
|
|
{
|
|
Environment.SetEnvironmentVariable("NUGET_PUBLISH_FEED", "https://dotnet.myget.org/F/aspnetcore-volatile-dev/api/v2/package");
|
|
}
|
|
|
|
var universeToolsVersion = File.ReadAllText(Path.Combine(Directory.GetCurrentDirectory(), "ToolsVersion.txt")).Trim();
|
|
|
|
foreach (var batch in batchedRepos)
|
|
{
|
|
Parallel.ForEach(batch.ToArray(), new ParallelOptions { MaxDegreeOfParallelism = threads }, repo =>
|
|
{
|
|
var blockName = string.Format("Building {0}", repo);
|
|
|
|
if (blockLogger != null)
|
|
{
|
|
blockLogger.StartBlock(blockName);
|
|
}
|
|
|
|
// begin workaround
|
|
// as we transition to VS 2017, some repos will build with project.json and others with csproj
|
|
// the 'csproj' version of KoreBuild is the 'feature/msbuild' branch of that repo. project.json
|
|
// support is on 'dev'. This workaround avoids installing the version of KoreBuild used by
|
|
// individual repos unless it is already upgraded to the version currently used by Universe.
|
|
var repoToolsVersion = universeToolsVersion;
|
|
var buildPs1 = Path.Combine(repo, "build.ps1");
|
|
if (File.Exists(buildPs1))
|
|
{
|
|
foreach (var line in File.ReadAllLines(buildPs1))
|
|
{
|
|
var match = Regex.Match(line, @".*https://github.com/aspnet/KoreBuild/archive/(.+)\.zip.*");
|
|
if (match != null && match.Success)
|
|
{
|
|
repoToolsVersion = match.Groups[1].Value;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
Log.Info(repo + " KoreBuild version = " + repoToolsVersion + ", Universe = " + universeToolsVersion);
|
|
// end workaround
|
|
|
|
if (repoToolsVersion == universeToolsVersion)
|
|
{
|
|
if (!IsLinux)
|
|
{
|
|
Exec("cmd", "/C xcopy /S/Q/I/Y .build " + Path.Combine(repo, ".build"), "");
|
|
}
|
|
else
|
|
{
|
|
CopyFolder(".build", Path.Combine(repo, ".build"), true);
|
|
}
|
|
}
|
|
|
|
try
|
|
{
|
|
Exec(CreateBuildWithFlowId(repo), buildTarget, repo);
|
|
var repoArtifacts = Path.Combine(repo, "artifacts");
|
|
var repoBuild = Path.Combine(repoArtifacts, "build");
|
|
if (Directory.Exists(repoBuild))
|
|
{
|
|
foreach (var source in Directory.EnumerateFiles(repoBuild, "*.nupkg"))
|
|
{
|
|
File.Copy(source, Path.Combine(universeBuild, Path.GetFileName(source)), overwrite: true);
|
|
}
|
|
|
|
if (Environment.GetEnvironmentVariable("KOREBUILD_ADD_ASSEMBLY_INFO") == "1")
|
|
{
|
|
var commitFile = Path.Combine(repoArtifacts, "commit");
|
|
if (File.Exists(commitFile))
|
|
{
|
|
commits.TryAdd(repo, File.ReadAllLines(commitFile)[0]);
|
|
}
|
|
}
|
|
|
|
if (!string.IsNullOrEmpty(ciVolatileShare))
|
|
{
|
|
Log.Info("Publishing packages to " + ciVolatileShare);
|
|
NuGetPackagesAdd(repoBuild, ciVolatileShare);
|
|
}
|
|
}
|
|
|
|
Log.Info(string.Format("Build {0} succeeded", repo));
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Log.Error("Building '" + repo + "' failed: " + ex);
|
|
throw;
|
|
}
|
|
finally
|
|
{
|
|
if (blockLogger != null)
|
|
{
|
|
blockLogger.EndBlock(blockName);
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
var commitsAsString = string.Join("\n", commits.Select(c => c.Key + ":" + c.Value));
|
|
File.WriteAllText(Path.Combine(universeArtifacts, "commits"), commitsAsString);
|
|
}
|
|
|
|
#remove-src-folders
|
|
@{
|
|
foreach (var repo in GetRepositoriesToBuild())
|
|
{
|
|
if (!noSrcRepositoryExclude.Contains(repo))
|
|
{
|
|
RemoveSrcFolder(repo);
|
|
}
|
|
else
|
|
{
|
|
Console.WriteLine("Keeping the sources for " + repo + " because it's explicitly excluded");
|
|
}
|
|
}
|
|
}
|
|
|
|
#use-local-coherence description='Sets up environment variables to use Coherence at COHERENCE_PATH'
|
|
@{
|
|
if (string.IsNullOrEmpty(coherencePath))
|
|
{
|
|
throw new Exception("COHERENCE_PATH not specified.");
|
|
}
|
|
|
|
if (string.IsNullOrEmpty(universeCommitsFile))
|
|
{
|
|
universeCommitsFile = Path.Combine(coherencePath, UniverseCommitsFileName);
|
|
if (!File.Exists(universeCommitsFile))
|
|
{
|
|
throw new Exception("Universe commits file could not be found at " + universeCommitsFile);
|
|
}
|
|
}
|
|
|
|
var coherencePackages = Path.Combine(coherencePath, "packages-expanded");
|
|
if (!Directory.Exists(coherencePackages))
|
|
{
|
|
coherencePackages = Path.Combine(coherencePath, "build");
|
|
}
|
|
if (!Directory.Exists(coherencePackages))
|
|
{
|
|
throw new Exception("Packages directory could not be located under " + coherencePath);
|
|
}
|
|
|
|
Environment.SetEnvironmentVariable("NUGET_VOLATILE_FEED_AspNetCore", coherencePackages);
|
|
}
|
|
|
|
#change-default-build-target-to-verify
|
|
- buildTarget = "verify";
|
|
|
|
#change-default-build-target-for-coherence-build
|
|
- buildTarget = koreBuildTargets ?? "compile nuget-install";
|
|
|
|
#init
|
|
@{
|
|
var templatePath = Path.Combine(baseDir, "build-template");
|
|
var templateFiles = Files.Include(templatePath + Path.DirectorySeparatorChar + "*.*").Select(Path.GetFileName).ToList();
|
|
|
|
foreach(var repo in repositories)
|
|
{
|
|
foreach (string fileName in templateFiles)
|
|
{
|
|
var targetFile = Path.Combine(Directory.GetCurrentDirectory(), repo, fileName);
|
|
var sourceFile = Path.Combine(Directory.GetCurrentDirectory(), templatePath, fileName);
|
|
|
|
// Don't update the makefile
|
|
if (fileName.Equals("makefile.shade", StringComparison.OrdinalIgnoreCase) && File.Exists(targetFile))
|
|
{
|
|
continue;
|
|
}
|
|
|
|
// Don't update extra configuration files
|
|
if (fileName.Equals("NuGet.master.config", StringComparison.OrdinalIgnoreCase) ||
|
|
fileName.Equals("NuGet.release.config", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (!File.Exists(targetFile) ||
|
|
(File.ReadAllText(sourceFile) != File.ReadAllText(targetFile)))
|
|
{
|
|
Log.Info("Updating " + fileName + " to " + repo);
|
|
File.Copy(sourceFile, targetFile, true);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
|
|
#pull-all
|
|
-Parallel.ForEach(GetAllRepos(), CloneOrUpdate);
|
|
|
|
#only-compile .build-all
|
|
-Log.Warn("only-compile target is deprecated. Use build-all");
|
|
|
|
#build-all target='compile'
|
|
@{
|
|
var failed = new Dictionary<string, Exception>();
|
|
var skipped = new List<string>();
|
|
var universeArtifacts = Path.Combine("artifacts", "build");
|
|
Directory.CreateDirectory(universeArtifacts);
|
|
|
|
foreach(var repo in repositories)
|
|
{
|
|
var blockName = string.Format("Building {0}", repo);
|
|
if (IsTeamCity)
|
|
{
|
|
Log.Info(string.Format("##teamcity[blockOpened name='{0}']", FormatForTeamCity(blockName)));
|
|
}
|
|
try
|
|
{
|
|
Log.Info(blockName);
|
|
|
|
if (skipNoCredentials)
|
|
{
|
|
if (!Directory.Exists(repo))
|
|
{
|
|
// The directory was not cloned because the credentials were not specified so don't try to build it
|
|
skipped.Add(repo);
|
|
Log.Warn(string.Format("The repo {0} does not exist locally and it will not be built.", repo));
|
|
continue;
|
|
}
|
|
}
|
|
|
|
if (IsMono)
|
|
{
|
|
Exec(Path.Combine(repo, "build.sh"), buildTarget, repo);
|
|
}
|
|
else
|
|
{
|
|
Exec("build.cmd", buildTarget, repo);
|
|
}
|
|
|
|
var repoArtifacts = Path.Combine(repo, "artifacts", "build");
|
|
if (Directory.Exists(repoArtifacts))
|
|
{
|
|
foreach (var source in Directory.EnumerateFiles(repoArtifacts, "*.nupkg"))
|
|
{
|
|
File.Copy(source, Path.Combine(universeArtifacts, Path.GetFileName(source)), overwrite: true);
|
|
}
|
|
|
|
if (!string.IsNullOrEmpty(ciVolatileShare))
|
|
{
|
|
Log.Info("Publishing packages to " + ciVolatileShare);
|
|
if (string.IsNullOrEmpty(nugetExe))
|
|
{
|
|
Log.Warn("PUSH_NUGET_EXE not specified.");
|
|
}
|
|
else
|
|
{
|
|
NuGetPackagesAdd(repoArtifacts, ciVolatileShare);
|
|
}
|
|
}
|
|
}
|
|
|
|
Log.Info(string.Format("Build {0} succeeded", repo));
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Log.Warn(string.Format("Build {0} failed: {1}", repo, ex.Message));
|
|
failed[repo] = ex;
|
|
}
|
|
finally
|
|
{
|
|
if (IsTeamCity)
|
|
{
|
|
Log.Info(string.Format("##teamcity[blockClosed name='{0}']", FormatForTeamCity(blockName)));
|
|
}
|
|
}
|
|
}
|
|
|
|
foreach(var repo in repositories)
|
|
{
|
|
Exception ex;
|
|
if (failed.TryGetValue(repo, out ex))
|
|
{
|
|
Log.Warn(string.Format("Build {0} failed: {1}", repo, ex.Message));
|
|
if (IsTeamCity)
|
|
{
|
|
Log.Warn(string.Format("##teamcity[message text='{0}' errorDetails='{1}' status='ERROR']",
|
|
FormatForTeamCity(ex.Message),
|
|
FormatForTeamCity(ex.StackTrace)));
|
|
}
|
|
|
|
}
|
|
else if (skipped.Contains(repo))
|
|
{
|
|
Log.Warn(string.Format("Build {0} skipped", repo));
|
|
}
|
|
else
|
|
{
|
|
Log.Info(string.Format("Build {0} succeeded", repo));
|
|
}
|
|
}
|
|
|
|
if (failed.Any())
|
|
{
|
|
throw new Exception("One or more repos failed to build");
|
|
}
|
|
}
|
|
|
|
#only-install target='install'
|
|
@{
|
|
foreach(var repo in repositories)
|
|
{
|
|
if (IsMono)
|
|
{
|
|
Exec(Path.Combine(repo, "build.sh"), "install", repo);
|
|
}
|
|
else
|
|
{
|
|
Exec("build.cmd", "install", repo);
|
|
}
|
|
}
|
|
}
|
|
|
|
#git-status description='Show status of repos known by Universe'
|
|
@{
|
|
foreach(var repo in repositories)
|
|
{
|
|
GitStatus(repo);
|
|
}
|
|
}
|
|
|
|
#git-clean description='REMOVE ALL CHANGES to the working directory'
|
|
@{
|
|
Console.WriteLine("This runs `git clean -xfd` in all non-Universe repos.");
|
|
Console.WriteLine("This should REMOVE ALL CHANGES to the working directory.");
|
|
Console.Write("***** Are you sure? ***** (Y or anything else)? ");
|
|
if (Console.ReadLine() != "Y")
|
|
{
|
|
throw new Exception("git-clean cancelled");
|
|
}
|
|
foreach(var repo in repositories)
|
|
{
|
|
GitClean(repo);
|
|
}
|
|
}
|
|
|
|
macro name='ExecClr' program='string' commandline='string'
|
|
exec-clr
|
|
|
|
macro name='Git' gitCommand='string' gitFolder='string'
|
|
git
|
|
|
|
macro name='GitPull' gitUri='string' gitBranch='string' gitFolder='string'
|
|
git-pull
|
|
|
|
macro name='GitClone' gitUri='string' gitBranch='string'
|
|
git-clone
|
|
|
|
macro name='GitConfig' gitOptionName='string' gitOptionValue='string' gitFolder='string'
|
|
git-config
|
|
|
|
macro name='GitStatus' gitFolder='string'
|
|
git gitCommand='status'
|
|
|
|
macro name='GitClean' gitFolder='string'
|
|
git gitCommand='clean -xdf'
|
|
|
|
macro name='GitCommand' gitFolder='string' gitCommand='string'
|
|
git
|
|
|
|
macro name='Exec' program='string' commandline='string' workingdir='string'
|
|
exec
|
|
|
|
macro name='NuGetPackagesAdd' sourcePackagesDir='string' targetPackagesDir='string'
|
|
nuget-packages-add
|
|
|
|
macro name="CopyFolder" sourceDir='string' outputDir='string' overwrite='bool'
|
|
copy
|
|
|
|
functions
|
|
@{
|
|
static IDictionary<string, string> GetCommitsToSync(string commitsFile)
|
|
{
|
|
var commits = new Dictionary<string, string>();
|
|
|
|
if (string.IsNullOrEmpty(commitsFile))
|
|
{
|
|
return commits;
|
|
}
|
|
|
|
Console.WriteLine("Using commits file: " + commitsFile);
|
|
var lines = File.ReadAllLines(commitsFile);
|
|
foreach(var line in lines)
|
|
{
|
|
var parts = line.Split(new string[] {":"}, StringSplitOptions.RemoveEmptyEntries);
|
|
commits.Add(parts[0], parts[1]);
|
|
}
|
|
|
|
return commits;
|
|
}
|
|
|
|
static bool UseHttps(string directory)
|
|
{
|
|
var filename = Path.Combine(directory, ".git", "config");
|
|
if (!File.Exists(filename))
|
|
{
|
|
Console.WriteLine("Unable to find '{0}' file", filename);
|
|
return false;
|
|
}
|
|
|
|
var url = ReadOriginUrl(filename);
|
|
return IsHttpsUrl(url);
|
|
}
|
|
|
|
// Perform equivalent of `git config remote.origin.url` but directly
|
|
// read config file to get value.
|
|
static string ReadOriginUrl(string filename)
|
|
{
|
|
// Subsection portion of configuration name is case-sensitive; rest
|
|
// of name is case-insensitive.
|
|
var beginOriginSection = new Regex(@"^\[(?i:remote) ""origin""\]\s*$");
|
|
var beginSection = new Regex(@"^\[");
|
|
var urlLine = new Regex(@"^\s+url = (\S+)\s*$", RegexOptions.IgnoreCase);
|
|
|
|
var inRemoteOriginSection = false;
|
|
foreach (var line in File.ReadAllLines(filename))
|
|
{
|
|
if (beginOriginSection.IsMatch(line))
|
|
{
|
|
inRemoteOriginSection = true;
|
|
continue;
|
|
}
|
|
|
|
if (inRemoteOriginSection)
|
|
{
|
|
if (beginSection.IsMatch(line))
|
|
{
|
|
// Read through the section without finding URL line.
|
|
break;
|
|
}
|
|
|
|
var match = urlLine.Match(line);
|
|
if (match.Success && match.Groups.Count == 2 && match.Groups[1].Success)
|
|
{
|
|
return match.Groups[1].Value;
|
|
}
|
|
}
|
|
}
|
|
|
|
Console.WriteLine("Unable to parse '{0}' file", filename);
|
|
return null;
|
|
}
|
|
|
|
static bool IsHttpsUrl(string url)
|
|
{
|
|
if (string.IsNullOrEmpty(url))
|
|
{
|
|
return false;
|
|
}
|
|
|
|
return url.StartsWith("https://", System.StringComparison.OrdinalIgnoreCase);
|
|
}
|
|
|
|
static bool IsAccessible(string key)
|
|
{
|
|
var req = WebRequest.CreateHttp("https://github.com/aspnet/" + key);
|
|
req.Method = "HEAD";
|
|
try
|
|
{
|
|
using (req.GetResponse())
|
|
{
|
|
// intentionally empty
|
|
}
|
|
}
|
|
catch (WebException ex)
|
|
{
|
|
if (ex.Response != null &&
|
|
((HttpWebResponse)ex.Response).StatusCode == HttpStatusCode.NotFound)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
// Ignore any other exception. They should surface as part of git clone with well-known messages.
|
|
}
|
|
return true;
|
|
}
|
|
|
|
void RemoveSrcFolder(string repo)
|
|
{
|
|
var srcDir = Path.Combine(repo, "src");
|
|
if (Directory.Exists(srcDir))
|
|
{
|
|
Console.WriteLine("Deleting " + srcDir);
|
|
Directory.Delete(srcDir, recursive: true);
|
|
}
|
|
}
|
|
|
|
void CloneOrUpdate(string repo)
|
|
{
|
|
var repoUrl = gitHubUriPrefix + repo + ".git";
|
|
|
|
if (Directory.Exists(repo))
|
|
{
|
|
GitPull(repoUrl, buildBranch, repo);
|
|
}
|
|
else
|
|
{
|
|
if (useHttps &&
|
|
!IsAccessible(repo))
|
|
{
|
|
if (skipNoCredentials)
|
|
{
|
|
// This is used on the XPlat CI. Must return because otherwise git will wait for user
|
|
// input and hang the build
|
|
Log.Warn(string.Format("The repo at '{0}' is not accesible and it will not be cloned.", repoUrl));
|
|
return;
|
|
}
|
|
|
|
Log.Warn(string.Format("The repo at '{0}' is not accessible. If you do not have access to this repository, skip the git prompt" +
|
|
" for credentials to skip cloning this repository. To avoid this prompt, re-clone the Universe repository over ssh.",
|
|
repoUrl));
|
|
}
|
|
|
|
try
|
|
{
|
|
GitClone(repoUrl, buildBranch);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Log.Warn(string.Format("Unable to clone repository at '{0}': {1}", repoUrl, ex.Message));
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
|
|
string FormatForTeamCity(string input)
|
|
{
|
|
return input.Replace("|", "||")
|
|
.Replace("'", "|'")
|
|
.Replace("\r", "|r")
|
|
.Replace("\n", "|n")
|
|
.Replace("]", "|]");
|
|
}
|
|
|
|
static IEnumerable<string> GetRepositoriesToBuild()
|
|
{
|
|
IEnumerable<string> reposToBuild = new HashSet<string>(StringComparer.OrdinalIgnoreCase)
|
|
{
|
|
"PlatformAbstractions",
|
|
"Common",
|
|
"JsonPatch",
|
|
"FileSystem",
|
|
"Configuration",
|
|
"DependencyInjection",
|
|
"EventNotification",
|
|
"Options",
|
|
"Logging",
|
|
"DotNetTools",
|
|
"HtmlAbstractions",
|
|
"DataProtection",
|
|
"HttpAbstractions",
|
|
"Testing",
|
|
"Microsoft.Data.Sqlite",
|
|
"Caching",
|
|
"Razor",
|
|
"RazorTooling",
|
|
"Hosting",
|
|
"AzureIntegration",
|
|
"EntityFramework",
|
|
"EntityFramework.Tools",
|
|
"WebListener",
|
|
"KestrelHttpServer",
|
|
"IISIntegration",
|
|
"ServerTests",
|
|
"ResponseCaching",
|
|
"Session",
|
|
"CORS",
|
|
"Routing",
|
|
"StaticFiles",
|
|
"Diagnostics",
|
|
"Security",
|
|
"Antiforgery",
|
|
"WebSockets",
|
|
"Localization",
|
|
"BasicMiddleware",
|
|
"Proxy",
|
|
"Mvc",
|
|
"MvcPrecompilation",
|
|
"Identity",
|
|
"Scaffolding",
|
|
"SignalR",
|
|
"BrowserLink",
|
|
"Entropy",
|
|
"MusicStore",
|
|
"MetaPackages",
|
|
};
|
|
|
|
if (!string.IsNullOrEmpty(repositoryInclude))
|
|
{
|
|
reposToBuild = new HashSet<string>(
|
|
repositoryInclude.Split(new string[] {","}, StringSplitOptions.RemoveEmptyEntries),
|
|
StringComparer.OrdinalIgnoreCase);
|
|
}
|
|
|
|
if (!string.IsNullOrEmpty(repositoryExclude))
|
|
{
|
|
var reposToExclude = repositoryExclude.Split(new string[] {","}, StringSplitOptions.RemoveEmptyEntries);
|
|
reposToBuild = reposToBuild.Except(reposToExclude);
|
|
}
|
|
|
|
return reposToBuild.ToList();
|
|
}
|
|
|
|
static IEnumerable<string> GetNoSrcDeleteRepositories()
|
|
{
|
|
var repositoryExclude = Environment.GetEnvironmentVariable("KOREBUILD_NOSRC_EXCLUDE");
|
|
if (string.IsNullOrEmpty(repositoryExclude))
|
|
{
|
|
return new string[0];
|
|
}
|
|
|
|
return repositoryExclude.Split(new string[] {","}, StringSplitOptions.RemoveEmptyEntries);
|
|
}
|
|
|
|
static string DefaultDropsShare(string value)
|
|
{
|
|
return value ?? (Environment.OSVersion.Platform == PlatformID.Unix ? "/aspnetci-drops" : @"\\aspnetci\drops");
|
|
}
|
|
|
|
string CreateBuildWithFlowId(string repo)
|
|
{
|
|
string output;
|
|
if (IsLinux)
|
|
{
|
|
output = Path.Combine(repo, "build.sh");
|
|
}
|
|
else
|
|
{
|
|
output = Path.Combine(repo, "build-with-flow-id.cmd");
|
|
File.WriteAllText(
|
|
output,
|
|
string.Format("set KOREBUILD_FLOWID=KOREBUILD_{0}& call build.cmd %*", repo));
|
|
}
|
|
|
|
return Path.GetFullPath(output);
|
|
}
|
|
|
|
static IList<IGrouping<int, string>> GetBuildGraph()
|
|
{
|
|
var repositoryLookup = GetRepositoriesToBuild()
|
|
.Select(RepoInfoFactory.Create)
|
|
.ToList();
|
|
|
|
foreach (var info in repositoryLookup)
|
|
{
|
|
foreach (var item in info.DependencyNames)
|
|
{
|
|
var dependency = repositoryLookup.Find(r => r.RepositoryNames.Contains(item));
|
|
if (dependency != null)
|
|
{
|
|
info.Dependencies.Add(dependency);
|
|
}
|
|
}
|
|
}
|
|
|
|
return repositoryLookup.GroupBy(r => r.Order, r => r.Name).OrderBy(r => r.Key).ToArray();
|
|
}
|
|
|
|
private class RepoInfoFactory
|
|
{
|
|
public static RepositoryInfo Create(string repo)
|
|
{
|
|
var info = new RepositoryInfo { Name = repo };
|
|
ResolveProjectJsonProjects(info, repo);
|
|
ResolveMsBuildProjects(info, repo);
|
|
info.DependencyNames.ExceptWith(info.RepositoryNames);
|
|
return info;
|
|
}
|
|
|
|
private static void ResolveMsBuildProjects(RepositoryInfo info, string repo)
|
|
{
|
|
foreach (var sln in Directory.EnumerateFiles(repo, "*.sln"))
|
|
{
|
|
var lines = File.ReadAllLines(sln);
|
|
foreach (var line in lines)
|
|
{
|
|
var match = Regex.Match(line, @"^# Visual Studio (\d+)\s?");
|
|
if (match == null || match.Groups.Count < 2)
|
|
{
|
|
continue;
|
|
}
|
|
int version;
|
|
if (int.TryParse(match.Groups[1].Value, out version) && version >= 15)
|
|
{
|
|
ResolveMsBuildProject(info, sln);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
private static void ResolveMsBuildProject(RepositoryInfo info, string projectFile)
|
|
{
|
|
var intermediate = Path.Combine(Directory.GetCurrentDirectory(), "obj");
|
|
Directory.CreateDirectory(intermediate);
|
|
var dgJson = Path.Combine(intermediate, Path.GetFileName(projectFile) + ".graph.json");
|
|
var psi = new ProcessStartInfo
|
|
{
|
|
UseShellExecute = false,
|
|
FileName = "dotnet",
|
|
Arguments = "msbuild \"" + projectFile + "\" /t:GenerateRestoreGraphFile \"/p:RestoreGraphOutputPath=" + dgJson + "\""
|
|
};
|
|
var p = Process.Start(psi);
|
|
p.WaitForExit();
|
|
if (p.ExitCode != 0)
|
|
{
|
|
Console.WriteLine("warn: Failed to get restore graph from " + projectFile);
|
|
return;
|
|
}
|
|
|
|
JsonObject graph;
|
|
try
|
|
{
|
|
graph = (JsonObject)Json.Deserialize(File.ReadAllText(dgJson));
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.Error.WriteLine("Failed to parse dependency graph");
|
|
throw;
|
|
}
|
|
|
|
var format = graph.ValueAsInt("format");
|
|
if (format != 1)
|
|
{
|
|
Console.Error.WriteLine("Dependency graph file in unsupported format version: " + format);
|
|
throw new FormatException();
|
|
}
|
|
|
|
var projects = graph.ValueAsJsonObject("projects");
|
|
foreach (var projectKey in projects.Keys)
|
|
{
|
|
var project = projects.ValueAsJsonObject(projectKey);
|
|
var restore = project.ValueAsJsonObject("restore");
|
|
var projectName = restore.ValueAsString("projectName");
|
|
info.RepositoryNames.Add(projectName);
|
|
|
|
var frameworks = project.ValueAsJsonObject("frameworks");
|
|
foreach (var fxKey in frameworks.Keys)
|
|
{
|
|
var fxDeps = frameworks.ValueAsJsonObject(fxKey).ValueAsJsonObject("dependencies");
|
|
foreach (var depKey in fxDeps.Keys)
|
|
{
|
|
var dep = fxDeps.ValueAsJsonObject(depKey);
|
|
if (dep.ValueAsString("target") != "Package")
|
|
{
|
|
continue;
|
|
}
|
|
// todo version?
|
|
info.DependencyNames.Add(depKey);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
private static void ResolveProjectJsonProjects(RepositoryInfo info, string repo)
|
|
{
|
|
var srcDir = Path.Combine(repo, "src");
|
|
|
|
if (Directory.Exists(srcDir))
|
|
{
|
|
foreach (var directory in Directory.EnumerateDirectories(srcDir))
|
|
{
|
|
info.RepositoryNames.Add(Path.GetFileName(directory));
|
|
var projectJson = Path.Combine(directory, "project.json");
|
|
if (File.Exists(projectJson))
|
|
{
|
|
ResolveDependencies(projectJson, info.DependencyNames);
|
|
}
|
|
}
|
|
}
|
|
|
|
var otherDirs = new[] { "test", "samples", "tools" };
|
|
for (var i = 0; i < otherDirs.Length; i++)
|
|
{
|
|
var otherDir = Path.Combine(repo, otherDirs[i]);
|
|
if (Directory.Exists(otherDir))
|
|
{
|
|
foreach (var directory in Directory.EnumerateDirectories(otherDir))
|
|
{
|
|
var projectJson = Path.Combine(directory, "project.json");
|
|
if (File.Exists(projectJson))
|
|
{
|
|
ResolveDependencies(projectJson, info.DependencyNames);
|
|
}
|
|
else
|
|
{
|
|
// Look for test\Websites\WebsiteName\project.json
|
|
foreach (var subDirectory in Directory.EnumerateDirectories(directory))
|
|
{
|
|
projectJson = Path.Combine(subDirectory, "project.json");
|
|
if (File.Exists(projectJson))
|
|
{
|
|
ResolveDependencies(projectJson, info.DependencyNames);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
private static void ResolveDependencies(string projectJsonPath, HashSet<string> dependencies)
|
|
{
|
|
JsonObject project;
|
|
try
|
|
{
|
|
project = (JsonObject)Json.Deserialize(File.ReadAllText(projectJsonPath));
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.Error.WriteLine("Failed to parse project at path " + projectJsonPath + Environment.NewLine + ex.Message);
|
|
throw;
|
|
}
|
|
var dependenciesNode = project.ValueAsJsonObject("dependencies");
|
|
AddKeys(dependencies, dependenciesNode);
|
|
|
|
var frameworkNodes = project.ValueAsJsonObject("frameworks");
|
|
if (frameworkNodes != null)
|
|
{
|
|
foreach (var framework in frameworkNodes.Keys)
|
|
{
|
|
dependenciesNode = frameworkNodes.ValueAsJsonObject(framework).ValueAsJsonObject("dependencies");
|
|
AddKeys(dependencies, dependenciesNode);
|
|
}
|
|
}
|
|
|
|
AddKeys(dependencies, project.ValueAsJsonObject("tools"));
|
|
}
|
|
|
|
private static void AddKeys(HashSet<string> target, JsonObject source)
|
|
{
|
|
if (source != null)
|
|
{
|
|
foreach (var key in source.Keys)
|
|
{
|
|
target.Add(key);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
private class RepositoryInfo
|
|
{
|
|
public string Name;
|
|
|
|
public HashSet<string> RepositoryNames = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
|
|
|
public HashSet<string> DependencyNames = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
|
|
|
public HashSet<RepositoryInfo> Dependencies = new HashSet<RepositoryInfo>();
|
|
|
|
public int Order
|
|
{
|
|
get
|
|
{
|
|
return GetOrder(new List<RepositoryInfo>(), this);
|
|
}
|
|
}
|
|
|
|
private static int GetOrder(List<RepositoryInfo> visited, RepositoryInfo info)
|
|
{
|
|
if (visited.Contains(info))
|
|
{
|
|
throw new Exception("A cyclic dependency between the following repositories has been detected: " +
|
|
string.Join(" -> ", visited));
|
|
}
|
|
|
|
visited.Add(info);
|
|
|
|
var order = 0;
|
|
foreach (var dependency in info.Dependencies)
|
|
{
|
|
order = Math.Max(order, GetOrder(visited, dependency));
|
|
}
|
|
|
|
visited.RemoveAt(visited.Count - 1);
|
|
|
|
return order + 1;
|
|
}
|
|
|
|
public override string ToString()
|
|
{
|
|
return Name;
|
|
}
|
|
}
|
|
}
|