huahaofeng
/
aspnetcore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Resolve dependencies and packages from VS2017 solutions (#464) 

* Update 'GetBuildGraph' to understand VS2017 solutions
* Ensure Universe allows repos to self-determine their version of KoreBuild
This commit is contained in:
Nate McMaster 2016-12-14 11:09:47 -08:00 committed by GitHub
parent a6c8fcca2c
commit 503b4c97ed
4 changed files with 273 additions and 130 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
ToolsVersion.txt Normal file
View File

@ -0,0 +1 @@
feature/msbuild

4
build.ps1
View File

@ -32,8 +32,8 @@ cd $PSScriptRoot
$repoFolder = $PSScriptRoot
$env:REPO_FOLDER = $repoFolder
$koreBuildZip="https://github.com/aspnet/KoreBuild/archive/dev.zip"
$korebuildVersion = $(Get-Content -Raw $PSScriptRoot/ToolsVersion.txt).Trim()
$koreBuildZip="https://github.com/aspnet/KoreBuild/archive/$korebuildVersion.zip"
if ($env:KOREBUILD_ZIP)
{
$koreBuildZip=$env:KOREBUILD_ZIP

3
build.sh
View File

@ -2,7 +2,8 @@
repoFolder="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd $repoFolder
koreBuildZip="https://github.com/aspnet/KoreBuild/archive/dev.zip"
toolsVersion=$(cat $repoFolder/ToolsVersion.txt)
koreBuildZip="https://github.com/aspnet/KoreBuild/archive/$toolsVersion.zip"
if [ ! -z $KOREBUILD_ZIP ]; then
koreBuildZip=$KOREBUILD_ZIP
fi

273
makefile.shade
View File

@ -91,6 +91,14 @@ var buildTarget = "compile"
});
}
#update
@{
Parallel.ForEach(repositories, repo =>
{
Git("pull --ff-only", repo);
});
}
#sync-commits
@{
if (string.IsNullOrEmpty(universeCommitsFile))
@ -163,6 +171,16 @@ var buildTarget = "compile"
}
}
#show-build-graph
@{
var batchedRepos = GetBuildGraph();
Log.Info("Building repositories in batches: ");
foreach (var repos in batchedRepos)
{
Log.Info(string.Format("{0} - {1}", repos.Key, string.Join(", ", repos)));
}
}
#ci-build
@{
var nugetExe = Path.Combine(Directory.GetCurrentDirectory(), ".build", "nuget.exe");
@ -218,6 +236,8 @@ var buildTarget = "compile"
Environment.SetEnvironmentVariable("NUGET_PUBLISH_FEED", "https://dotnet.myget.org/F/aspnetcore-volatile-dev/api/v2/package");
}
var universeToolsVersion = File.ReadAllText(Path.Combine(Directory.GetCurrentDirectory(), "ToolsVersion.txt")).Trim();
foreach (var batch in batchedRepos)
{
Parallel.ForEach(batch.ToArray(), new ParallelOptions { MaxDegreeOfParallelism = threads }, repo =>
@ -229,6 +249,30 @@ var buildTarget = "compile"
blockLogger.StartBlock(blockName);
}
// begin workaround
// as we transition to VS 2017, some repos will build with project.json and others with csproj
// the 'csproj' version of KoreBuild is the 'feature/msbuild' branch of that repo. project.json
// support is on 'dev'. This workaround avoids installing the version of KoreBuild used by
// individual repos unless it is already upgraded to the version currently used by Universe.
var repoToolsVersion = universeToolsVersion;
var buildPs1 = Path.Combine(repo, "build.ps1");
if (File.Exists(buildPs1))
{
foreach (var line in File.ReadAllLines(buildPs1))
{
var match = Regex.Match(line, @".*https://github.com/aspnet/KoreBuild/archive/(.+)\.zip.*");
if (match != null && match.Success)
{
repoToolsVersion = match.Groups[1].Value;
break;
}
}
}
Log.Info(repo + " KoreBuild version = " + repoToolsVersion + ", Universe = " + universeToolsVersion);
// end workaround
if (repoToolsVersion == universeToolsVersion)
{
if (!IsLinux)
{
Exec("cmd", "/C xcopy /S/Q/I/Y .build " + Path.Combine(repo, ".build"), "");
@ -237,6 +281,7 @@ var buildTarget = "compile"
{
CopyFolder(".build", Path.Combine(repo, ".build"), true);
}
}
try
{
@ -802,60 +847,35 @@ functions
return repositoryExclude.Split(new string[] {","}, StringSplitOptions.RemoveEmptyEntries);
}
static IList<IGrouping<int, string>> GetBuildGraph()
static string DefaultDropsShare(string value)
{
var repositoryLookup = new List<RepositoryInfo>();
foreach (var repo in GetRepositoriesToBuild())
{
var info = new RepositoryInfo { Name = repo };
var srcDir = Path.Combine(repo, "src");
if (Directory.Exists(srcDir))
{
foreach (var directory in Directory.EnumerateDirectories(srcDir))
{
info.RepositoryNames.Add(Path.GetFileName(directory));
var projectJson = Path.Combine(directory, "project.json");
if (File.Exists(projectJson))
{
GetDependencies(projectJson, info.DependencyNames);
}
}
return value ?? (Environment.OSVersion.Platform == PlatformID.Unix ? "/aspnetci-drops" : @"\\aspnetci\drops");
}
var otherDirs = new[] { "test", "samples", "tools" };
for (var i = 0; i < otherDirs.Length; i++)
string CreateBuildWithFlowId(string repo)
{
var otherDir = Path.Combine(repo, otherDirs[i]);
if (Directory.Exists(otherDir))
string output;
if (IsLinux)
{
foreach (var directory in Directory.EnumerateDirectories(otherDir))
{
var projectJson = Path.Combine(directory, "project.json");
if (File.Exists(projectJson))
{
GetDependencies(projectJson, info.DependencyNames);
output = Path.Combine(repo, "build.sh");
}
else
{
// Look for test\Websites\WebsiteName\project.json
foreach (var subDirectory in Directory.EnumerateDirectories(directory))
{
projectJson = Path.Combine(subDirectory, "project.json");
if (File.Exists(projectJson))
{
GetDependencies(projectJson, info.DependencyNames);
}
}
}
}
}
output = Path.Combine(repo, "build-with-flow-id.cmd");
File.WriteAllText(
output,
string.Format("set KOREBUILD_FLOWID=KOREBUILD_{0}& call build.cmd %*", repo));
}
info.DependencyNames.ExceptWith(info.RepositoryNames);
repositoryLookup.Add(info);
return Path.GetFullPath(output);
}
static IList<IGrouping<int, string>> GetBuildGraph()
{
var repositoryLookup = GetRepositoriesToBuild()
.Select(RepoInfoFactory.Create)
.ToList();
foreach (var info in repositoryLookup)
{
foreach (var item in info.DependencyNames)
@ -871,7 +891,150 @@ functions
return repositoryLookup.GroupBy(r => r.Order, r => r.Name).OrderBy(r => r.Key).ToArray();
}
static void GetDependencies(string projectJsonPath, HashSet<string> dependencies)
private class RepoInfoFactory
{
public static RepositoryInfo Create(string repo)
{
var info = new RepositoryInfo { Name = repo };
ResolveProjectJsonProjects(info, repo);
ResolveMsBuildProjects(info, repo);
info.DependencyNames.ExceptWith(info.RepositoryNames);
return info;
}
private static void ResolveMsBuildProjects(RepositoryInfo info, string repo)
{
foreach (var sln in Directory.EnumerateFiles(repo, "*.sln"))
{
var lines = File.ReadAllLines(sln);
foreach (var line in lines)
{
var match = Regex.Match(line, @"^# Visual Studio (\d+)\s?");
if (match == null || match.Groups.Count < 2)
{
continue;
}
int version;
if (int.TryParse(match.Groups[1].Value, out version) && version >= 15)
{
ResolveMsBuildProject(info, sln);
break;
}
}
}
}
private static void ResolveMsBuildProject(RepositoryInfo info, string projectFile)
{
var intermediate = Path.Combine(Directory.GetCurrentDirectory(), "obj");
Directory.CreateDirectory(intermediate);
var dgJson = Path.Combine(intermediate, Path.GetFileName(projectFile) + ".graph.json");
var psi = new ProcessStartInfo
{
UseShellExecute = false,
FileName = "dotnet",
Arguments = "msbuild \"" + projectFile + "\" /t:GenerateRestoreGraphFile \"/p:RestoreGraphOutputPath=" + dgJson + "\""
};
var p = Process.Start(psi);
p.WaitForExit();
if (p.ExitCode != 0)
{
Console.WriteLine("warn: Failed to get restore graph from " + projectFile);
return;
}
JsonObject graph;
try
{
graph = (JsonObject)Json.Deserialize(File.ReadAllText(dgJson));
}
catch (Exception ex)
{
Console.Error.WriteLine("Failed to parse dependency graph");
throw;
}
var format = graph.ValueAsInt("format");
if (format != 1)
{
Console.Error.WriteLine("Dependency graph file in unsupported format version: " + format);
throw new FormatException();
}
var projects = graph.ValueAsJsonObject("projects");
foreach (var projectKey in projects.Keys)
{
var project = projects.ValueAsJsonObject(projectKey);
var restore = project.ValueAsJsonObject("restore");
var projectName = restore.ValueAsString("projectName");
info.RepositoryNames.Add(projectName);
var frameworks = project.ValueAsJsonObject("frameworks");
foreach (var fxKey in frameworks.Keys)
{
var fxDeps = frameworks.ValueAsJsonObject(fxKey).ValueAsJsonObject("dependencies");
foreach (var depKey in fxDeps.Keys)
{
var dep = fxDeps.ValueAsJsonObject(depKey);
if (dep.ValueAsString("target") != "Package")
{
continue;
}
// todo version?
info.DependencyNames.Add(depKey);
}
}
}
}
private static void ResolveProjectJsonProjects(RepositoryInfo info, string repo)
{
var srcDir = Path.Combine(repo, "src");
if (Directory.Exists(srcDir))
{
foreach (var directory in Directory.EnumerateDirectories(srcDir))
{
info.RepositoryNames.Add(Path.GetFileName(directory));
var projectJson = Path.Combine(directory, "project.json");
if (File.Exists(projectJson))
{
ResolveDependencies(projectJson, info.DependencyNames);
}
}
}
var otherDirs = new[] { "test", "samples", "tools" };
for (var i = 0; i < otherDirs.Length; i++)
{
var otherDir = Path.Combine(repo, otherDirs[i]);
if (Directory.Exists(otherDir))
{
foreach (var directory in Directory.EnumerateDirectories(otherDir))
{
var projectJson = Path.Combine(directory, "project.json");
if (File.Exists(projectJson))
{
ResolveDependencies(projectJson, info.DependencyNames);
}
else
{
// Look for test\Websites\WebsiteName\project.json
foreach (var subDirectory in Directory.EnumerateDirectories(directory))
{
projectJson = Path.Combine(subDirectory, "project.json");
if (File.Exists(projectJson))
{
ResolveDependencies(projectJson, info.DependencyNames);
}
}
}
}
}
}
}
private static void ResolveDependencies(string projectJsonPath, HashSet<string> dependencies)
{
JsonObject project;
try
@ -899,7 +1062,7 @@ functions
AddKeys(dependencies, project.ValueAsJsonObject("tools"));
}
static void AddKeys(HashSet<string> target, JsonObject source)
private static void AddKeys(HashSet<string> target, JsonObject source)
{
if (source != null)
{
@ -909,28 +1072,6 @@ functions
}
}
}
static string DefaultDropsShare(string value)
{
return value ?? (Environment.OSVersion.Platform == PlatformID.Unix ? "/aspnetci-drops" : @"\\aspnetci\drops");
}
string CreateBuildWithFlowId(string repo)
{
string output;
if (IsLinux)
{
output = Path.Combine(repo, "build.sh");
}
else
{
output = Path.Combine(repo, "build-with-flow-id.cmd");
File.WriteAllText(
output,
string.Format("set KOREBUILD_FLOWID=KOREBUILD_{0}& call build.cmd %*", repo));
}
return Path.GetFullPath(output);
}
private class RepositoryInfo