diff --git a/ToolsVersion.txt b/ToolsVersion.txt new file mode 100644 index 0000000000..42edac083a --- /dev/null +++ b/ToolsVersion.txt @@ -0,0 +1 @@ +feature/msbuild \ No newline at end of file diff --git a/build.ps1 b/build.ps1 index 8f2f99691a..8841950991 100644 --- a/build.ps1 +++ b/build.ps1 @@ -1,6 +1,6 @@ $ErrorActionPreference = "Stop" -function DownloadWithRetry([string] $url, [string] $downloadLocation, [int] $retries) +function DownloadWithRetry([string] $url, [string] $downloadLocation, [int] $retries) { while($true) { @@ -19,7 +19,7 @@ function DownloadWithRetry([string] $url, [string] $downloadLocation, [int] $ret Start-Sleep -Seconds 10 } - else + else { $exception = $_.Exception throw $exception @@ -32,8 +32,8 @@ cd $PSScriptRoot $repoFolder = $PSScriptRoot $env:REPO_FOLDER = $repoFolder - -$koreBuildZip="https://github.com/aspnet/KoreBuild/archive/dev.zip" +$korebuildVersion = $(Get-Content -Raw $PSScriptRoot/ToolsVersion.txt).Trim() +$koreBuildZip="https://github.com/aspnet/KoreBuild/archive/$korebuildVersion.zip" if ($env:KOREBUILD_ZIP) { $koreBuildZip=$env:KOREBUILD_ZIP @@ -43,18 +43,18 @@ $buildFolder = ".build" $buildFile="$buildFolder\KoreBuild.ps1" if (!(Test-Path $buildFolder)) { - Write-Host "Downloading KoreBuild from $koreBuildZip" - + Write-Host "Downloading KoreBuild from $koreBuildZip" + $tempFolder=$env:TEMP + "\KoreBuild-" + [guid]::NewGuid() New-Item -Path "$tempFolder" -Type directory | Out-Null $localZipFile="$tempFolder\korebuild.zip" - + DownloadWithRetry -url $koreBuildZip -downloadLocation $localZipFile -retries 6 Add-Type -AssemblyName System.IO.Compression.FileSystem [System.IO.Compression.ZipFile]::ExtractToDirectory($localZipFile, $tempFolder) - + New-Item -Path "$buildFolder" -Type directory | Out-Null copy-item "$tempFolder\**\build\*" $buildFolder -Recurse diff --git a/build.sh b/build.sh index 4fd7ede788..c830978741 100755 --- a/build.sh +++ b/build.sh @@ -2,7 +2,8 @@ repoFolder="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd $repoFolder -koreBuildZip="https://github.com/aspnet/KoreBuild/archive/dev.zip" +toolsVersion=$(cat $repoFolder/ToolsVersion.txt) +koreBuildZip="https://github.com/aspnet/KoreBuild/archive/$toolsVersion.zip" if [ ! -z $KOREBUILD_ZIP ]; then koreBuildZip=$KOREBUILD_ZIP fi @@ -12,12 +13,12 @@ buildFile="$buildFolder/KoreBuild.sh" if test ! -d $buildFolder; then echo "Downloading KoreBuild from $koreBuildZip" - - tempFolder="/tmp/KoreBuild-$(uuidgen)" + + tempFolder="/tmp/KoreBuild-$(uuidgen)" mkdir $tempFolder - + localZipFile="$tempFolder/korebuild.zip" - + retries=6 until (wget -O $localZipFile $koreBuildZip 2>/dev/null || curl -o $localZipFile --location $koreBuildZip 2>/dev/null) do @@ -29,17 +30,17 @@ if test ! -d $buildFolder; then echo "Waiting 10 seconds before retrying. Retries left: $retries" sleep 10s done - + unzip -q -d $tempFolder $localZipFile - + mkdir $buildFolder cp -r $tempFolder/**/build/** $buildFolder - + chmod +x $buildFile - + # Cleanup if test -d $tempFolder; then - rm -rf $tempFolder + rm -rf $tempFolder fi fi diff --git a/makefile.shade b/makefile.shade index f7bc525ec4..dedc42b3ca 100644 --- a/makefile.shade +++ b/makefile.shade @@ -91,6 +91,14 @@ var buildTarget = "compile" }); } +#update + @{ + Parallel.ForEach(repositories, repo => + { + Git("pull --ff-only", repo); + }); + } + #sync-commits @{ if (string.IsNullOrEmpty(universeCommitsFile)) @@ -163,6 +171,16 @@ var buildTarget = "compile" } } +#show-build-graph + @{ + var batchedRepos = GetBuildGraph(); + Log.Info("Building repositories in batches: "); + foreach (var repos in batchedRepos) + { + Log.Info(string.Format("{0} - {1}", repos.Key, string.Join(", ", repos))); + } + } + #ci-build @{ var nugetExe = Path.Combine(Directory.GetCurrentDirectory(), ".build", "nuget.exe"); @@ -218,6 +236,8 @@ var buildTarget = "compile" Environment.SetEnvironmentVariable("NUGET_PUBLISH_FEED", "https://dotnet.myget.org/F/aspnetcore-volatile-dev/api/v2/package"); } + var universeToolsVersion = File.ReadAllText(Path.Combine(Directory.GetCurrentDirectory(), "ToolsVersion.txt")).Trim(); + foreach (var batch in batchedRepos) { Parallel.ForEach(batch.ToArray(), new ParallelOptions { MaxDegreeOfParallelism = threads }, repo => @@ -229,13 +249,38 @@ var buildTarget = "compile" blockLogger.StartBlock(blockName); } - if (!IsLinux) + // begin workaround + // as we transition to VS 2017, some repos will build with project.json and others with csproj + // the 'csproj' version of KoreBuild is the 'feature/msbuild' branch of that repo. project.json + // support is on 'dev'. This workaround avoids installing the version of KoreBuild used by + // individual repos unless it is already upgraded to the version currently used by Universe. + var repoToolsVersion = universeToolsVersion; + var buildPs1 = Path.Combine(repo, "build.ps1"); + if (File.Exists(buildPs1)) { - Exec("cmd", "/C xcopy /S/Q/I/Y .build " + Path.Combine(repo, ".build"), ""); + foreach (var line in File.ReadAllLines(buildPs1)) + { + var match = Regex.Match(line, @".*https://github.com/aspnet/KoreBuild/archive/(.+)\.zip.*"); + if (match != null && match.Success) + { + repoToolsVersion = match.Groups[1].Value; + break; + } + } } - else + Log.Info(repo + " KoreBuild version = " + repoToolsVersion + ", Universe = " + universeToolsVersion); + // end workaround + + if (repoToolsVersion == universeToolsVersion) { - CopyFolder(".build", Path.Combine(repo, ".build"), true); + if (!IsLinux) + { + Exec("cmd", "/C xcopy /S/Q/I/Y .build " + Path.Combine(repo, ".build"), ""); + } + else + { + CopyFolder(".build", Path.Combine(repo, ".build"), true); + } } try @@ -802,114 +847,6 @@ functions return repositoryExclude.Split(new string[] {","}, StringSplitOptions.RemoveEmptyEntries); } - static IList> GetBuildGraph() - { - var repositoryLookup = new List(); - foreach (var repo in GetRepositoriesToBuild()) - { - var info = new RepositoryInfo { Name = repo }; - var srcDir = Path.Combine(repo, "src"); - - if (Directory.Exists(srcDir)) - { - foreach (var directory in Directory.EnumerateDirectories(srcDir)) - { - info.RepositoryNames.Add(Path.GetFileName(directory)); - var projectJson = Path.Combine(directory, "project.json"); - if (File.Exists(projectJson)) - { - GetDependencies(projectJson, info.DependencyNames); - } - } - } - - var otherDirs = new[] { "test", "samples", "tools" }; - for (var i = 0; i < otherDirs.Length; i++) - { - var otherDir = Path.Combine(repo, otherDirs[i]); - if (Directory.Exists(otherDir)) - { - foreach (var directory in Directory.EnumerateDirectories(otherDir)) - { - var projectJson = Path.Combine(directory, "project.json"); - if (File.Exists(projectJson)) - { - GetDependencies(projectJson, info.DependencyNames); - } - else - { - // Look for test\Websites\WebsiteName\project.json - foreach (var subDirectory in Directory.EnumerateDirectories(directory)) - { - projectJson = Path.Combine(subDirectory, "project.json"); - if (File.Exists(projectJson)) - { - GetDependencies(projectJson, info.DependencyNames); - } - } - } - } - } - } - - info.DependencyNames.ExceptWith(info.RepositoryNames); - repositoryLookup.Add(info); - } - - foreach (var info in repositoryLookup) - { - foreach (var item in info.DependencyNames) - { - var dependency = repositoryLookup.Find(r => r.RepositoryNames.Contains(item)); - if (dependency != null) - { - info.Dependencies.Add(dependency); - } - } - } - - return repositoryLookup.GroupBy(r => r.Order, r => r.Name).OrderBy(r => r.Key).ToArray(); - } - - static void GetDependencies(string projectJsonPath, HashSet dependencies) - { - JsonObject project; - try - { - project = (JsonObject)Json.Deserialize(File.ReadAllText(projectJsonPath)); - } - catch (Exception ex) - { - Console.Error.WriteLine("Failed to parse project at path " + projectJsonPath + Environment.NewLine + ex.Message); - throw; - } - var dependenciesNode = project.ValueAsJsonObject("dependencies"); - AddKeys(dependencies, dependenciesNode); - - var frameworkNodes = project.ValueAsJsonObject("frameworks"); - if (frameworkNodes != null) - { - foreach (var framework in frameworkNodes.Keys) - { - dependenciesNode = frameworkNodes.ValueAsJsonObject(framework).ValueAsJsonObject("dependencies"); - AddKeys(dependencies, dependenciesNode); - } - } - - AddKeys(dependencies, project.ValueAsJsonObject("tools")); - } - - static void AddKeys(HashSet target, JsonObject source) - { - if (source != null) - { - foreach (var key in source.Keys) - { - target.Add(key); - } - } - } - static string DefaultDropsShare(string value) { return value ?? (Environment.OSVersion.Platform == PlatformID.Unix ? "/aspnetci-drops" : @"\\aspnetci\drops"); @@ -933,6 +870,210 @@ functions return Path.GetFullPath(output); } + static IList> GetBuildGraph() + { + var repositoryLookup = GetRepositoriesToBuild() + .Select(RepoInfoFactory.Create) + .ToList(); + + foreach (var info in repositoryLookup) + { + foreach (var item in info.DependencyNames) + { + var dependency = repositoryLookup.Find(r => r.RepositoryNames.Contains(item)); + if (dependency != null) + { + info.Dependencies.Add(dependency); + } + } + } + + return repositoryLookup.GroupBy(r => r.Order, r => r.Name).OrderBy(r => r.Key).ToArray(); + } + + private class RepoInfoFactory + { + public static RepositoryInfo Create(string repo) + { + var info = new RepositoryInfo { Name = repo }; + ResolveProjectJsonProjects(info, repo); + ResolveMsBuildProjects(info, repo); + info.DependencyNames.ExceptWith(info.RepositoryNames); + return info; + } + + private static void ResolveMsBuildProjects(RepositoryInfo info, string repo) + { + foreach (var sln in Directory.EnumerateFiles(repo, "*.sln")) + { + var lines = File.ReadAllLines(sln); + foreach (var line in lines) + { + var match = Regex.Match(line, @"^# Visual Studio (\d+)\s?"); + if (match == null || match.Groups.Count < 2) + { + continue; + } + int version; + if (int.TryParse(match.Groups[1].Value, out version) && version >= 15) + { + ResolveMsBuildProject(info, sln); + break; + } + } + } + } + + private static void ResolveMsBuildProject(RepositoryInfo info, string projectFile) + { + var intermediate = Path.Combine(Directory.GetCurrentDirectory(), "obj"); + Directory.CreateDirectory(intermediate); + var dgJson = Path.Combine(intermediate, Path.GetFileName(projectFile) + ".graph.json"); + var psi = new ProcessStartInfo + { + UseShellExecute = false, + FileName = "dotnet", + Arguments = "msbuild \"" + projectFile + "\" /t:GenerateRestoreGraphFile \"/p:RestoreGraphOutputPath=" + dgJson + "\"" + }; + var p = Process.Start(psi); + p.WaitForExit(); + if (p.ExitCode != 0) + { + Console.WriteLine("warn: Failed to get restore graph from " + projectFile); + return; + } + + JsonObject graph; + try + { + graph = (JsonObject)Json.Deserialize(File.ReadAllText(dgJson)); + } + catch (Exception ex) + { + Console.Error.WriteLine("Failed to parse dependency graph"); + throw; + } + + var format = graph.ValueAsInt("format"); + if (format != 1) + { + Console.Error.WriteLine("Dependency graph file in unsupported format version: " + format); + throw new FormatException(); + } + + var projects = graph.ValueAsJsonObject("projects"); + foreach (var projectKey in projects.Keys) + { + var project = projects.ValueAsJsonObject(projectKey); + var restore = project.ValueAsJsonObject("restore"); + var projectName = restore.ValueAsString("projectName"); + info.RepositoryNames.Add(projectName); + + var frameworks = project.ValueAsJsonObject("frameworks"); + foreach (var fxKey in frameworks.Keys) + { + var fxDeps = frameworks.ValueAsJsonObject(fxKey).ValueAsJsonObject("dependencies"); + foreach (var depKey in fxDeps.Keys) + { + var dep = fxDeps.ValueAsJsonObject(depKey); + if (dep.ValueAsString("target") != "Package") + { + continue; + } + // todo version? + info.DependencyNames.Add(depKey); + } + } + } + } + + private static void ResolveProjectJsonProjects(RepositoryInfo info, string repo) + { + var srcDir = Path.Combine(repo, "src"); + + if (Directory.Exists(srcDir)) + { + foreach (var directory in Directory.EnumerateDirectories(srcDir)) + { + info.RepositoryNames.Add(Path.GetFileName(directory)); + var projectJson = Path.Combine(directory, "project.json"); + if (File.Exists(projectJson)) + { + ResolveDependencies(projectJson, info.DependencyNames); + } + } + } + + var otherDirs = new[] { "test", "samples", "tools" }; + for (var i = 0; i < otherDirs.Length; i++) + { + var otherDir = Path.Combine(repo, otherDirs[i]); + if (Directory.Exists(otherDir)) + { + foreach (var directory in Directory.EnumerateDirectories(otherDir)) + { + var projectJson = Path.Combine(directory, "project.json"); + if (File.Exists(projectJson)) + { + ResolveDependencies(projectJson, info.DependencyNames); + } + else + { + // Look for test\Websites\WebsiteName\project.json + foreach (var subDirectory in Directory.EnumerateDirectories(directory)) + { + projectJson = Path.Combine(subDirectory, "project.json"); + if (File.Exists(projectJson)) + { + ResolveDependencies(projectJson, info.DependencyNames); + } + } + } + } + } + } + } + + private static void ResolveDependencies(string projectJsonPath, HashSet dependencies) + { + JsonObject project; + try + { + project = (JsonObject)Json.Deserialize(File.ReadAllText(projectJsonPath)); + } + catch (Exception ex) + { + Console.Error.WriteLine("Failed to parse project at path " + projectJsonPath + Environment.NewLine + ex.Message); + throw; + } + var dependenciesNode = project.ValueAsJsonObject("dependencies"); + AddKeys(dependencies, dependenciesNode); + + var frameworkNodes = project.ValueAsJsonObject("frameworks"); + if (frameworkNodes != null) + { + foreach (var framework in frameworkNodes.Keys) + { + dependenciesNode = frameworkNodes.ValueAsJsonObject(framework).ValueAsJsonObject("dependencies"); + AddKeys(dependencies, dependenciesNode); + } + } + + AddKeys(dependencies, project.ValueAsJsonObject("tools")); + } + + private static void AddKeys(HashSet target, JsonObject source) + { + if (source != null) + { + foreach (var key in source.Keys) + { + target.Add(key); + } + } + } + } + private class RepositoryInfo { public string Name;