From 8271552cc6b9a0483de3908c8961197ed5964160 Mon Sep 17 00:00:00 2001 From: Pranav K Date: Mon, 20 Jul 2015 10:23:55 -0700 Subject: [PATCH] Change pin-version to use the coherence build results Fies #247 --- .gitignore | 1 + makefile.shade | 133 ++++++++++-------------------- tools/PinVersion.sln | 22 +++++ tools/PinVersion/PinVersion.xproj | 19 +++++ tools/PinVersion/Program.cs | 74 +++++++++++++++++ tools/PinVersion/project.json | 15 ++++ 6 files changed, 175 insertions(+), 89 deletions(-) create mode 100644 tools/PinVersion.sln create mode 100644 tools/PinVersion/PinVersion.xproj create mode 100644 tools/PinVersion/Program.cs create mode 100644 tools/PinVersion/project.json diff --git a/.gitignore b/.gitignore index 5dc98f5c69..bf70ff6f5b 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,4 @@ StyleCop.Cache node_modules *.snk .nuget/NuGet.exe +project.lock.json \ No newline at end of file diff --git a/makefile.shade b/makefile.shade index f2f46e91d1..5eae84ff43 100644 --- a/makefile.shade +++ b/makefile.shade @@ -56,7 +56,7 @@ functions "CORS", "Entropy", }; - + // Doesn't build on Mono since their contracts don't match string[] excludeReposOnMono = new[] { "Helios" }; @@ -89,9 +89,9 @@ var buildTarget = "compile" nuget-pack nuspecFile='${Path.Combine(BASE_DIR, "KoreBuild.nuspec")}' packageVersion='${VERSION}' outputDir='${TARGET_DIR}' #pack-install .pack - nuget-local-publish sourcePackagesDir='${TARGET_DIR}' + nuget-local-publish sourcePackagesDir='${TARGET_DIR}' -#git-pull target='pull' +#git-pull target='pull' @{ Parallel.ForEach(repos, repo => { @@ -177,7 +177,7 @@ var buildTarget = "compile" foreach (var repo in GetAllRepos()) { CloneOrUpdate(repo); - + GitCommand(repo, "checkout origin/dev -B release"); if(!ShouldSkipCopyingNugetConfig(repo)) @@ -223,82 +223,37 @@ var buildTarget = "compile" -// Pin versions of packages in project.json and updated project.lock.json -// More information https://github.com/aspnet/Universe/wiki/%23pin-version-:-Pinning-package-version-for-a-particular-release-in-project.json @{ - // NOTE: Before running update the below monikers as needed - var branch = Environment.GetEnvironmentVariable("BUILD_BRANCH"); - var stableBranches = new[] {"master"}; - - var oldPreRelease = Environment.GetEnvironmentVariable("OLD_RELEASE"); - var newPreRelease = Environment.GetEnvironmentVariable("NEW_RELEASE"); - var excludeReposForJson = new[] - { - "Coherence", - "Coherence-Signed", - "dnvm", - "Entropy" - }; - - var oldCoreCLRBuild = Environment.GetEnvironmentVariable("CORECLR_RELEASE_BUILD_OLD"); - var newCoreCLRBuild = Environment.GetEnvironmentVariable("CORECLR_RELEASE_BUILD_NEW"); - var oldCoreCLRPrerelease = Environment.GetEnvironmentVariable("OLD_CORECLR_RELEASE"); - var newCoreCLRPrerelease = Environment.GetEnvironmentVariable("NEW_CORECLR_RELEASE"); - - var oldRoslynPrerelease = Environment.GetEnvironmentVariable("OLD_ROSLYN_RELEASE"); - var newRoslynPrerelease = Environment.GetEnvironmentVariable("NEW_ROSLYN_RELEASE"); - - foreach (var repo in GetAllRepos().Except(excludeReposForJson)) + var coherenceFeed = Environment.GetEnvironmentVariable("COHERENCE_FEED"); + if (string.IsNullOrEmpty(coherenceFeed)) { - CloneOrUpdate(repo); + throw new Exception("COHERENCE_FEED not specified. Usually this is Packages-NoTimestamp directory of Coherence-Signed."); + } - GitCommand(repo, string.Format("checkout {0}", branch)); + var excludeReposForJson = new[] + { + "Coherence", + "Coherence-Signed", + "dnvm", + "Entropy" + }; - if(File.Exists(Path.Combine(repo, "build.cmd"))) + var repos = GetAllRepos().Except(excludeReposForJson); + Parallel.ForEach(repos, CloneOrUpdate); + + foreach (var repo in repos) + { + GitCommand(repo, "checkout master"); + + if (File.Exists(Path.Combine(repo, "build.cmd"))) { File.Copy(Path.Combine("build-template", "build.cmd"), Path.Combine(repo, "build.cmd"), overwrite: true); } - // Process project.json files to pin version numbers - // Replace version in project.json - foreach(var file in Directory.GetFiles(repo,"project.json", SearchOption.AllDirectories)) - { - var builder = new StringBuilder(); - var contents = File.ReadAllLines(file); + var repoPath = Path.Combine(Directory.GetCurrentDirectory(), repo); - foreach(var jsonEntry in contents) - { - if(jsonEntry.Contains("\"System.") || jsonEntry.Contains("CSharp")) - { - var oldSearchExp = BuildVersionExpression(branch, oldCoreCLRPrerelease, oldCoreCLRBuild); - var newReplaceExp = BuildVersionExpression(branch, newCoreCLRPrerelease, newCoreCLRBuild); - - builder.AppendLine(jsonEntry.Replace(oldSearchExp, newReplaceExp)); - } - else if(jsonEntry.Contains("CodeAnalysis.")) - { - var oldSearchExp = BuildVersionExpression(branch, oldRoslynPrerelease); - var newReplaceExp = BuildVersionExpression(branch, newRoslynPrerelease); - - builder.AppendLine(jsonEntry.Replace(oldSearchExp, newReplaceExp)); - } - else if(jsonEntry.Contains("IdentityModel")) - { - // IdentityModel packages don't have a stable build and are updated very frequently on the NuGet feed - builder.AppendLine(jsonEntry); - } - else - { - var oldSearchExp = BuildVersionExpression(branch, oldPreRelease); - var newReplaceExp = BuildVersionExpression(branch, newPreRelease); - - var newEntry = ProcessExcludesForPackages(jsonEntry, stableBranches.Contains(branch)); - - builder.AppendLine(newEntry.Replace(oldSearchExp, newReplaceExp)); - } - } - - File.WriteAllText(file, builder.ToString()); - } + Exec("dnx", string.Format(@".\tools\PinVersion run ""{0}"" ""{1}""", repo, coherenceFeed), string.Empty); // Build projects to produce project.lock.json Exec("build.cmd", "compile", repo); @@ -321,11 +276,11 @@ var buildTarget = "compile" { GitCommand(repo, string.Format("add -f {0}", gitFilePath)); } - } + } - GitCommand(repo, "commit -am \"Updating json files to pin versions and build.cmd to pin KoreBuild and DNX\""); + GitCommand(repo, "commit -am \"Updating json files to pin versions and build.cmd to pin KoreBuild and DNX\""); - GitCommand(repo, string.Format("push origin {0}:{0} -f", branch)); + GitCommand(repo, "push origin master:master -f"); } } @@ -380,8 +335,8 @@ var buildTarget = "compile" @{ var failed = new Dictionary(); var skipped = new List(); - - foreach(var repo in repos) + + foreach(var repo in repos) { var blockName = string.Format("Building {0}", repo); if (IsTeamCity) @@ -391,7 +346,7 @@ var buildTarget = "compile" try { Log.Info(blockName); - + if (SKIP_NO_CREDENTIALS) { if (!Directory.Exists(repo)) @@ -411,7 +366,7 @@ var buildTarget = "compile" { Exec("build.cmd", buildTarget, repo); } - + Log.Info(string.Format("Build {0} succeeded", repo)); } catch(Exception ex) @@ -419,7 +374,7 @@ var buildTarget = "compile" Log.Warn(string.Format("Build {0} failed: {1}", repo, ex.Message)); failed[repo] = ex; } - finally + finally { if (IsTeamCity) { @@ -451,7 +406,7 @@ var buildTarget = "compile" Log.Info(string.Format("Build {0} succeeded", repo)); } } - + if (failed.Any()) { throw new Exception("One or more repos failed to build"); @@ -460,7 +415,7 @@ var buildTarget = "compile" #only-install target='install' @{ - foreach(var repo in repos) + foreach(var repo in repos) { if (IsMono) { @@ -472,7 +427,7 @@ var buildTarget = "compile" } } } - + #run-snapshot-manager @{ Exec(@".nuget\nuget.exe", @"restore -out packages tools\TCDependencyManager\packages.config", ""); @@ -484,7 +439,7 @@ var buildTarget = "compile" #git-status description='Show status of repos known by Universe' @{ - foreach(var repo in repos) + foreach(var repo in repos) { GitStatus(repo); } @@ -499,7 +454,7 @@ var buildTarget = "compile" { throw new Exception("git-clean cancelled"); } - foreach(var repo in repos) + foreach(var repo in repos) { GitClean(repo); } @@ -590,7 +545,7 @@ functions return url.StartsWith("https://", System.StringComparison.OrdinalIgnoreCase); } - + static bool IsAccessible(string key) { var req = WebRequest.CreateHttp("https://github.com/aspnet/" + key); @@ -601,12 +556,12 @@ functions } catch (WebException ex) { - if (ex.Response != null && + if (ex.Response != null && ((HttpWebResponse)ex.Response).StatusCode == HttpStatusCode.NotFound) { return false; } - + // Ignore any other exception. They should surface as part of git clone with well-known messages. } return true; @@ -646,7 +601,7 @@ functions if (useHttps && !IsAccessible(repo)) { - if (SKIP_NO_CREDENTIALS) + if (SKIP_NO_CREDENTIALS) { // This is used on the XPlat CI. Must return because otherwise git will wait for user // input and hang the build @@ -687,7 +642,7 @@ functions .Replace("]", "|]"); } - // Currently there are two packages that need to be special cased. + // Currently there are two packages that need to be special cased. // TODO: Revisit as this is changed in the near future. string ProcessExcludesForPackages(string jsonEntry, bool stable=false) { @@ -710,7 +665,7 @@ functions builder.Append(preRelease); } - // If buildnumber is provided, append. + // If buildnumber is provided, append. // This for CORE CLR packages if(!String.IsNullOrEmpty(buildNumber)) { diff --git a/tools/PinVersion.sln b/tools/PinVersion.sln new file mode 100644 index 0000000000..654e556bc6 --- /dev/null +++ b/tools/PinVersion.sln @@ -0,0 +1,22 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 14 +VisualStudioVersion = 14.0.23107.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "PinVersion", "PinVersion\PinVersion.xproj", "{3D3B5750-0C24-4F1C-9304-B5E4D85CF5A0}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {3D3B5750-0C24-4F1C-9304-B5E4D85CF5A0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3D3B5750-0C24-4F1C-9304-B5E4D85CF5A0}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3D3B5750-0C24-4F1C-9304-B5E4D85CF5A0}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3D3B5750-0C24-4F1C-9304-B5E4D85CF5A0}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/tools/PinVersion/PinVersion.xproj b/tools/PinVersion/PinVersion.xproj new file mode 100644 index 0000000000..98a40175b2 --- /dev/null +++ b/tools/PinVersion/PinVersion.xproj @@ -0,0 +1,19 @@ + + + + 14.0 + $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion) + + + + 3d3b5750-0c24-4f1c-9304-b5e4d85cf5a0 + PinVersion + ..\artifacts\obj\$(MSBuildProjectName) + ..\artifacts\bin\$(MSBuildProjectName)\ + + + + 2.0 + + + \ No newline at end of file diff --git a/tools/PinVersion/Program.cs b/tools/PinVersion/Program.cs new file mode 100644 index 0000000000..5b2117b299 --- /dev/null +++ b/tools/PinVersion/Program.cs @@ -0,0 +1,74 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.IO; +using System.Linq; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using NuGet; + +namespace PinVersion +{ + public class Program + { + public void Main(string[] args) + { + if (args.Length != 2) + { + Console.Error.WriteLine("Usage "); + } + + var repositoryPath = args[0]; + var packageSource = args[1]; + + var packageRepository = PackageRepositoryFactory.Default.CreateRepository(packageSource); + + foreach (var file in Directory.EnumerateFiles(repositoryPath, "project.json", SearchOption.AllDirectories)) + { + var projectJson = JObject.Parse(File.ReadAllText(file)); + var directoryName = Path.GetFileName(Path.GetDirectoryName(file)); + var latestPackage = packageRepository.FindPackage(directoryName); + if (latestPackage != null) + { + ((JValue)projectJson["version"]).Value = latestPackage.Version.ToNormalizedString(); + } + + var frameworkDependencies = projectJson["frameworks"] + ?.Cast() + ?.Select(f => ((JObject)f.Value)["dependencies"]) + ?? Enumerable.Empty(); + var dependencies = Enumerable.Concat(new[] { projectJson["dependencies"] }, frameworkDependencies) + .Where(d => d != null) + .SelectMany(d => d) + .Cast(); + + foreach (var dependency in dependencies) + { + latestPackage = packageRepository.FindPackage(dependency.Name); + if (latestPackage != null) + { + if (dependency.Value.Type == JTokenType.Object) + { + // "key": { "version": "1.0.0-*", "type": "build" } + var value = (JObject)dependency.Value; + value["version"] = latestPackage.Version.ToNormalizedString(); + } + else + { + // "key": "version" + dependency.Value = latestPackage.Version.ToNormalizedString(); + } + } + } + + using (var fileWriter = new JsonTextWriter(new StreamWriter(file))) + { + fileWriter.Formatting = Formatting.Indented; + fileWriter.Indentation = 4; + projectJson.WriteTo(fileWriter); + } + } + } + } +} diff --git a/tools/PinVersion/project.json b/tools/PinVersion/project.json new file mode 100644 index 0000000000..f781996852 --- /dev/null +++ b/tools/PinVersion/project.json @@ -0,0 +1,15 @@ +{ + "version": "1.0.0-*", + "dependencies": { + "Newtonsoft.Json": "6.0.8", + "NuGet.Core": "2.8.6" + }, + + "commands": { + "PinVersion": "PinVersion" + }, + + "frameworks": { + "dnx451": { } + } +}