diff --git a/makefile.shade b/makefile.shade index ed2de380ce..67309489ce 100644 --- a/makefile.shade +++ b/makefile.shade @@ -136,6 +136,15 @@ var repos='${new Dictionary { Exec("build.cmd", "install", repo.Key); } } + +#run-snapshot-manager + @{ + Exec(@".nuget\nuget.exe", @"restore -out packages tools\TCDependencyManager\packages.config", ""); + var programFiles = Environment.GetFolderPath(Environment.SpecialFolder.ProgramFilesX86); + var msbuildPath = Path.Combine(programFiles, "MSBuild", "12.0", "Bin", "MsBuild.exe"); + Exec(msbuildPath, "TCDependencyManager.csproj", @"tools\TCDependencyManager"); + Exec(@"tools\TCDependencyManager\bin\Debug\TCDependencyManager.exe", "", ""); + } macro name='GitPull' gitUri='string' gitBranch='string' gitFolder='string' git-pull diff --git a/tools/TCDependencyManager/App.config b/tools/TCDependencyManager/App.config new file mode 100644 index 0000000000..317956653f --- /dev/null +++ b/tools/TCDependencyManager/App.config @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/tools/TCDependencyManager/GitHubAPI.cs b/tools/TCDependencyManager/GitHubAPI.cs new file mode 100644 index 0000000000..d8cbd6c5a9 --- /dev/null +++ b/tools/TCDependencyManager/GitHubAPI.cs @@ -0,0 +1,97 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; + +namespace TCDependencyManager +{ + public class GitHubAPI + { + private const string BaseUrl = "https://api.github.com/"; + private readonly string _oauthToken; + + public GitHubAPI(string oauthToken) + { + _oauthToken = oauthToken; + } + + public List GetRepos() + { + using (var client = GetClient()) + { + var response = client.GetAsync("orgs/aspnet/repos?page=1&per_page=100").Result; + return response.EnsureSuccessStatusCode() + .Content + .ReadAsAsync>().Result; + } + } + + public List GetProjects(Repository repo) + { + IEnumerable projectNames = null; + using (var client = GetClient()) + { + string path = string.Format("/repos/aspnet/{0}/contents/src?ref=dev", repo.Name); + var response = client.GetAsync(path).Result; + if (response.IsSuccessStatusCode) + { + var result = response.Content.ReadAsAsync().Result; + projectNames = result.Select(r => r["name"].Value()); + } + else + { + projectNames = Enumerable.Empty(); + } + } + return projectNames + .AsParallel() + .Select(p => new Project + { + Repo = repo, + ProjectName = p, + Dependencies = ReadDependencies(repo, p) + }) + .ToList(); + } + + private List ReadDependencies(Repository repo, string project) + { + using (var client = GetClient()) + { + string path = string.Format("/repos/aspnet/{0}/contents/src/{1}/project.json?ref=dev", repo.Name, project); + var response = client.GetAsync(path).Result; + if (response.IsSuccessStatusCode) + { + var result = response.Content.ReadAsAsync().Result; + var content = JsonConvert.DeserializeObject( + Encoding.UTF8.GetString( + Convert.FromBase64String(result["content"].Value()))); + var dependencies = (JObject)content["dependencies"]; + if (dependencies != null) + { + return dependencies.Cast() + .Where(prop => !String.IsNullOrEmpty(prop.Value.Value())) + .Select(prop => prop.Name) + .ToList(); + } + } + } + return new List(0); + } + + private HttpClient GetClient() + { + var client = new HttpClient + { + BaseAddress = new Uri(BaseUrl) + }; + client.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("AspNet-CI", "1.0")); + client.DefaultRequestHeaders.Add("Authorization", "token " + _oauthToken); + return client; + } + } +} diff --git a/tools/TCDependencyManager/Models/Projects.cs b/tools/TCDependencyManager/Models/Projects.cs new file mode 100644 index 0000000000..9b608eba4e --- /dev/null +++ b/tools/TCDependencyManager/Models/Projects.cs @@ -0,0 +1,13 @@ +using System.Collections.Generic; + +namespace TCDependencyManager +{ + public class Project + { + public Repository Repo { get; set; } + + public string ProjectName { get; set; } + + public List Dependencies { get; set; } + } +} diff --git a/tools/TCDependencyManager/Models/Repository.cs b/tools/TCDependencyManager/Models/Repository.cs new file mode 100644 index 0000000000..fbe3eedb05 --- /dev/null +++ b/tools/TCDependencyManager/Models/Repository.cs @@ -0,0 +1,15 @@ +using System.Collections.Generic; + +namespace TCDependencyManager +{ + public class Repository + { + private readonly HashSet _dependencies = new HashSet(); + + public string Id { get; set; } + + public string Name { get; set; } + + public HashSet Dependencies { get { return _dependencies; } } + } +} diff --git a/tools/TCDependencyManager/Models/SnapshotDependency.cs b/tools/TCDependencyManager/Models/SnapshotDependency.cs new file mode 100644 index 0000000000..117dbd0622 --- /dev/null +++ b/tools/TCDependencyManager/Models/SnapshotDependency.cs @@ -0,0 +1,54 @@ +using System.Collections.Generic; +using Newtonsoft.Json; + +namespace TCDependencyManager +{ + public class SnapshotDependencies + { + public int Count { get; set; } + + [JsonProperty("snapshot-dependency")] + public List Dependencies { get; set; } + } + + public class SnapshotDepedency + { + public string Id { get; set; } + + public string Type { get; set; } + + public Properties Properties { get; set; } + + [JsonProperty("source-buildType")] + public BuildType BuildType { get; set; } + } + + public class Properties + { + public List Property { get; set; } + } + + public class BuildType + { + public string Id { get; set; } + + public string Name { get; set; } + + public string ProjectId { get; set; } + + public string ProjectName { get; set; } + } + + public class NameValuePair + { + public NameValuePair(string name, string value) + { + Name = name; + Value = value; + } + + public string Name { get; set; } + + public string Value { get; set; } + } +} diff --git a/tools/TCDependencyManager/Program.cs b/tools/TCDependencyManager/Program.cs new file mode 100644 index 0000000000..bd49583760 --- /dev/null +++ b/tools/TCDependencyManager/Program.cs @@ -0,0 +1,77 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.Contracts; +using System.Linq; +using System.Net; + +namespace TCDependencyManager +{ + class Program + { + private static readonly string[] _excludedRepos = new[] { "xunit", "kruntime", "coreclr", "universe" }; + + static int Main(string[] args) + { + var teamCityUrl = GetEnv("TEAMCITY_SERVERURL"); + var teamCityUser = GetEnv("TEAMCITY_USER"); + var teamCityPass = GetEnv("TEAMCITY_PASSWORD"); + var githubCreds = GetEnv("GITHUB_CREDS"); + + var teamCity = new TeamCityAPI(teamCityUrl, + new NetworkCredential(teamCityUser, teamCityPass)); + + var gitHub = new GitHubAPI(githubCreds); + + Console.WriteLine("Listing GitHub repos"); + var repos = gitHub.GetRepos() + .Where(repo => !_excludedRepos.Contains(repo.Name, StringComparer.OrdinalIgnoreCase)) + .ToList(); + + Console.WriteLine("Listing projects under repos"); + var projects = repos.AsParallel() + .SelectMany(repo => gitHub.GetProjects(repo)) + .ToList(); + + + Console.WriteLine("Creating dependency tree"); + MapRepoDependencies(projects); + + Console.WriteLine("Ensuring depndencies are consistent on TeamCity"); + foreach (var repo in repos.Where(p => p.Dependencies.Any())) + { + teamCity.EnsureDependencies(repo.Name, repo.Dependencies.Select(r => r.Name)); + } + return 0; + } + + private static void MapRepoDependencies(List projects) + { + var projectLookup = projects.ToDictionary(project => project.ProjectName, StringComparer.OrdinalIgnoreCase); + + foreach (var project in projects) + { + foreach (var dependency in project.Dependencies) + { + Project dependencyProject; + if (projectLookup.TryGetValue(dependency, out dependencyProject) && + project.Repo != dependencyProject.Repo) + { + project.Repo.Dependencies.Add(dependencyProject.Repo); + } + } + + } + } + + private static string GetEnv(string key) + { + var envValue = Environment.GetEnvironmentVariable(key); + if (String.IsNullOrEmpty(envValue)) + { + throw new ArgumentNullException(key); + } + return envValue; + } + + } +} diff --git a/tools/TCDependencyManager/Properties/AssemblyInfo.cs b/tools/TCDependencyManager/Properties/AssemblyInfo.cs new file mode 100644 index 0000000000..96d743a4e1 --- /dev/null +++ b/tools/TCDependencyManager/Properties/AssemblyInfo.cs @@ -0,0 +1,36 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("TCDependencyManager")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("TCDependencyManager")] +[assembly: AssemblyCopyright("Copyright © 2014")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("9645f4a8-75b5-46ad-8539-7d9c5f61c4c9")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/tools/TCDependencyManager/TCDependencyManager.csproj b/tools/TCDependencyManager/TCDependencyManager.csproj new file mode 100644 index 0000000000..0a0f024d71 --- /dev/null +++ b/tools/TCDependencyManager/TCDependencyManager.csproj @@ -0,0 +1,73 @@ + + + + + Debug + AnyCPU + {72C96182-352E-44EC-B157-AFEBDC7A74DD} + Exe + Properties + TCDependencyManager + TCDependencyManager + v4.5.1 + 512 + true + + + AnyCPU + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + AnyCPU + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + ..\..\packages\Newtonsoft.Json.4.5.11\lib\net40\Newtonsoft.Json.dll + + + + + + False + ..\..\packages\Microsoft.AspNet.WebApi.Client.5.1.1\lib\net45\System.Net.Http.Formatting.dll + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tools/TCDependencyManager/TeamCityAPI.cs b/tools/TCDependencyManager/TeamCityAPI.cs new file mode 100644 index 0000000000..0778ed05d7 --- /dev/null +++ b/tools/TCDependencyManager/TeamCityAPI.cs @@ -0,0 +1,118 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using Newtonsoft.Json; +using Newtonsoft.Json.Serialization; + +namespace TCDependencyManager +{ + public class TeamCityAPI + { + private readonly string _teamCityUrl; + private readonly ICredentials _creds; + + public TeamCityAPI(string teamCityUrl, ICredentials creds) + { + _teamCityUrl = teamCityUrl; + _creds = creds; + } + + public bool TryGetDependencies(string configId, out List dependencies) + { + string url = String.Format("httpAuth/app/rest/buildTypes/{0}/snapshot-dependencies", configId); + var client = GetClient(); + var response = client.GetAsync(url).Result; + if (response.StatusCode == HttpStatusCode.NotFound) + { + // We don't have the config setup on the CI. That is ok. + dependencies = null; + return false; + } + dependencies = response.EnsureSuccessStatusCode() + .Content.ReadAsAsync() + .Result + .Dependencies.Select(f => f.Id) + .ToList(); + return true; + } + + public void SetDependencies(string configId, IEnumerable dependencies) + { + foreach (var dependencyId in dependencies) + { + Console.WriteLine("For {0} adding: {1}", configId, dependencyId); + + string url = String.Format("httpAuth/app/rest/buildTypes/{0}/snapshot-dependencies", configId); + var client = GetClient(); + var props = new Properties + { + Property = new List + { + new NameValuePair("run-build-if-dependency-failed", "true"), + new NameValuePair("take-successful-builds-only", "true"), + new NameValuePair("take-started-build-with-same-revisions", "true") + } + }; + + var snapshotDependency = new SnapshotDepedency + { + Id = dependencyId, + Type = "snapshot_dependency", + Properties = props, + BuildType = new BuildType + { + Id = dependencyId, + Name = dependencyId, + ProjectId = "AspNet", + ProjectName = "AspNet" + } + }; + var serialized = JsonConvert.SerializeObject(snapshotDependency, new JsonSerializerSettings { ContractResolver = new CamelCasePropertyNamesContractResolver() }); + var content = new StringContent(serialized); + content.Headers.ContentType = new MediaTypeHeaderValue("application/json"); + var response = client.PostAsync(url, content).Result; + response.EnsureSuccessStatusCode(); + } + } + + private static string NormalizeId(string dependencyId) + { + return dependencyId.Replace(".", ""); + } + + public void EnsureDependencies(string configId, IEnumerable dependencies) + { + List currentDepenencies; + if (TryGetDependencies(configId, out currentDepenencies)) + { + var dependenciesToAdd = dependencies.Select(NormalizeId) + .Except(currentDepenencies, StringComparer.OrdinalIgnoreCase); + + SetDependencies(configId, dependenciesToAdd); + } + } + + private HttpClient GetClient() + { + var handler = new HttpClientHandler + { + PreAuthenticate = true, + Credentials = _creds + }; + + var client = new HttpClient(handler) + { + BaseAddress = new Uri(_teamCityUrl) + }; + client.DefaultRequestHeaders.Accept.Clear(); + client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); + + return client; + } + + + } +} diff --git a/tools/TCDependencyManager/packages.config b/tools/TCDependencyManager/packages.config new file mode 100644 index 0000000000..28c2793ab3 --- /dev/null +++ b/tools/TCDependencyManager/packages.config @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file