Adding tooling to maintain snapshot dependencies
This commit is contained in:
Pranav K
parent
84c2e7cffe
commit
1d5b5f4f62
|
|
@ -136,6 +136,15 @@ var repos='${new Dictionary<string,string> {
|
|||
Exec("build.cmd", "install", repo.Key);
|
||||
}
|
||||
}
|
||||
|
||||
#run-snapshot-manager
|
||||
@{
|
||||
Exec(@".nuget\nuget.exe", @"restore -out packages tools\TCDependencyManager\packages.config", "");
|
||||
var programFiles = Environment.GetFolderPath(Environment.SpecialFolder.ProgramFilesX86);
|
||||
var msbuildPath = Path.Combine(programFiles, "MSBuild", "12.0", "Bin", "MsBuild.exe");
|
||||
Exec(msbuildPath, "TCDependencyManager.csproj", @"tools\TCDependencyManager");
|
||||
Exec(@"tools\TCDependencyManager\bin\Debug\TCDependencyManager.exe", "", "");
|
||||
}
|
||||
|
||||
macro name='GitPull' gitUri='string' gitBranch='string' gitFolder='string'
|
||||
git-pull
|
||||
|
|
|
|||
|
|
@ -0,0 +1,3 @@
|
|||
<configuration>
|
||||
|
||||
</configuration>
|
||||
|
|
@ -0,0 +1,97 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Net.Http;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Text;
|
||||
using Newtonsoft.Json;
|
||||
using Newtonsoft.Json.Linq;
|
||||
|
||||
namespace TCDependencyManager
|
||||
{
|
||||
public class GitHubAPI
|
||||
{
|
||||
private const string BaseUrl = "https://api.github.com/";
|
||||
private readonly string _oauthToken;
|
||||
|
||||
public GitHubAPI(string oauthToken)
|
||||
{
|
||||
_oauthToken = oauthToken;
|
||||
}
|
||||
|
||||
public List<Repository> GetRepos()
|
||||
{
|
||||
using (var client = GetClient())
|
||||
{
|
||||
var response = client.GetAsync("orgs/aspnet/repos?page=1&per_page=100").Result;
|
||||
return response.EnsureSuccessStatusCode()
|
||||
.Content
|
||||
.ReadAsAsync<List<Repository>>().Result;
|
||||
}
|
||||
}
|
||||
|
||||
public List<Project> GetProjects(Repository repo)
|
||||
{
|
||||
IEnumerable<string> projectNames = null;
|
||||
using (var client = GetClient())
|
||||
{
|
||||
string path = string.Format("/repos/aspnet/{0}/contents/src?ref=dev", repo.Name);
|
||||
var response = client.GetAsync(path).Result;
|
||||
if (response.IsSuccessStatusCode)
|
||||
{
|
||||
var result = response.Content.ReadAsAsync<JArray>().Result;
|
||||
projectNames = result.Select(r => r["name"].Value<string>());
|
||||
}
|
||||
else
|
||||
{
|
||||
projectNames = Enumerable.Empty<string>();
|
||||
}
|
||||
}
|
||||
return projectNames
|
||||
.AsParallel()
|
||||
.Select(p => new Project
|
||||
{
|
||||
Repo = repo,
|
||||
ProjectName = p,
|
||||
Dependencies = ReadDependencies(repo, p)
|
||||
})
|
||||
.ToList();
|
||||
}
|
||||
|
||||
private List<string> ReadDependencies(Repository repo, string project)
|
||||
{
|
||||
using (var client = GetClient())
|
||||
{
|
||||
string path = string.Format("/repos/aspnet/{0}/contents/src/{1}/project.json?ref=dev", repo.Name, project);
|
||||
var response = client.GetAsync(path).Result;
|
||||
if (response.IsSuccessStatusCode)
|
||||
{
|
||||
var result = response.Content.ReadAsAsync<JObject>().Result;
|
||||
var content = JsonConvert.DeserializeObject<JObject>(
|
||||
Encoding.UTF8.GetString(
|
||||
Convert.FromBase64String(result["content"].Value<string>())));
|
||||
var dependencies = (JObject)content["dependencies"];
|
||||
if (dependencies != null)
|
||||
{
|
||||
return dependencies.Cast<JProperty>()
|
||||
.Where(prop => !String.IsNullOrEmpty(prop.Value.Value<string>()))
|
||||
.Select(prop => prop.Name)
|
||||
.ToList();
|
||||
}
|
||||
}
|
||||
}
|
||||
return new List<string>(0);
|
||||
}
|
||||
|
||||
private HttpClient GetClient()
|
||||
{
|
||||
var client = new HttpClient
|
||||
{
|
||||
BaseAddress = new Uri(BaseUrl)
|
||||
};
|
||||
client.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("AspNet-CI", "1.0"));
|
||||
client.DefaultRequestHeaders.Add("Authorization", "token " + _oauthToken);
|
||||
return client;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
using System.Collections.Generic;
|
||||
|
||||
namespace TCDependencyManager
|
||||
{
|
||||
public class Project
|
||||
{
|
||||
public Repository Repo { get; set; }
|
||||
|
||||
public string ProjectName { get; set; }
|
||||
|
||||
public List<string> Dependencies { get; set; }
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
using System.Collections.Generic;
|
||||
|
||||
namespace TCDependencyManager
|
||||
{
|
||||
public class Repository
|
||||
{
|
||||
private readonly HashSet<Repository> _dependencies = new HashSet<Repository>();
|
||||
|
||||
public string Id { get; set; }
|
||||
|
||||
public string Name { get; set; }
|
||||
|
||||
public HashSet<Repository> Dependencies { get { return _dependencies; } }
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,54 @@
|
|||
using System.Collections.Generic;
|
||||
using Newtonsoft.Json;
|
||||
|
||||
namespace TCDependencyManager
|
||||
{
|
||||
public class SnapshotDependencies
|
||||
{
|
||||
public int Count { get; set; }
|
||||
|
||||
[JsonProperty("snapshot-dependency")]
|
||||
public List<SnapshotDepedency> Dependencies { get; set; }
|
||||
}
|
||||
|
||||
public class SnapshotDepedency
|
||||
{
|
||||
public string Id { get; set; }
|
||||
|
||||
public string Type { get; set; }
|
||||
|
||||
public Properties Properties { get; set; }
|
||||
|
||||
[JsonProperty("source-buildType")]
|
||||
public BuildType BuildType { get; set; }
|
||||
}
|
||||
|
||||
public class Properties
|
||||
{
|
||||
public List<NameValuePair> Property { get; set; }
|
||||
}
|
||||
|
||||
public class BuildType
|
||||
{
|
||||
public string Id { get; set; }
|
||||
|
||||
public string Name { get; set; }
|
||||
|
||||
public string ProjectId { get; set; }
|
||||
|
||||
public string ProjectName { get; set; }
|
||||
}
|
||||
|
||||
public class NameValuePair
|
||||
{
|
||||
public NameValuePair(string name, string value)
|
||||
{
|
||||
Name = name;
|
||||
Value = value;
|
||||
}
|
||||
|
||||
public string Name { get; set; }
|
||||
|
||||
public string Value { get; set; }
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,77 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics.Contracts;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
|
||||
namespace TCDependencyManager
|
||||
{
|
||||
class Program
|
||||
{
|
||||
private static readonly string[] _excludedRepos = new[] { "xunit", "kruntime", "coreclr", "universe" };
|
||||
|
||||
static int Main(string[] args)
|
||||
{
|
||||
var teamCityUrl = GetEnv("TEAMCITY_SERVERURL");
|
||||
var teamCityUser = GetEnv("TEAMCITY_USER");
|
||||
var teamCityPass = GetEnv("TEAMCITY_PASSWORD");
|
||||
var githubCreds = GetEnv("GITHUB_CREDS");
|
||||
|
||||
var teamCity = new TeamCityAPI(teamCityUrl,
|
||||
new NetworkCredential(teamCityUser, teamCityPass));
|
||||
|
||||
var gitHub = new GitHubAPI(githubCreds);
|
||||
|
||||
Console.WriteLine("Listing GitHub repos");
|
||||
var repos = gitHub.GetRepos()
|
||||
.Where(repo => !_excludedRepos.Contains(repo.Name, StringComparer.OrdinalIgnoreCase))
|
||||
.ToList();
|
||||
|
||||
Console.WriteLine("Listing projects under repos");
|
||||
var projects = repos.AsParallel()
|
||||
.SelectMany(repo => gitHub.GetProjects(repo))
|
||||
.ToList();
|
||||
|
||||
|
||||
Console.WriteLine("Creating dependency tree");
|
||||
MapRepoDependencies(projects);
|
||||
|
||||
Console.WriteLine("Ensuring depndencies are consistent on TeamCity");
|
||||
foreach (var repo in repos.Where(p => p.Dependencies.Any()))
|
||||
{
|
||||
teamCity.EnsureDependencies(repo.Name, repo.Dependencies.Select(r => r.Name));
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
private static void MapRepoDependencies(List<Project> projects)
|
||||
{
|
||||
var projectLookup = projects.ToDictionary(project => project.ProjectName, StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
foreach (var project in projects)
|
||||
{
|
||||
foreach (var dependency in project.Dependencies)
|
||||
{
|
||||
Project dependencyProject;
|
||||
if (projectLookup.TryGetValue(dependency, out dependencyProject) &&
|
||||
project.Repo != dependencyProject.Repo)
|
||||
{
|
||||
project.Repo.Dependencies.Add(dependencyProject.Repo);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
private static string GetEnv(string key)
|
||||
{
|
||||
var envValue = Environment.GetEnvironmentVariable(key);
|
||||
if (String.IsNullOrEmpty(envValue))
|
||||
{
|
||||
throw new ArgumentNullException(key);
|
||||
}
|
||||
return envValue;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
using System.Reflection;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
// General Information about an assembly is controlled through the following
|
||||
// set of attributes. Change these attribute values to modify the information
|
||||
// associated with an assembly.
|
||||
[assembly: AssemblyTitle("TCDependencyManager")]
|
||||
[assembly: AssemblyDescription("")]
|
||||
[assembly: AssemblyConfiguration("")]
|
||||
[assembly: AssemblyCompany("")]
|
||||
[assembly: AssemblyProduct("TCDependencyManager")]
|
||||
[assembly: AssemblyCopyright("Copyright © 2014")]
|
||||
[assembly: AssemblyTrademark("")]
|
||||
[assembly: AssemblyCulture("")]
|
||||
|
||||
// Setting ComVisible to false makes the types in this assembly not visible
|
||||
// to COM components. If you need to access a type in this assembly from
|
||||
// COM, set the ComVisible attribute to true on that type.
|
||||
[assembly: ComVisible(false)]
|
||||
|
||||
// The following GUID is for the ID of the typelib if this project is exposed to COM
|
||||
[assembly: Guid("9645f4a8-75b5-46ad-8539-7d9c5f61c4c9")]
|
||||
|
||||
// Version information for an assembly consists of the following four values:
|
||||
//
|
||||
// Major Version
|
||||
// Minor Version
|
||||
// Build Number
|
||||
// Revision
|
||||
//
|
||||
// You can specify all the values or you can default the Build and Revision Numbers
|
||||
// by using the '*' as shown below:
|
||||
// [assembly: AssemblyVersion("1.0.*")]
|
||||
[assembly: AssemblyVersion("1.0.0.0")]
|
||||
[assembly: AssemblyFileVersion("1.0.0.0")]
|
||||
|
|
@ -0,0 +1,73 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
|
||||
<Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
|
||||
<PropertyGroup>
|
||||
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
|
||||
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
|
||||
<ProjectGuid>{72C96182-352E-44EC-B157-AFEBDC7A74DD}</ProjectGuid>
|
||||
<OutputType>Exe</OutputType>
|
||||
<AppDesignerFolder>Properties</AppDesignerFolder>
|
||||
<RootNamespace>TCDependencyManager</RootNamespace>
|
||||
<AssemblyName>TCDependencyManager</AssemblyName>
|
||||
<TargetFrameworkVersion>v4.5.1</TargetFrameworkVersion>
|
||||
<FileAlignment>512</FileAlignment>
|
||||
<AutoGenerateBindingRedirects>true</AutoGenerateBindingRedirects>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
|
||||
<PlatformTarget>AnyCPU</PlatformTarget>
|
||||
<DebugSymbols>true</DebugSymbols>
|
||||
<DebugType>full</DebugType>
|
||||
<Optimize>false</Optimize>
|
||||
<OutputPath>bin\Debug\</OutputPath>
|
||||
<DefineConstants>DEBUG;TRACE</DefineConstants>
|
||||
<ErrorReport>prompt</ErrorReport>
|
||||
<WarningLevel>4</WarningLevel>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
|
||||
<PlatformTarget>AnyCPU</PlatformTarget>
|
||||
<DebugType>pdbonly</DebugType>
|
||||
<Optimize>true</Optimize>
|
||||
<OutputPath>bin\Release\</OutputPath>
|
||||
<DefineConstants>TRACE</DefineConstants>
|
||||
<ErrorReport>prompt</ErrorReport>
|
||||
<WarningLevel>4</WarningLevel>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<Reference Include="Newtonsoft.Json">
|
||||
<HintPath>..\..\packages\Newtonsoft.Json.4.5.11\lib\net40\Newtonsoft.Json.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="System" />
|
||||
<Reference Include="System.Core" />
|
||||
<Reference Include="System.Net.Http" />
|
||||
<Reference Include="System.Net.Http.Formatting, Version=5.1.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\..\packages\Microsoft.AspNet.WebApi.Client.5.1.1\lib\net45\System.Net.Http.Formatting.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="System.Xml.Linq" />
|
||||
<Reference Include="System.Data.DataSetExtensions" />
|
||||
<Reference Include="Microsoft.CSharp" />
|
||||
<Reference Include="System.Data" />
|
||||
<Reference Include="System.Xml" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<Compile Include="TeamCityAPI.cs" />
|
||||
<Compile Include="GitHubAPI.cs" />
|
||||
<Compile Include="Program.cs" />
|
||||
<Compile Include="Models\Projects.cs" />
|
||||
<Compile Include="Properties\AssemblyInfo.cs" />
|
||||
<Compile Include="Models\Repository.cs" />
|
||||
<Compile Include="Models\SnapshotDependency.cs" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<None Include="App.config" />
|
||||
<None Include="packages.config" />
|
||||
</ItemGroup>
|
||||
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
|
||||
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
|
||||
Other similar extension points exist, see Microsoft.Common.targets.
|
||||
<Target Name="BeforeBuild">
|
||||
</Target>
|
||||
<Target Name="AfterBuild">
|
||||
</Target>
|
||||
-->
|
||||
</Project>
|
||||
|
|
@ -0,0 +1,118 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Net.Http;
|
||||
using System.Net.Http.Headers;
|
||||
using Newtonsoft.Json;
|
||||
using Newtonsoft.Json.Serialization;
|
||||
|
||||
namespace TCDependencyManager
|
||||
{
|
||||
public class TeamCityAPI
|
||||
{
|
||||
private readonly string _teamCityUrl;
|
||||
private readonly ICredentials _creds;
|
||||
|
||||
public TeamCityAPI(string teamCityUrl, ICredentials creds)
|
||||
{
|
||||
_teamCityUrl = teamCityUrl;
|
||||
_creds = creds;
|
||||
}
|
||||
|
||||
public bool TryGetDependencies(string configId, out List<string> dependencies)
|
||||
{
|
||||
string url = String.Format("httpAuth/app/rest/buildTypes/{0}/snapshot-dependencies", configId);
|
||||
var client = GetClient();
|
||||
var response = client.GetAsync(url).Result;
|
||||
if (response.StatusCode == HttpStatusCode.NotFound)
|
||||
{
|
||||
// We don't have the config setup on the CI. That is ok.
|
||||
dependencies = null;
|
||||
return false;
|
||||
}
|
||||
dependencies = response.EnsureSuccessStatusCode()
|
||||
.Content.ReadAsAsync<SnapshotDependencies>()
|
||||
.Result
|
||||
.Dependencies.Select(f => f.Id)
|
||||
.ToList();
|
||||
return true;
|
||||
}
|
||||
|
||||
public void SetDependencies(string configId, IEnumerable<string> dependencies)
|
||||
{
|
||||
foreach (var dependencyId in dependencies)
|
||||
{
|
||||
Console.WriteLine("For {0} adding: {1}", configId, dependencyId);
|
||||
|
||||
string url = String.Format("httpAuth/app/rest/buildTypes/{0}/snapshot-dependencies", configId);
|
||||
var client = GetClient();
|
||||
var props = new Properties
|
||||
{
|
||||
Property = new List<NameValuePair>
|
||||
{
|
||||
new NameValuePair("run-build-if-dependency-failed", "true"),
|
||||
new NameValuePair("take-successful-builds-only", "true"),
|
||||
new NameValuePair("take-started-build-with-same-revisions", "true")
|
||||
}
|
||||
};
|
||||
|
||||
var snapshotDependency = new SnapshotDepedency
|
||||
{
|
||||
Id = dependencyId,
|
||||
Type = "snapshot_dependency",
|
||||
Properties = props,
|
||||
BuildType = new BuildType
|
||||
{
|
||||
Id = dependencyId,
|
||||
Name = dependencyId,
|
||||
ProjectId = "AspNet",
|
||||
ProjectName = "AspNet"
|
||||
}
|
||||
};
|
||||
var serialized = JsonConvert.SerializeObject(snapshotDependency, new JsonSerializerSettings { ContractResolver = new CamelCasePropertyNamesContractResolver() });
|
||||
var content = new StringContent(serialized);
|
||||
content.Headers.ContentType = new MediaTypeHeaderValue("application/json");
|
||||
var response = client.PostAsync(url, content).Result;
|
||||
response.EnsureSuccessStatusCode();
|
||||
}
|
||||
}
|
||||
|
||||
private static string NormalizeId(string dependencyId)
|
||||
{
|
||||
return dependencyId.Replace(".", "");
|
||||
}
|
||||
|
||||
public void EnsureDependencies(string configId, IEnumerable<string> dependencies)
|
||||
{
|
||||
List<string> currentDepenencies;
|
||||
if (TryGetDependencies(configId, out currentDepenencies))
|
||||
{
|
||||
var dependenciesToAdd = dependencies.Select(NormalizeId)
|
||||
.Except(currentDepenencies, StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
SetDependencies(configId, dependenciesToAdd);
|
||||
}
|
||||
}
|
||||
|
||||
private HttpClient GetClient()
|
||||
{
|
||||
var handler = new HttpClientHandler
|
||||
{
|
||||
PreAuthenticate = true,
|
||||
Credentials = _creds
|
||||
};
|
||||
|
||||
var client = new HttpClient(handler)
|
||||
{
|
||||
BaseAddress = new Uri(_teamCityUrl)
|
||||
};
|
||||
client.DefaultRequestHeaders.Accept.Clear();
|
||||
client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
|
||||
|
||||
return client;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<packages>
|
||||
<package id="Microsoft.AspNet.WebApi.Client" version="5.1.1" targetFramework="net451" />
|
||||
<package id="Newtonsoft.Json" version="4.5.11" targetFramework="net451" />
|
||||
</packages>
|
||||
Loading…
Reference in New Issue