huahaofeng
/
aspnetcore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Remove build-time service reference download feature (#8981) 

- #7500
- skip dotnet-watch `RenameCompiledFile()` test
    - #8987
This commit is contained in:
Doug Bunting 2019-04-02 09:24:27 -07:00 committed by GitHub
parent 6a6a870f08
commit 91dcbd44c1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 36 additions and 481 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

229
src/Mvc/Extensions.ApiDescription.Design/src/DownloadFile.cs
View File

@ -1,229 +0,0 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System;
using System.IO;
using System.Net.Http;
using System.Net.Sockets;
using System.Reflection;
using System.Security.Cryptography;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Build.Framework;
using Microsoft.Build.Utilities;
using Task = System.Threading.Tasks.Task;
using Utilities = Microsoft.Build.Utilities;
namespace Microsoft.Extensions.ApiDescription.Tasks
{
/// <summary>
/// Downloads a file.
/// </summary>
public class DownloadFile : Utilities.Task, ICancelableTask
{
private readonly CancellationTokenSource _cts = new CancellationTokenSource();
/// <summary>
/// The URI to download.
/// </summary>
[Required]
public string Uri { get; set; }
/// <summary>
/// Destination for the downloaded file. If the file already exists, it is not re-downloaded unless
/// <see cref="Overwrite"/> is true.
/// </summary>
[Required]
public string DestinationPath { get; set; }
/// <summary>
/// Should <see cref="DestinationPath"/> be overwritten. When <c>true</c>, the file is downloaded and its hash
/// compared to the existing file. If those hashes do not match (or <see cref="DestinationPath"/> does not
/// exist), <see cref="DestinationPath"/> is overwritten.
/// </summary>
public bool Overwrite { get; set; }
/// <summary>
/// The maximum amount of time in seconds to allow for downloading the file. Defaults to 2 minutes.
/// </summary>
public int TimeoutSeconds { get; set; } = 60 * 2;
/// <inheritdoc/>
public void Cancel() => _cts.Cancel();
/// <inheritdoc/>
public override bool Execute() => ExecuteAsync().Result;
public async Task<bool> ExecuteAsync()
{
if (string.IsNullOrEmpty(Uri))
{
Log.LogError("Uri parameter must not be null or empty.");
return false;
}
if (string.IsNullOrEmpty(Uri))
{
Log.LogError("DestinationPath parameter must not be null or empty.");
return false;
}
var builder = new UriBuilder(Uri);
if (!string.Equals(System.Uri.UriSchemeHttp, builder.Scheme, StringComparison.OrdinalIgnoreCase) &&
!string.Equals(System.Uri.UriSchemeHttps, builder.Scheme, StringComparison.OrdinalIgnoreCase))
{
Log.LogError($"{nameof(Uri)} parameter does not have scheme {System.Uri.UriSchemeHttp} or " +
$"{System.Uri.UriSchemeHttps}.");
return false;
}
await DownloadFileAsync(Uri, DestinationPath, Overwrite, _cts.Token, TimeoutSeconds, Log);
return !Log.HasLoggedErrors;
}
private static async Task DownloadFileAsync(
string uri,
string destinationPath,
bool overwrite,
CancellationToken cancellationToken,
int timeoutSeconds,
TaskLoggingHelper log)
{
var destinationExists = File.Exists(destinationPath);
if (destinationExists && !overwrite)
{
log.LogMessage($"Not downloading '{uri}' to overwrite existing file '{destinationPath}'.");
return;
}
log.LogMessage(MessageImportance.High, $"Downloading '{uri}' to '{destinationPath}'.");
using (var httpClient = new HttpClient())
{
await DownloadAsync(uri, destinationPath, httpClient, cancellationToken, log, timeoutSeconds);
}
}
public static async Task DownloadAsync(
string uri,
string destinationPath,
HttpClient httpClient,
CancellationToken cancellationToken,
TaskLoggingHelper log,
int timeoutSeconds)
{
// Timeout if the response has not begun within 1 minute
httpClient.Timeout = TimeSpan.FromMinutes(1);
var destinationExists = File.Exists(destinationPath);
var reachedCopy = false;
try
{
using (var response = await httpClient.GetAsync(uri, cancellationToken))
{
response.EnsureSuccessStatusCode();
cancellationToken.ThrowIfCancellationRequested();
using (var responseStreamTask = response.Content.ReadAsStreamAsync())
{
var finished = await Task.WhenAny(
responseStreamTask,
Task.Delay(TimeSpan.FromSeconds(timeoutSeconds)));
if (!ReferenceEquals(responseStreamTask, finished))
{
throw new TimeoutException($"Download failed to complete in {timeoutSeconds} seconds.");
}
using (var responseStream = await responseStreamTask)
{
if (destinationExists)
{
// Check hashes before using the downloaded information.
var downloadHash = GetHash(responseStream);
responseStream.Position = 0L;
byte[] destinationHash;
using (var destinationStream = File.OpenRead(destinationPath))
{
destinationHash = GetHash(destinationStream);
}
var sameHashes = downloadHash.Length == destinationHash.Length;
for (var i = 0; sameHashes && i < downloadHash.Length; i++)
{
sameHashes = downloadHash[i] == destinationHash[i];
}
if (sameHashes)
{
log.LogMessage($"Not overwriting existing and matching file '{destinationPath}'.");
return;
}
}
else
{
// May need to create directory to hold the file.
var destinationDirectory = Path.GetDirectoryName(destinationPath);
if (!string.IsNullOrEmpty(destinationDirectory))
{
Directory.CreateDirectory(destinationDirectory);
}
}
// Create or overwrite the destination file.
reachedCopy = true;
using (var outStream = File.Create(destinationPath))
{
await responseStream.CopyToAsync(outStream);
await outStream.FlushAsync();
}
}
}
}
}
catch (HttpRequestException ex) when (destinationExists)
{
if (ex.InnerException is SocketException socketException)
{
log.LogWarning($"Unable to download {uri}, socket error code '{socketException.SocketErrorCode}'.");
}
else
{
log.LogWarning($"Unable to download {uri}: {ex.Message}");
}
}
catch (Exception ex)
{
log.LogError($"Downloading '{uri}' failed.");
log.LogErrorFromException(ex, showStackTrace: true);
if (reachedCopy)
{
File.Delete(destinationPath);
}
}
}
private static byte[] GetHash(Stream stream)
{
SHA256 algorithm;
try
{
algorithm = SHA256.Create();
}
catch (TargetInvocationException)
{
// SHA256.Create is documented to throw this exception on FIPS-compliant machines. See
// https://msdn.microsoft.com/en-us/library/z08hz7ad Fall back to a FIPS-compliant SHA256 algorithm.
algorithm = new SHA256CryptoServiceProvider();
}
using (algorithm)
{
return algorithm.ComputeHash(stream);
}
}
}
}

4
src/Mvc/Extensions.ApiDescription.Design/src/GetFileReferenceMetadata.cs
View File

@ -66,10 +66,6 @@ namespace Microsoft.Extensions.ApiDescription.Tasks
{
type = "ServiceProjectReference";
}
else if (!string.IsNullOrEmpty(item.GetMetadata("SourceUri")))
{
type = "ServiceUriReference";
}
else
{
type = "ServiceFileReference";

138
src/Mvc/Extensions.ApiDescription.Design/src/GetUriReferenceMetadata.cs
View File

@ -1,138 +0,0 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
using System;
using System.Collections.Generic;
using System.IO;
using System.Text;
using Microsoft.Build.Framework;
using Microsoft.Build.Utilities;
namespace Microsoft.Extensions.ApiDescription.Tasks
{
/// <summary>
/// Adds or corrects DocumentPath metadata in ServiceUriReference items.
/// </summary>
public class GetUriReferenceMetadata : Task
{
private static readonly char[] InvalidFilenameCharacters = Path.GetInvalidFileNameChars();
private static readonly string[] InvalidFilenameStrings = new[] { ".." };
/// <summary>
/// Default directory for DocumentPath metadata values.
/// </summary>
public string DocumentDirectory { get; set; }
/// <summary>
/// The ServiceUriReference items to update.
/// </summary>
[Required]
public ITaskItem[] Inputs { get; set; }
/// <summary>
/// The updated ServiceUriReference items. Will include DocumentPath metadata with full paths.
/// </summary>
[Output]
public ITaskItem[] Outputs{ get; set; }
/// <inheritdoc />
public override bool Execute()
{
var outputs = new List<ITaskItem>(Inputs.Length);
var destinations = new HashSet<string>();
foreach (var item in Inputs)
{
var newItem = new TaskItem(item);
outputs.Add(newItem);
var uri = item.ItemSpec;
var builder = new UriBuilder(uri);
if (!builder.Uri.IsAbsoluteUri)
{
Log.LogError($"{nameof(Inputs)} item '{uri}' is not an absolute URI.");
return false;
}
if (!string.Equals(Uri.UriSchemeHttp, builder.Scheme, StringComparison.OrdinalIgnoreCase) &&
!string.Equals(Uri.UriSchemeHttps, builder.Scheme, StringComparison.OrdinalIgnoreCase))
{
Log.LogError($"{nameof(Inputs)} item '{uri}' does not have scheme {Uri.UriSchemeHttp} or " +
$"{Uri.UriSchemeHttps}.");
return false;
}
// If not specified, base filename on the URI.
var documentPath = item.GetMetadata("DocumentPath");
if (string.IsNullOrEmpty(documentPath))
{
// Default to a fairly long but identifiable and fairly unique filename.
var documentPathBuilder = new StringBuilder(builder.Host);
if (!string.IsNullOrEmpty(builder.Path) &&
!string.Equals("/", builder.Path, StringComparison.Ordinal))
{
documentPathBuilder.Append(builder.Path);
}
if (!string.IsNullOrEmpty(builder.Query) &&
!string.Equals("?", builder.Query, StringComparison.Ordinal))
{
documentPathBuilder.Append(builder.Query);
}
// Sanitize the string because it likely contains illegal filename characters such as '/' and '?'.
// (Do not treat slashes as folder separators.)
documentPath = documentPathBuilder.ToString();
documentPath = string.Join("_", documentPath.Split(InvalidFilenameCharacters));
while (documentPath.Contains(InvalidFilenameStrings[0]))
{
documentPath = string.Join(
".",
documentPath.Split(InvalidFilenameStrings, StringSplitOptions.None));
}
// URI might end with ".json". Don't duplicate that or a final period.
if (!documentPath.EndsWith(".json", StringComparison.OrdinalIgnoreCase))
{
if (documentPath.EndsWith(".", StringComparison.Ordinal))
{
documentPath += "json";
}
else
{
documentPath += ".json";
}
}
}
documentPath = GetFullPath(documentPath);
if (!destinations.Add(documentPath))
{
// This case may occur when user is experimenting e.g. with multiple code generators or options.
// May also occur when user accidentally duplicates DocumentPath metadata.
Log.LogError(Resources.FormatDuplicateUriDocumentPaths(documentPath));
}
MetadataSerializer.SetMetadata(newItem, "DocumentPath", documentPath);
}
Outputs = outputs.ToArray();
return !Log.HasLoggedErrors;
}
private string GetFullPath(string path)
{
if (!Path.IsPathRooted(path))
{
if (!string.IsNullOrEmpty(DocumentDirectory))
{
path = Path.Combine(DocumentDirectory, path);
}
path = Path.GetFullPath(path);
}
return path;
}
}
}

18
src/Mvc/Extensions.ApiDescription.Design/src/Properties/Resources.Designer.cs generated
View File

@ -11,7 +11,7 @@ namespace Microsoft.Extensions.ApiDescription.Tasks
= new ResourceManager("Microsoft.Extensions.ApiDescription.Tasks.Resources", typeof(Resources).GetTypeInfo().Assembly);
/// <summary>
/// Multiple items have OutputPath='{0}'. All ServiceFileReference, ServiceProjectReference and ServiceUriReference items must have unique OutputPath metadata.
/// Multiple items have OutputPath='{0}'. All ServiceFileReference and ServiceProjectReference items must have unique OutputPath metadata.
/// </summary>
internal static string DuplicateFileOutputPaths
{
@ -19,7 +19,7 @@ namespace Microsoft.Extensions.ApiDescription.Tasks
}
/// <summary>
/// Multiple items have OutputPath='{0}'. All ServiceFileReference, ServiceProjectReference and ServiceUriReference items must have unique OutputPath metadata.
/// Multiple items have OutputPath='{0}'. All ServiceFileReference and ServiceProjectReference items must have unique OutputPath metadata.
/// </summary>
internal static string FormatDuplicateFileOutputPaths(object p0)
=> string.Format(CultureInfo.CurrentCulture, GetString("DuplicateFileOutputPaths"), p0);
@ -38,20 +38,6 @@ namespace Microsoft.Extensions.ApiDescription.Tasks
internal static string FormatDuplicateProjectDocumentPaths(object p0)
=> string.Format(CultureInfo.CurrentCulture, GetString("DuplicateProjectDocumentPaths"), p0);
/// <summary>
/// Mutliple ServiceUriReference items have DocumentPath='{0}'. ServiceUriReference items must have unique DocumentPath metadata.
/// </summary>
internal static string DuplicateUriDocumentPaths
{
get => GetString("DuplicateUriDocumentPaths");
}
/// <summary>
/// Mutliple ServiceUriReference items have DocumentPath='{0}'. ServiceUriReference items must have unique DocumentPath metadata.
/// </summary>
internal static string FormatDuplicateUriDocumentPaths(object p0)
=> string.Format(CultureInfo.CurrentCulture, GetString("DuplicateUriDocumentPaths"), p0);
/// <summary>
/// Invalid {0} metadata value for {1} item '{2}'. {0} metadata must not be set to the empty string.
/// </summary>

62
src/Mvc/Extensions.ApiDescription.Design/src/Resources.resx
View File

@ -1,17 +1,17 @@
<?xml version="1.0" encoding="utf-8"?>
<root>
<!--
Microsoft ResX Schema
<!--
Microsoft ResX Schema
Version 2.0
The primary goals of this format is to allow a simple XML format
that is mostly human readable. The generation and parsing of the
various data types are done through the TypeConverter classes
The primary goals of this format is to allow a simple XML format
that is mostly human readable. The generation and parsing of the
various data types are done through the TypeConverter classes
associated with the data types.
Example:
... ado.net/XML headers & schema ...
<resheader name="resmimetype">text/microsoft-resx</resheader>
<resheader name="version">2.0</resheader>
@ -26,36 +26,36 @@
<value>[base64 mime encoded string representing a byte array form of the .NET Framework object]</value>
<comment>This is a comment</comment>
</data>
There are any number of "resheader" rows that contain simple
There are any number of "resheader" rows that contain simple
name/value pairs.
Each data row contains a name, and value. The row also contains a
type or mimetype. Type corresponds to a .NET class that support
text/value conversion through the TypeConverter architecture.
Classes that don't support this are serialized and stored with the
Each data row contains a name, and value. The row also contains a
type or mimetype. Type corresponds to a .NET class that support
text/value conversion through the TypeConverter architecture.
Classes that don't support this are serialized and stored with the
mimetype set.
The mimetype is used for serialized objects, and tells the
ResXResourceReader how to depersist the object. This is currently not
The mimetype is used for serialized objects, and tells the
ResXResourceReader how to depersist the object. This is currently not
extensible. For a given mimetype the value must be set accordingly:
Note - application/x-microsoft.net.object.binary.base64 is the format
that the ResXResourceWriter will generate, however the reader can
Note - application/x-microsoft.net.object.binary.base64 is the format
that the ResXResourceWriter will generate, however the reader can
read any of the formats listed below.
mimetype: application/x-microsoft.net.object.binary.base64
value : The object must be serialized with
value : The object must be serialized with
: System.Runtime.Serialization.Formatters.Binary.BinaryFormatter
: and then encoded with base64 encoding.
mimetype: application/x-microsoft.net.object.soap.base64
value : The object must be serialized with
value : The object must be serialized with
: System.Runtime.Serialization.Formatters.Soap.SoapFormatter
: and then encoded with base64 encoding.
mimetype: application/x-microsoft.net.object.bytearray.base64
value : The object must be serialized into a byte array
value : The object must be serialized into a byte array
: using a System.ComponentModel.TypeConverter
: and then encoded with base64 encoding.
-->
@ -118,16 +118,12 @@
<value>System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
</resheader>
<data name="DuplicateFileOutputPaths" xml:space="preserve">
<value>Multiple items have OutputPath='{0}'. All ServiceFileReference, ServiceProjectReference and ServiceUriReference items must have unique OutputPath metadata.</value>
<comment>ServiceProjectReference and ServiceUriReference items become ServiceFileReference items and all ServiceFileReference items must have unique OutputPath metadata.</comment>
<value>Multiple items have OutputPath='{0}'. All ServiceFileReference and ServiceProjectReference items must have unique OutputPath metadata.</value>
<comment>ServiceProjectReference items become ServiceFileReference items and all ServiceFileReference items must have unique OutputPath metadata.</comment>
</data>
<data name="DuplicateProjectDocumentPaths" xml:space="preserve">
<value>Mutliple ServiceProjectReference items have DocumentPath='{0}'. ServiceProjectReference items must have unique DocumentPath metadata.</value>
</data>
<data name="DuplicateUriDocumentPaths" xml:space="preserve">
<value>Mutliple ServiceUriReference items have DocumentPath='{0}'. ServiceUriReference items must have unique DocumentPath metadata.</value>
<comment>Ignore corner case of ServiceProjectReference and ServiceUriReference items having the same DocumentPath.</comment>
</data>
<data name="InvalidEmptyMetadataValue" xml:space="preserve">
<value>Invalid {0} metadata value for {1} item '{2}'. {0} metadata must not be set to the empty string.</value>
</data>

20
src/Mvc/Extensions.ApiDescription.Design/src/build/Microsoft.Extensions.ApiDescription.Design.props
View File

@ -11,9 +11,6 @@
<UsingTask TaskName="GetCurrentItems" AssemblyFile="$(_ApiDescriptionTasksAssemblyPath)" />
<UsingTask TaskName="GetFileReferenceMetadata" AssemblyFile="$(_ApiDescriptionTasksAssemblyPath)" />
<UsingTask TaskName="GetProjectReferenceMetadata" AssemblyFile="$(_ApiDescriptionTasksAssemblyPath)" />
<UsingTask TaskName="GetUriReferenceMetadata" AssemblyFile="$(_ApiDescriptionTasksAssemblyPath)" />
<UsingTask TaskName="Microsoft.Extensions.ApiDescription.Tasks.DownloadFile"
AssemblyFile="$(_ApiDescriptionTasksAssemblyPath)" />
<!--
Settings users may update as they see fit. All $(...Directory) values are interpreted relative to the client
@ -25,11 +22,6 @@
<ServiceProjectReferenceDirectory
Condition="'$(ServiceProjectReferenceDirectory)' != ''">$([MSBuild]::EnsureTrailingSlash('$(ServiceProjectReferenceDirectory)'))</ServiceProjectReferenceDirectory>
<ServiceUriReferenceCheckIfNewer
Condition="'$(ServiceUriReferenceCheckIfNewer)' == ''">true</ServiceUriReferenceCheckIfNewer>
<ServiceUriReferenceDirectory
Condition="'$(ServiceUriReferenceDirectory)' != ''">$([MSBuild]::EnsureTrailingSlash('$(ServiceUriReferenceDirectory)'))</ServiceUriReferenceDirectory>
<ServiceFileReferenceCheckIfNewer
Condition="'$(ServiceFileReferenceCheckIfNewer)' == ''">true</ServiceFileReferenceCheckIfNewer>
<ServiceFileReferenceDirectory
@ -39,8 +31,8 @@
</PropertyGroup>
<!--
Well-known metadata of the code and document generator item groups. ServiceProjectReference and ServiceUriReference
items may also include ServiceFileReference metadata.
Well-known metadata of the code and document generator item groups. ServiceProjectReference items may also include
ServiceFileReference metadata.
-->
<ItemDefinitionGroup>
<ServiceProjectReference>
@ -107,14 +99,6 @@
<Service />
</ServiceProjectReference>
<ServiceUriReference>
<!--
Full path where the API description document is placed. Default filename is based on %(Identity).
Filenames and relative paths (if explicitly set) are combined with $(ServiceUriReferenceDirectory).
-->
<DocumentPath />
</ServiceUriReference>
<ServiceFileReference>
<!-- Name of the class to generate. Defaults to match final segment of %(OutputPath). -->
<ClassName />

44
src/Mvc/Extensions.ApiDescription.Design/src/build/Microsoft.Extensions.ApiDescription.Design.targets
View File

@ -10,13 +10,8 @@
_GenerateServiceProjectReferenceDocuments;
_CreateFileItemsForServiceProjectReferences
</GenerateServiceProjectReferenceDocumentsDependsOn>
<GenerateServiceUriReferenceDocumentsDependsOn>
_GetMetadataForServiceUriReferences;
_GenerateServiceUriReferenceDocuments
</GenerateServiceUriReferenceDocumentsDependsOn>
<GenerateServiceFileReferenceCodesDependsOn>
GenerateServiceProjectReferenceDocuments;
GenerateServiceUriReferenceDocuments;
_GetMetadataForServiceFileReferences;
_GenerateServiceFileReferenceCodes;
_CreateCompileItemsForServiceFileReferences
@ -210,41 +205,6 @@
IgnoreExitCode="$([System.IO.File]::Exists('%(DocumentPath)'))" />
</Target>
<!-- ServiceUriReference support -->
<Target Name="_GetMetadataForServiceUriReferences" Condition="'@(ServiceUriReference)' != ''">
<ItemGroup>
<_Temporary Remove="@(_Temporary)" />
</ItemGroup>
<GetUriReferenceMetadata Inputs="@(ServiceUriReference)" DocumentDirectory="$(ServiceUriReferenceDirectory)">
<Output TaskParameter="Outputs" ItemName="_Temporary" />
</GetUriReferenceMetadata>
<ItemGroup>
<ServiceUriReference Remove="@(ServiceUriReference)" />
<ServiceUriReference Include="@(_Temporary)" />
<_Temporary Remove="@(_Temporary)" />
</ItemGroup>
</Target>
<Target Name="_GenerateServiceUriReferenceDocuments" Condition="'@(ServiceUriReference)' != ''">
<Microsoft.Extensions.ApiDescription.Tasks.DownloadFile Uri="%(ServiceUriReference.Identity)"
DestinationPath="%(DocumentPath)"
Overwrite="$(ServiceUriReferenceCheckIfNewer)" />
<!-- GetUriReferenceMetadata task guarantees %(DocumentPath) values are unique. -->
<ItemGroup>
<ServiceFileReference Remove="@(ServiceUriReference -> '%(DocumentPath)')" />
<ServiceFileReference Include="@(ServiceUriReference -> '%(DocumentPath)')">
<SourceUri>%(ServiceUriReference.Identity)</SourceUri>
</ServiceFileReference>
</ItemGroup>
</Target>
<Target Name="GenerateServiceUriReferenceDocuments"
DependsOnTargets="$(GenerateServiceUriReferenceDocumentsDependsOn)" />
<!-- ServiceFileReference support -->
<Target Name="_GetMetadataForServiceFileReferences" Condition="'@(ServiceFileReference)' != ''">
@ -285,8 +245,8 @@
<Target Name="_CreateCompileItemsForServiceFileReferences" Condition="'@(ServiceFileReference)' != ''">
<!--
While %(DocumentPath) metadata may include duplicates (due to overlaps between ServiceUriReference and
ServiceProjectReference items), GetFileReferenceMetadata task guarantees %(OutputPath) values are unique.
While %(DocumentPath) metadata may include duplicates, GetFileReferenceMetadata task guarantees %(OutputPath)
values are unique.
-->
<ItemGroup>
<_Files Remove="@(_Files)" />

2
src/Tools/dotnet-watch/test/GlobbingAppTests.cs
View File

@ -74,7 +74,7 @@ namespace Microsoft.DotNet.Watcher.Tools.FunctionalTests
Assert.Equal(1, types);
}
[Fact]
[Fact(Skip = "https://github.com/aspnet/AspNetCore/issues/8987")]
public async Task RenameCompiledFile()
{
await _app.StartWatcherAsync();