Remove IISIntegration submodule, add CI (#3867)

.azure/pipelines/fast-pr-validation.yml

@@ -2,23 +2,15 @@ trigger:
 - master
 - release/*
 
-# See https://github.com/aspnet/BuildTools
+jobs:
-resources:
+- template: project-ci.yml
-  repositories:
-  - repository: buildtools
-    type: github
-    endpoint: DotNet-Bot GitHub Connection
-    name: aspnet/BuildTools
-    ref: refs/heads/release/2.1
-
-phases:
-- template: .vsts-pipelines/templates/project-ci.yml@buildtools
   parameters:
     buildArgs: "/t:FastCheck"
-- phase: RepoBuilds
+- job: RepoBuilds
-  queue:
+  pool:
-    name: Hosted VS2017
+    vmImage: vs2017-win2016
-    parallel: 2
+  strategy:
+    maxParallel: 3
     matrix:
       DataProtection:
         _FolderName: DataProtection
@@ -33,3 +25,7 @@ phases:
     inputs:
       testRunner: vstest
       testResultsFiles: 'src/$(_FolderName)/artifacts/logs/**/*.trx'
+- template: jobs/iisintegration-job.yml
+  parameters:
+    variables:
+      _FolderName: IISIntegration

.azure/pipelines/jobs/default-build.yml

@@ -0,0 +1,147 @@
+# default-build.yml
+# Description: Defines a build phase for invoking build.sh/cmd
+# Parameters:
+#   jobName: string
+#       The name of the job. Defaults to the name of the OS. No spaces allowed
+#   jobDisplayName: string
+#       The friendly job name to display in the UI. Defaults to the name of the OS.
+#   poolName: string
+#       The name of the VSTS agent queue to use.
+#   agentOs: string
+#       Used in templates to define variables which are OS specific. Typically from the set { Windows, Linux, macOS }
+#   buildArgs: string
+#       Additional arguments to pass to the build.sh/cmd script.
+#       Note: -ci is always passed
+#   beforeBuild: [steps]
+#       Additional steps to run before build.sh/cmd
+#   afterBuild: [steps]
+#       Additional steps to run after build.sh/cmd
+#   artifacts:
+#      publish: boolean
+#           Should artifacts be published
+#      path: string
+#           The file path to artifacts output
+#      name: string
+#           The name of the artifact container
+#   variables: { string: string }
+#     A map of custom variables
+#   matrix: { string: { string: string } }
+#     A map of matrix configurations and variables. https://docs.microsoft.com/en-us/vsts/pipelines/yaml-schema?view=vsts#matrix
+#   demands: string | [ string ]
+#     A list of agent demands. https://docs.microsoft.com/en-us/vsts/pipelines/yaml-schema?view=vsts#demands
+#   dependsOn: string | [ string ]
+#     For fan-out/fan-in. https://docs.microsoft.com/en-us/vsts/pipelines/yaml-schema?view=vsts#phase
+#   codeSign: boolean
+#       This build definition is enabled for code signing. (Only applies to Windows)
+#   buildDirectory: string
+#       Specifies what directory to run build.sh/cmd
+
+#
+# See https://docs.microsoft.com/en-us/vsts/pipelines/yaml-schema for details
+#
+
+parameters:
+  agentOs: 'Windows'
+  poolName: ''
+  buildArgs: ''
+  configuration: 'Release'
+  demands: []
+  beforeBuild: []
+  afterBuild: []
+  codeSign: false
+  variables: {}
+  dependsOn: ''
+  # jobName: '' - use agentOs by default.
+  # jobDisplayName: '' - use agentOs by default.
+  # matrix: {} - don't define an empty object default because there is no way in template expression yet to check "if isEmpty(parameters.matrix)"
+  artifacts:
+    publish: true
+    path: 'artifacts/'
+  buildDirectory: ''
+
+jobs:
+- job: ${{ coalesce(parameters.jobName, parameters.agentOs) }}
+  displayName: ${{ coalesce(parameters.jobDisplayName, parameters.agentOs) }}
+  dependsOn: ${{ parameters.dependsOn }}
+  workspace:
+    clean: all
+  strategy:
+    ${{ if ne(parameters.matrix, '') }}:
+      maxParallel: 8
+      matrix: ${{ parameters.matrix }}
+  # Map friendly OS names to the right queue
+  pool:
+    ${{ if ne(parameters.poolName, '') }}:
+      name: ${{ parameters.poolName }}
+    ${{ if and(eq(parameters.poolName, ''), eq(parameters.agentOs, 'macOS')) }}:
+      name: Hosted macOS
+      vmImage: macOS-10.13
+    ${{ if and(eq(parameters.poolName, ''), eq(parameters.agentOs, 'Linux')) }}:
+      name: Hosted Ubuntu 1604
+      vmImage: ubuntu-16.04
+    ${{ if and(eq(parameters.poolName, ''), eq(parameters.agentOs, 'Windows')) }}:
+      ${{ if ne(parameters.codeSign, 'true') }}:
+        name: Hosted VS2017
+        vmImage: vs2017-win2016
+      ${{ if eq(parameters.codeSign, 'true') }}:
+        name: DotNetCore-Windows
+  variables:
+    AgentOsName: ${{ parameters.agentOs }}
+    ASPNETCORE_TEST_LOG_MAXPATH: "200" # Keep test log file name length low enough for artifact zipping
+    DOTNET_HOME: $(Agent.WorkFolder)/.dotnet
+    BuildScriptArgs: ${{ parameters.buildArgs }}
+    BuildConfiguration: ${{ parameters.configuration }}
+    BuildDirectory: ${{ parameters.buildDirectory }}
+    VSTS_OVERWRITE_TEMP: false # Workaround for https://github.com/dotnet/core-eng/issues/2812
+    ${{ if eq(parameters.codeSign, 'true') }}:
+      TeamName: AspNetCore
+      _SignType: real
+    ${{ if ne(parameters.codeSign, 'true') }}:
+      _SignType:
+    ${{ insert }}: ${{ parameters.variables }}
+  steps:
+  - checkout: self
+    clean: true
+  - ${{ if and(eq(parameters.agentOs, 'Windows'), eq(parameters.codeSign, 'true')) }}:
+    - task: MicroBuildSigningPlugin@1
+      displayName: Install MicroBuild Signing plugin
+      condition: and(succeeded(), in(variables['_SignType'], 'test', 'real'))
+      inputs:
+        signType: $(_SignType)
+        zipSources: false
+  - ${{ parameters.beforeBuild }}
+  - ${{ if eq(parameters.agentOs, 'Windows') }}:
+    - script: .\$(BuildDirectory)\build.cmd -ci /p:SignType=$(_SignType) /p:Configuration=$(BuildConfiguration) $(BuildScriptArgs)
+      displayName: Run build.cmd
+  - ${{ if ne(parameters.agentOs, 'Windows') }}:
+    - script: ./$(BuildDirectory)/build.sh -ci -p:Configuration=$(BuildConfiguration) $(BuildScriptArgs)
+      displayName: Run build.sh
+  - task: PublishTestResults@2
+    displayName: Publish test results
+    condition: always()
+    inputs:
+      testRunTitle: $(AgentOsName)-$(BuildConfiguration)
+      testRunner: vstest
+      testResultsFiles: 'artifacts/logs/**/*.trx'
+      mergeTestResults: true
+  - ${{ if eq(parameters.artifacts.publish, 'true') }}:
+    - task: PublishBuildArtifacts@1
+      displayName: Upload artifacts
+      condition: eq(variables['system.pullrequest.isfork'], false)
+      inputs:
+        ${{ if eq(parameters.buildDirectory, '') }}:
+          pathtoPublish: ${{ parameters.artifacts.path }}
+        ${{ if ne(parameters.artifacts.name, '') }}:
+          pathtoPublish: ${{ parameters.buildDirectory }}\${{ parameters.artifacts.path }}
+        ${{ if eq(parameters.artifacts.name, '') }}:
+          artifactName: artifacts-$(AgentOsName)-$(BuildConfiguration)
+        ${{ if ne(parameters.artifacts.name, '') }}:
+          artifactName: ${{ parameters.artifacts.name }}
+        artifactType: Container
+        parallel: true
+  - ${{ parameters.afterBuild }}
+  - ${{ if and(eq(parameters.agentOs, 'Windows'), eq(parameters.codeSign, 'true')) }}:
+    - task: MicroBuildCleanup@1
+      displayName: Cleanup MicroBuild tasks
+      condition: always()
+

.azure/pipelines/jobs/iisintegration-job.yml

@@ -0,0 +1,20 @@
+jobs:
+- template: default-build.yml
+  parameters:
+    jobName: 'IISIntegration'
+    jobDisplayName: 'IISIntegration'
+    beforeBuild:
+      - powershell: "& ./src/IISIntegration/tools/UpdateIISExpressCertificate.ps1; & ./src/IISIntegration/tools/SetupTestEnvironment.ps1 Setup"
+        displayName: Prepare repo
+    afterBuild:
+      - powershell: "& ./src/IISIntegration/tools/SetupTestEnvironment.ps1 Shutdown"
+        displayName: Stop AppVerifier
+        condition: always()
+      - task: PublishBuildArtifacts@1
+        displayName: Upload logs
+        condition: eq(variables['system.pullrequest.isfork'], false)
+        inputs:
+          artifactName: logs
+          artifactType: Container
+          pathtoPublish: src/IISIntegration/artifacts/logs
+    buildDirectory: src/IISIntegration

.azure/pipelines/project-ci.yml

@@ -0,0 +1,60 @@
+# Description: Runs build.cmd/sh on macOS, Linux, and Windows
+# Parameters:
+#   buildArgs: string
+#       Additional arguments to pass to the build.sh/cmd script.
+#       Note: -ci is always passed
+#   beforeBuild: [steps]
+#       Additional steps to run before build.sh/cmd
+#   afterBuild: [steps]
+#       Additional steps to run after build.sh/cmd
+#   variables: {}
+#       VSTS build and environment variables
+#   matrix: {}
+#       The matrix of configurations to run. By default, it runs a Debug and Release build on all platforms
+#   codeSign: boolean
+#       This build definition is enabled for code signing. (Only applies to Windows)
+#   buildDirectory: string
+#       Specifies what directory to run build.sh/cmd
+
+parameters:
+  buildArgs: ''
+  beforeBuild: []
+  afterBuild: []
+  codeSign: false
+  variables: {}
+  matrix:
+    Release:
+      BuildConfiguration: Release
+    Debug:
+      BuildConfiguration: Debug
+  buildDirectory: ''
+
+jobs:
+- template: jobs/default-build.yml
+  parameters:
+    agentOs: Windows
+    matrix: ${{ parameters.matrix }}
+    buildArgs: ${{ parameters.buildArgs }}
+    beforeBuild: ${{ parameters.beforeBuild }}
+    afterBuild: ${{ parameters.afterBuild }}
+    codeSign: ${{ parameters.codeSign }}
+    variables: ${{ parameters.variables }}
+    buildDirectory: ${{ parameters.buildDirectory }}
+- template: jobs/default-build.yml
+  parameters:
+    agentOs: macOS
+    matrix: ${{ parameters.matrix }}
+    buildArgs: ${{ parameters.buildArgs }}
+    beforeBuild: ${{ parameters.beforeBuild }}
+    afterBuild: ${{ parameters.afterBuild }}
+    variables: ${{ parameters.variables }}
+    buildDirectory: ${{ parameters.buildDirectory }}
+- template: jobs/default-build.yml
+  parameters:
+    agentOs: Linux
+    matrix: ${{ parameters.matrix }}
+    buildArgs: ${{ parameters.buildArgs }}
+    beforeBuild: ${{ parameters.beforeBuild }}
+    afterBuild: ${{ parameters.afterBuild }}
+    variables: ${{ parameters.variables }}
+    buildDirectory: ${{ parameters.buildDirectory }}

.gitmodules

@@ -86,10 +86,6 @@
 	path = modules/Identity
 	url = https://github.com/aspnet/Identity.git
 	branch = release/2.1
-[submodule "modules/IISIntegration"]
-	path = modules/IISIntegration
-	url = https://github.com/aspnet/IISIntegration.git
-	branch = release/2.1
 [submodule "modules/JavaScriptServices"]
 	path = modules/JavaScriptServices
 	url = https://github.com/aspnet/JavaScriptServices.git

build/buildorder.props

@@ -30,7 +30,7 @@
     <RepositoryBuildOrder Include="BrowserLink" Order="8" />
     <RepositoryBuildOrder Include="BasicMiddleware" Order="9" />
     <RepositoryBuildOrder Include="Antiforgery" Order="9" />
-    <RepositoryBuildOrder Include="IISIntegration" Order="10" />
+    <RepositoryBuildOrder Include="IISIntegration" Order="10" RootPath="$(RepositoryRoot)src\IISIntegration\" />
     <RepositoryBuildOrder Include="CORS" Order="11" />
     <RepositoryBuildOrder Include="StaticFiles" Order="11" />
     <RepositoryBuildOrder Include="Routing" Order="11" />

build/submodules.props

@@ -69,7 +69,7 @@
     <ShippedRepository Include="HttpAbstractions" />
     <ShippedRepository Include="HttpClientFactory" />
     <ShippedRepository Include="HttpSysServer" />
-    <ShippedRepository Include="IISIntegration" />
+    <ShippedRepository Include="IISIntegration" RootPath="$(RepositoryRoot)src\IISIntegration\" />
     <ShippedRepository Include="JavaScriptServices" />
     <ShippedRepository Include="JsonPatch" />
     <ShippedRepository Include="KestrelHttpServer" />

modules/IISIntegration

@@ -1 +0,0 @@
-Subproject commit b5ff991eb95c71915aff1eb6283388dc02e9c67e

src/IISIntegration/tools/SetupTestEnvironment.ps1

@@ -0,0 +1,135 @@
+param($Mode)
+
+$DumpFolder = "$env:ASPNETCORE_TEST_LOG_DIR\dumps"
+if (!($DumpFolder))
+{
+    $DumpFolder = "$PSScriptRoot\..\artifacts\dumps"
+}
+if (!(Test-Path $DumpFolder))
+{
+    New-Item $DumpFolder -ItemType Directory;
+}
+$DumpFolder = Resolve-Path $DumpFolder
+
+$LogsFolder = "$PSScriptRoot\..\artifacts\logs"
+if (!(Test-Path $LogsFolder))
+{
+    New-Item $LogsFolder -ItemType Directory;
+}
+$LogsFolder = Resolve-Path $LogsFolder
+
+$werHive = "HKLM:\SOFTWARE\Microsoft\Windows\Windows Error Reporting";
+$ldHive = "$werHive\LocalDumps";
+
+
+function Setup-appverif($application)
+{
+    appverif.exe -enable Exceptions Handles Heaps Leak Locks Memory Threadpool TLS SRWLock -for $application
+    $level = 0x1E1;
+    $codes = @(
+        # Exceptions
+        0x650,
+        # Handles
+        0x300, 0x301, 0x302, 0x303, 0x304, # 0x305,
+        # Heaps
+        0x001, 0x002, 0x003, 0x004, 0x005, 0x006, 0x007, 0x008, 0x009, 0x00A, 0x00B, 0x00C, 0x00D, 0x00E, 0x00F, 0x010, 0x011, 0x012, 0x013, 0x014,
+        # Leak
+        0x900, 0x901, 0x902, 0x903, 0x904, 0x905, 0x906,
+        # Locks
+        0x200, 0x201, 0x202, 0x203, 0x204, 0x205, 0x206, 0x207, 0x208, 0x209, 0x210, 0x211, 0x212, 0x213, 0x214, 0x215,
+        # Memory
+        0x600, 0x601, 0x602, 0x603, 0x604, 0x605, 0x606, 0x607, 0x608, 0x609, 0x60A, 0x60B, 0x60C, 0x60D, 0x60E, 0x60F, 0x610, 0x612, 0x613, 0x614, 0x615, 0x616, 0x617, 0x618, 0x619, 0x61A, 0x61B, 0x61C, 0x61D, 0x61E,
+        # SRWLock
+        0x250, 0x251, 0x252, 0x253, 0x254, 0x255, 0x256, 0x257,
+        # TSL
+        0x350, 0x351, 0x352,
+        # ThreadPool
+        0x700, 0x701, 0x702, 0x703, 0x704, 0x705, 0x706, 0x707, 0x708, 0x709, 0x70A, 0x70B, 0x70C, 0x70D
+    );
+
+    setx APPVERIFIER_ENABLED_CODES "$codes";
+    setx APPVERIFIER_LEVEL $level;
+    appverif.exe -configure $codes -for $application -with ErrorReport=$level
+
+    # 0x305, - disabled because coreclr.dll!SetThreadName(void *) ofthen passes invalid handle (0xffffff)
+    appverif.exe -configure 0x305 -for $application -with ErrorReport=0
+}
+
+function Shutdown-appverif($application)
+{
+    setx APPVERIFIER_ENABLED_CODES "NONE";
+    setx APPVERIFIER_LEVEL "NONE";
+
+    appverif.exe -disable * -for $application
+}
+
+function Setup-Dumps()
+{
+    if (!(Test-Path $ldHive ))
+    {
+        New-Item -Path $werHive -Name LocalDumps
+    }
+
+    Move-Item $env:windir\System32\vsjitdebugger.exe $env:windir\System32\_vsjitdebugger.exe;
+
+    New-ItemProperty $werHive -Name "DontShowUI" -Value 1 -PropertyType "DWORD" -Force;
+
+    New-ItemProperty $ldHive -Name "DumpFolder" -Value $DumpFolder -PropertyType "ExpandString" -Force;
+    New-ItemProperty $ldHive -Name "DumpCount" -Value 15 -PropertyType "DWORD" -Force;
+    New-ItemProperty $ldHive -Name "DumpType" -Value 2 -PropertyType "DWORD" -Force;
+
+    Restart-Service WerSvc
+}
+
+function Shutdown-Dumps()
+{
+    Move-Item $env:windir\System32\_vsjitdebugger.exe $env:windir\System32\vsjitdebugger.exe;
+
+    Remove-Item $ldHive -Recurse -Force
+
+    New-ItemProperty $werHive -Name "DontShowUI" -Value 0 -PropertyType "DWORD" -Force;
+
+    $cdb = "c:\Program Files (x86)\Windows Kits\10\Debuggers\x64\cdb.exe"
+    if (!(Test-Path $cdb))
+    {
+        $downloadedFile = [System.IO.Path]::GetTempFileName();
+        $downloadedFile = "$downloadedFile.exe";
+        Invoke-WebRequest -Uri "https://go.microsoft.com/fwlink/p/?linkid=870807" -OutFile $downloadedFile;
+        & $downloadedFile /features OptionId.WindowsDesktopDebuggers /norestart /q;
+    }
+
+    foreach ($dump in (Get-ChildItem -Path $DumpFolder -Filter "*.dmp"))
+    {
+        if (Test-Path $cdb)
+        {
+            & $cdb -z $dump.FullName -y "https://msdl.microsoft.com/download/symbols" -c ".loadby sos coreclr;!sym noisy;.reload /f;.dumpcab -a $($dump.FullName).cab;q;"
+            Remove-Item $dump.FullName
+        }
+    }
+}
+
+if ($Mode -eq "Setup")
+{
+    Setup-appverif w3wp.exe
+    Setup-appverif iisexpress.exe
+
+    Setup-Dumps;
+}
+
+if ($Mode -eq "SetupDumps")
+{
+    Shutdown-appverif w3wp.exe
+    Shutdown-appverif iisexpress.exe
+
+    Setup-Dumps;
+}
+
+if ($Mode -eq "Shutdown")
+{
+    Shutdown-appverif w3wp.exe
+    Shutdown-appverif iisexpress.exe
+
+    Shutdown-Dumps;
+}
+
+Exit 0;

src/IISIntegration/tools/UpdateIISExpressCertificate.ps1

@@ -0,0 +1,20 @@
+$cert = New-SelfSignedCertificate -DnsName "localhost", "localhost" -CertStoreLocation "cert:\LocalMachine\My" -NotAfter (Get-Date).AddYears(5)
+$thumb = $cert.GetCertHashString()
+
+$Store = New-Object  -TypeName System.Security.Cryptography.X509Certificates.X509Store -ArgumentList 'root', 'LocalMachine'
+$Store.Open([System.Security.Cryptography.X509Certificates.OpenFlags]::ReadWrite)
+$Store.Add($cert)
+$Store.Close()
+
+$tempFile = [System.IO.Path]::GetTempFileName();
+$content = "";
+
+for ($i=44300; $i -le 44399; $i++) {
+    $content += "http delete sslcert ipport=0.0.0.0:$i`n";
+    $content += "http add sslcert ipport=0.0.0.0:$i certhash=$thumb appid=`{214124cd-d05b-4309-9af9-9caa44b2b74a`}`n";
+}
+
+[IO.File]::WriteAllLines($tempFile, $content)
+
+netsh -f $tempFile
+Remove-Item $tempFile;