Use different Ubuntu pools for Ubuntu tests to avoid disk space issues. (#20742)

- Use different Ubuntu pools for Ubuntu tests to avoid disk space issues.
  - Updated the `default-build.yml` to have a new parameter specific to Ubuntu listed `useHostedUbuntu`. If we feel the need to expand the parameters usage outside of the Ubuntu configuration we can always rename to `useHosted`. Also didn't want to touch the `isTestingJob` because I wasn't sure of hte implications.
  - Updated the Ubuntu test job to turn off hosted pools.

Fixes https://github.com/dotnet/aspnetcore-internal/issues/3574

- Hmm, are `queue` names case-sensitive?
- Correct `useHostedUbuntu: false` placement
- Correct placement of `useHostedUbuntu: true` default
- Try adding `--without-http_rewrite_module` to nginx configuration
  - also add `set -euo pipefail` to the script for fast failure

Co-authored-by: Doug Bunting <6431421+dougbu@users.noreply.github.com>

.azure/pipelines/ci.yml

@@ -633,6 +633,7 @@ stages:
       jobDisplayName: "Test: Ubuntu 16.04 x64"
       agentOs: Linux
       isTestingJob: true
+      useHostedUbuntu: false
       buildArgs: --all --test "/p:RunTemplateTests=false /p:SkipHelixReadyTests=true" $(_InternalRuntimeDownloadArgs)
       beforeBuild:
       - bash: "./eng/scripts/install-nginx-linux.sh"

.azure/pipelines/jobs/default-build.yml

@@ -59,6 +59,7 @@ parameters:
   installNodeJs: true
   installJdk: true
   timeoutInMinutes: 180
+  useHostedUbuntu: true
 
   # We need longer than the default amount of 5 minutes to upload our logs/artifacts. (We currently take around 5 mins in the best case).
   # This makes sure we have time to upload everything in the case of a build timeout - really important for investigating a build
@@ -91,7 +92,15 @@ jobs:
       ${{ if eq(parameters.agentOs, 'macOS') }}:
         vmImage: macOS-10.14
       ${{ if eq(parameters.agentOs, 'Linux') }}:
+        ${{ if eq(parameters.useHostedUbuntu, true) }}:
           vmImage: ubuntu-16.04
+        ${{ if eq(parameters.useHostedUbuntu, false) }}:
+          ${{ if eq(variables['System.TeamProject'], 'public') }}:
+            name: NetCorePublic-Pool
+            queue: BuildPool.Ubuntu.1604.Amd64.Open
+          ${{ if eq(variables['System.TeamProject'], 'internal') }}:
+            name: NetCoreInternal-Pool
+            queue: BuildPool.Ubuntu.1604.Amd64
       ${{ if eq(parameters.agentOs, 'Windows') }}:
         ${{ if eq(variables['System.TeamProject'], 'public') }}:
           name: NetCorePublic-Pool

eng/scripts/install-nginx-linux.sh

@@ -1,7 +1,9 @@
 #!/usr/bin/env bash
 
+set -euo pipefail
+
 curl -sSL http://nginx.org/download/nginx-1.14.2.tar.gz | tar zxfv - -C /tmp && cd /tmp/nginx-1.14.2/
-./configure --prefix=$HOME/nginxinstall --with-http_ssl_module
+./configure --prefix=$HOME/nginxinstall --with-http_ssl_module --without-http_rewrite_module
 make
 make install
 echo "##vso[task.prependpath]$HOME/nginxinstall/sbin"