msbuild

1
# Initialize variables if they aren't already defined.
2
# These may be defined as parameters of the importing script, or set after importing this script.
3

4
# CI mode - set to true on CI server for PR validation build or official build.
5
[bool]$ci = if (Test-Path variable:ci) { $ci } else { $false }
6

7
# Build configuration. Common values include 'Debug' and 'Release', but the repository may use other names.
8
[string]$configuration = if (Test-Path variable:configuration) { $configuration } else { 'Debug' }
9

10
# Set to true to opt out of outputting binary log while running in CI
11
[bool]$excludeCIBinarylog = if (Test-Path variable:excludeCIBinarylog) { $excludeCIBinarylog } else { $false }
12

13
# Set to true to output binary log from msbuild. Note that emitting binary log slows down the build.
14
[bool]$binaryLog = if (Test-Path variable:binaryLog) { $binaryLog } else { $ci -and !$excludeCIBinarylog }
15

16
# Set to true to use the pipelines logger which will enable Azure logging output.
17
# https://github.com/Microsoft/azure-pipelines-tasks/blob/master/docs/authoring/commands.md
18
# This flag is meant as a temporary opt-opt for the feature while validate it across
19
# our consumers. It will be deleted in the future.
20
[bool]$pipelinesLog = if (Test-Path variable:pipelinesLog) { $pipelinesLog } else { $ci }
21

22
# Turns on machine preparation/clean up code that changes the machine state (e.g. kills build processes).
23
[bool]$prepareMachine = if (Test-Path variable:prepareMachine) { $prepareMachine } else { $false }
24

25
# True to restore toolsets and dependencies.
26
[bool]$restore = if (Test-Path variable:restore) { $restore } else { $true }
27

28
# Adjusts msbuild verbosity level.
29
[string]$verbosity = if (Test-Path variable:verbosity) { $verbosity } else { 'minimal' }
30

31
# Set to true to reuse msbuild nodes. Recommended to not reuse on CI.
32
[bool]$nodeReuse = if (Test-Path variable:nodeReuse) { $nodeReuse } else { !$ci }
33

34
# Configures warning treatment in msbuild.
35
[bool]$warnAsError = if (Test-Path variable:warnAsError) { $warnAsError } else { $true }
36

37
# Specifies which msbuild engine to use for build: 'vs', 'dotnet' or unspecified (determined based on presence of tools.vs in global.json).
38
[string]$msbuildEngine = if (Test-Path variable:msbuildEngine) { $msbuildEngine } else { $null }
39

40
# True to attempt using .NET Core already that meets requirements specified in global.json
41
# installed on the machine instead of downloading one.
42
[bool]$useInstalledDotNetCli = if (Test-Path variable:useInstalledDotNetCli) { $useInstalledDotNetCli } else { $true }
43

44
# Enable repos to use a particular version of the on-line dotnet-install scripts.
45
#    default URL: https://dotnet.microsoft.com/download/dotnet/scripts/v1/dotnet-install.ps1
46
[string]$dotnetInstallScriptVersion = if (Test-Path variable:dotnetInstallScriptVersion) { $dotnetInstallScriptVersion } else { 'v1' }
47

48
# True to use global NuGet cache instead of restoring packages to repository-local directory.
49
[bool]$useGlobalNuGetCache = if (Test-Path variable:useGlobalNuGetCache) { $useGlobalNuGetCache } else { !$ci }
50

51
# True to exclude prerelease versions Visual Studio during build
52
[bool]$excludePrereleaseVS = if (Test-Path variable:excludePrereleaseVS) { $excludePrereleaseVS } else { $false }
53

54
# An array of names of processes to stop on script exit if prepareMachine is true.
55
$processesToStopOnExit = if (Test-Path variable:processesToStopOnExit) { $processesToStopOnExit } else { @('msbuild', 'dotnet', 'vbcscompiler') }
56

57
$disableConfigureToolsetImport = if (Test-Path variable:disableConfigureToolsetImport) { $disableConfigureToolsetImport } else { $null }
58

59
set-strictmode -version 2.0
60
$ErrorActionPreference = 'Stop'
61
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
62

63
# If specifies, provides an alternate path for getting .NET Core SDKs and Runtimes. This script will still try public sources first.
64
[string]$runtimeSourceFeed = if (Test-Path variable:runtimeSourceFeed) { $runtimeSourceFeed } else { $null }
65
# Base-64 encoded SAS token that has permission to storage container described by $runtimeSourceFeed
66
[string]$runtimeSourceFeedKey = if (Test-Path variable:runtimeSourceFeedKey) { $runtimeSourceFeedKey } else { $null }
67

68
function Create-Directory ([string[]] $path) {
69
    New-Item -Path $path -Force -ItemType 'Directory' | Out-Null
70
}
71

72
function Unzip([string]$zipfile, [string]$outpath) {
73
  Add-Type -AssemblyName System.IO.Compression.FileSystem
74
  [System.IO.Compression.ZipFile]::ExtractToDirectory($zipfile, $outpath)
75
}
76

77
# This will exec a process using the console and return it's exit code.
78
# This will not throw when the process fails.
79
# Returns process exit code.
80
function Exec-Process([string]$command, [string]$commandArgs) {
81
  $startInfo = New-Object System.Diagnostics.ProcessStartInfo
82
  $startInfo.FileName = $command
83
  $startInfo.Arguments = $commandArgs
84
  $startInfo.UseShellExecute = $false
85
  $startInfo.WorkingDirectory = Get-Location
86

87
  $process = New-Object System.Diagnostics.Process
88
  $process.StartInfo = $startInfo
89
  $process.Start() | Out-Null
90

91
  $finished = $false
92
  try {
93
    while (-not $process.WaitForExit(100)) {
94
      # Non-blocking loop done to allow ctr-c interrupts
95
    }
96

97
    $finished = $true
98
    return $global:LASTEXITCODE = $process.ExitCode
99
  }
100
  finally {
101
    # If we didn't finish then an error occurred or the user hit ctrl-c.  Either
102
    # way kill the process
103
    if (-not $finished) {
104
      $process.Kill()
105
    }
106
  }
107
}
108

109
# Take the given block, print it, print what the block probably references from the current set of
110
# variables using low-effort string matching, then run the block.
111
#
112
# This is intended to replace the pattern of manually copy-pasting a command, wrapping it in quotes,
113
# and printing it using "Write-Host". The copy-paste method is more readable in build logs, but less
114
# maintainable and less reliable. It is easy to make a mistake and modify the command without
115
# properly updating the "Write-Host" line, resulting in misleading build logs. The probability of
116
# this mistake makes the pattern hard to trust when it shows up in build logs. Finding the bug in
117
# existing source code can also be difficult, because the strings are not aligned to each other and
118
# the line may be 300+ columns long.
119
#
120
# By removing the need to maintain two copies of the command, Exec-BlockVerbosely avoids the issues.
121
#
122
# In Bash (or any posix-like shell), "set -x" prints usable verbose output automatically.
123
# "Set-PSDebug" appears to be similar at first glance, but unfortunately, it isn't very useful: it
124
# doesn't print any info about the variables being used by the command, which is normally the
125
# interesting part to diagnose.
126
function Exec-BlockVerbosely([scriptblock] $block) {
127
  Write-Host "--- Running script block:"
128
  $blockString = $block.ToString().Trim()
129
  Write-Host $blockString
130

131
  Write-Host "--- List of variables that might be used:"
132
  # For each variable x in the environment, check the block for a reference to x via simple "$x" or
133
  # "@x" syntax. This doesn't detect other ways to reference variables ("${x}" nor "$variable:x",
134
  # among others). It only catches what this function was originally written for: simple
135
  # command-line commands.
136
  $variableTable = Get-Variable |
137
    Where-Object {
138
      $blockString.Contains("`$$($_.Name)") -or $blockString.Contains("@$($_.Name)")
139
    } |
140
    Format-Table -AutoSize -HideTableHeaders -Wrap |
141
    Out-String
142
  Write-Host $variableTable.Trim()
143

144
  Write-Host "--- Executing:"
145
  & $block
146
  Write-Host "--- Done running script block!"
147
}
148

149
# createSdkLocationFile parameter enables a file being generated under the toolset directory
150
# which writes the sdk's location into. This is only necessary for cmd --> powershell invocations
151
# as dot sourcing isn't possible.
152
function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) {
153
  if (Test-Path variable:global:_DotNetInstallDir) {
154
    return $global:_DotNetInstallDir
155
  }
156

157
  # Don't resolve runtime, shared framework, or SDK from other locations to ensure build determinism
158
  $env:DOTNET_MULTILEVEL_LOOKUP=0
159

160
  # Disable first run since we do not need all ASP.NET packages restored.
161
  $env:DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1
162

163
  # Disable telemetry on CI.
164
  if ($ci) {
165
    $env:DOTNET_CLI_TELEMETRY_OPTOUT=1
166
  }
167

168
  # Source Build uses DotNetCoreSdkDir variable
169
  if ($env:DotNetCoreSdkDir -ne $null) {
170
    $env:DOTNET_INSTALL_DIR = $env:DotNetCoreSdkDir
171
  }
172

173
  # Find the first path on %PATH% that contains the dotnet.exe
174
  if ($useInstalledDotNetCli -and (-not $globalJsonHasRuntimes) -and ($env:DOTNET_INSTALL_DIR -eq $null)) {
175
    $dotnetExecutable = GetExecutableFileName 'dotnet'
176
    $dotnetCmd = Get-Command $dotnetExecutable -ErrorAction SilentlyContinue
177

178
    if ($dotnetCmd -ne $null) {
179
      $env:DOTNET_INSTALL_DIR = Split-Path $dotnetCmd.Path -Parent
180
    }
181
  }
182

183
  $dotnetSdkVersion = $GlobalJson.tools.dotnet
184

185
  # Use dotnet installation specified in DOTNET_INSTALL_DIR if it contains the required SDK version,
186
  # otherwise install the dotnet CLI and SDK to repo local .dotnet directory to avoid potential permission issues.
187
  if ((-not $globalJsonHasRuntimes) -and (-not [string]::IsNullOrEmpty($env:DOTNET_INSTALL_DIR)) -and (Test-Path(Join-Path $env:DOTNET_INSTALL_DIR "sdk\$dotnetSdkVersion"))) {
188
    $dotnetRoot = $env:DOTNET_INSTALL_DIR
189
  } else {
190
    $dotnetRoot = Join-Path $RepoRoot '.dotnet'
191

192
    if (-not (Test-Path(Join-Path $dotnetRoot "sdk\$dotnetSdkVersion"))) {
193
      if ($install) {
194
        InstallDotNetSdk $dotnetRoot $dotnetSdkVersion
195
      } else {
196
        Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Unable to find dotnet with SDK version '$dotnetSdkVersion'"
197
        ExitWithExitCode 1
198
      }
199
    }
200

201
    $env:DOTNET_INSTALL_DIR = $dotnetRoot
202
  }
203

204
  # Creates a temporary file under the toolset dir.
205
  # The following code block is protecting against concurrent access so that this function can
206
  # be called in parallel.
207
  if ($createSdkLocationFile) {
208
    do {
209
      $sdkCacheFileTemp = Join-Path $ToolsetDir $([System.IO.Path]::GetRandomFileName())
210
    }
211
    until (!(Test-Path $sdkCacheFileTemp))
212
    Set-Content -Path $sdkCacheFileTemp -Value $dotnetRoot
213

214
    try {
215
      Move-Item -Force $sdkCacheFileTemp (Join-Path $ToolsetDir 'sdk.txt')
216
    } catch {
217
      # Somebody beat us
218
      Remove-Item -Path $sdkCacheFileTemp
219
    }
220
  }
221

222
  # Add dotnet to PATH. This prevents any bare invocation of dotnet in custom
223
  # build steps from using anything other than what we've downloaded.
224
  # It also ensures that VS msbuild will use the downloaded sdk targets.
225
  $env:PATH = "$dotnetRoot;$env:PATH"
226

227
  # Make Sure that our bootstrapped dotnet cli is available in future steps of the Azure Pipelines build
228
  Write-PipelinePrependPath -Path $dotnetRoot
229

230
  Write-PipelineSetVariable -Name 'DOTNET_MULTILEVEL_LOOKUP' -Value '0'
231
  Write-PipelineSetVariable -Name 'DOTNET_SKIP_FIRST_TIME_EXPERIENCE' -Value '1'
232

233
  return $global:_DotNetInstallDir = $dotnetRoot
234
}
235

236
function Retry($downloadBlock, $maxRetries = 5) {
237
  $retries = 1
238

239
  while($true) {
240
    try {
241
      & $downloadBlock
242
      break
243
    }
244
    catch {
245
      Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_
246
    }
247

248
    if (++$retries -le $maxRetries) {
249
      $delayInSeconds = [math]::Pow(2, $retries) - 1 # Exponential backoff
250
      Write-Host "Retrying. Waiting for $delayInSeconds seconds before next attempt ($retries of $maxRetries)."
251
      Start-Sleep -Seconds $delayInSeconds
252
    }
253
    else {
254
      Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Unable to download file in $maxRetries attempts."
255
      break
256
    }
257

258
  }
259
}
260

261
function GetDotNetInstallScript([string] $dotnetRoot) {
262
  $installScript = Join-Path $dotnetRoot 'dotnet-install.ps1'
263
  if (!(Test-Path $installScript)) {
264
    Create-Directory $dotnetRoot
265
    $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit
266
    $uri = "https://dotnet.microsoft.com/download/dotnet/scripts/$dotnetInstallScriptVersion/dotnet-install.ps1"
267

268
    Retry({
269
      Write-Host "GET $uri"
270
      Invoke-WebRequest $uri -OutFile $installScript
271
    })
272
  }
273

274
  return $installScript
275
}
276

277
function InstallDotNetSdk([string] $dotnetRoot, [string] $version, [string] $architecture = '', [switch] $noPath) {
278
  InstallDotNet $dotnetRoot $version $architecture '' $false $runtimeSourceFeed $runtimeSourceFeedKey -noPath:$noPath
279
}
280

281
function InstallDotNet([string] $dotnetRoot,
282
  [string] $version,
283
  [string] $architecture = '',
284
  [string] $runtime = '',
285
  [bool] $skipNonVersionedFiles = $false,
286
  [string] $runtimeSourceFeed = '',
287
  [string] $runtimeSourceFeedKey = '',
288
  [switch] $noPath) {
289

290
  $dotnetVersionLabel = "'sdk v$version'"
291

292
  if ($runtime -ne '' -and $runtime -ne 'sdk') {
293
    $runtimePath = $dotnetRoot
294
    $runtimePath = $runtimePath + "\shared"
295
    if ($runtime -eq "dotnet") { $runtimePath = $runtimePath + "\Microsoft.NETCore.App" }
296
    if ($runtime -eq "aspnetcore") { $runtimePath = $runtimePath + "\Microsoft.AspNetCore.App" }
297
    if ($runtime -eq "windowsdesktop") { $runtimePath = $runtimePath + "\Microsoft.WindowsDesktop.App" }
298
    $runtimePath = $runtimePath + "\" + $version
299
  
300
    $dotnetVersionLabel = "runtime toolset '$runtime/$architecture v$version'"
301

302
    if (Test-Path $runtimePath) {
303
      Write-Host "  Runtime toolset '$runtime/$architecture v$version' already installed."
304
      $installSuccess = $true
305
      Exit
306
    }
307
  }
308

309
  $installScript = GetDotNetInstallScript $dotnetRoot
310
  $installParameters = @{
311
    Version = $version
312
    InstallDir = $dotnetRoot
313
  }
314

315
  if ($architecture) { $installParameters.Architecture = $architecture }
316
  if ($runtime) { $installParameters.Runtime = $runtime }
317
  if ($skipNonVersionedFiles) { $installParameters.SkipNonVersionedFiles = $skipNonVersionedFiles }
318
  if ($noPath) { $installParameters.NoPath = $True }
319

320
  $variations = @()
321
  $variations += @($installParameters)
322

323
  $dotnetBuilds = $installParameters.Clone()
324
  $dotnetbuilds.AzureFeed = "https://dotnetbuilds.azureedge.net/public"
325
  $variations += @($dotnetBuilds)
326

327
  if ($runtimeSourceFeed) {
328
    $runtimeSource = $installParameters.Clone()
329
    $runtimeSource.AzureFeed = $runtimeSourceFeed
330
    if ($runtimeSourceFeedKey) {
331
      $decodedBytes = [System.Convert]::FromBase64String($runtimeSourceFeedKey)
332
      $decodedString = [System.Text.Encoding]::UTF8.GetString($decodedBytes)
333
      $runtimeSource.FeedCredential = $decodedString
334
    }
335
    $variations += @($runtimeSource)
336
  }
337

338
  $installSuccess = $false
339
  foreach ($variation in $variations) {
340
    if ($variation | Get-Member AzureFeed) {
341
      $location = $variation.AzureFeed
342
    } else {
343
      $location = "public location";
344
    }
345
    Write-Host "  Attempting to install $dotnetVersionLabel from $location."
346
    try {
347
      & $installScript @variation
348
      $installSuccess = $true
349
      break
350
    }
351
    catch {
352
      Write-Host "  Failed to install $dotnetVersionLabel from $location."
353
    }
354
  }
355
  if (-not $installSuccess) {
356
    Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install $dotnetVersionLabel from any of the specified locations."
357
    ExitWithExitCode 1
358
  }
359
}
360

361
#
362
# Locates Visual Studio MSBuild installation.
363
# The preference order for MSBuild to use is as follows:
364
#
365
#   1. MSBuild from an active VS command prompt
366
#   2. MSBuild from a compatible VS installation
367
#   3. MSBuild from the xcopy tool package
368
#
369
# Returns full path to msbuild.exe.
370
# Throws on failure.
371
#
372
function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = $null) {
373
  if (-not (IsWindowsPlatform)) {
374
    throw "Cannot initialize Visual Studio on non-Windows"
375
  }
376

377
  if (Test-Path variable:global:_MSBuildExe) {
378
    return $global:_MSBuildExe
379
  }
380

381
  # Minimum VS version to require.
382
  $vsMinVersionReqdStr = '17.7'
383
  $vsMinVersionReqd = [Version]::new($vsMinVersionReqdStr)
384

385
  # If the version of msbuild is going to be xcopied,
386
  # use this version. Version matches a package here:
387
  # https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet-eng/NuGet/RoslynTools.MSBuild/versions/17.8.1-2
388
  $defaultXCopyMSBuildVersion = '17.8.1-2'
389

390
  if (!$vsRequirements) {
391
    if (Get-Member -InputObject $GlobalJson.tools -Name 'vs') {
392
      $vsRequirements = $GlobalJson.tools.vs
393
    }
394
    else {
395
      $vsRequirements = New-Object PSObject -Property @{ version = $vsMinVersionReqdStr }
396
    }
397
  }
398
  $vsMinVersionStr = if ($vsRequirements.version) { $vsRequirements.version } else { $vsMinVersionReqdStr }
399
  $vsMinVersion = [Version]::new($vsMinVersionStr)
400

401
  # Try msbuild command available in the environment.
402
  if ($env:VSINSTALLDIR -ne $null) {
403
    $msbuildCmd = Get-Command 'msbuild.exe' -ErrorAction SilentlyContinue
404
    if ($msbuildCmd -ne $null) {
405
      # Workaround for https://github.com/dotnet/roslyn/issues/35793
406
      # Due to this issue $msbuildCmd.Version returns 0.0.0.0 for msbuild.exe 16.2+
407
      $msbuildVersion = [Version]::new((Get-Item $msbuildCmd.Path).VersionInfo.ProductVersion.Split([char[]]@('-', '+'))[0])
408

409
      if ($msbuildVersion -ge $vsMinVersion) {
410
        return $global:_MSBuildExe = $msbuildCmd.Path
411
      }
412

413
      # Report error - the developer environment is initialized with incompatible VS version.
414
      throw "Developer Command Prompt for VS $($env:VisualStudioVersion) is not recent enough. Please upgrade to $vsMinVersionStr or build from a plain CMD window"
415
    }
416
  }
417

418
  # Locate Visual Studio installation or download x-copy msbuild.
419
  $vsInfo = LocateVisualStudio $vsRequirements
420
  if ($vsInfo -ne $null) {
421
    # Ensure vsInstallDir has a trailing slash
422
    $vsInstallDir = Join-Path $vsInfo.installationPath "\"
423
    $vsMajorVersion = $vsInfo.installationVersion.Split('.')[0]
424

425
    InitializeVisualStudioEnvironmentVariables $vsInstallDir $vsMajorVersion
426
  } else {
427

428
    if (Get-Member -InputObject $GlobalJson.tools -Name 'xcopy-msbuild') {
429
      $xcopyMSBuildVersion = $GlobalJson.tools.'xcopy-msbuild'
430
      $vsMajorVersion = $xcopyMSBuildVersion.Split('.')[0]
431
    } else {
432
      #if vs version provided in global.json is incompatible (too low) then use the default version for xcopy msbuild download
433
      if($vsMinVersion -lt $vsMinVersionReqd){
434
        Write-Host "Using xcopy-msbuild version of $defaultXCopyMSBuildVersion since VS version $vsMinVersionStr provided in global.json is not compatible"
435
        $xcopyMSBuildVersion = $defaultXCopyMSBuildVersion
436
        $vsMajorVersion = $xcopyMSBuildVersion.Split('.')[0]
437
      }
438
      else{
439
        # If the VS version IS compatible, look for an xcopy msbuild package
440
        # with a version matching VS.
441
        # Note: If this version does not exist, then an explicit version of xcopy msbuild
442
        # can be specified in global.json. This will be required for pre-release versions of msbuild.
443
        $vsMajorVersion = $vsMinVersion.Major
444
        $vsMinorVersion = $vsMinVersion.Minor
445
        $xcopyMSBuildVersion = "$vsMajorVersion.$vsMinorVersion.0"
446
      }
447
    }
448

449
    $vsInstallDir = $null
450
    if ($xcopyMSBuildVersion.Trim() -ine "none") {
451
        $vsInstallDir = InitializeXCopyMSBuild $xcopyMSBuildVersion $install
452
        if ($vsInstallDir -eq $null) {
453
            throw "Could not xcopy msbuild. Please check that package 'RoslynTools.MSBuild @ $xcopyMSBuildVersion' exists on feed 'dotnet-eng'."
454
        }
455
    }
456
    if ($vsInstallDir -eq $null) {
457
      throw 'Unable to find Visual Studio that has required version and components installed'
458
    }
459
  }
460

461
  $msbuildVersionDir = if ([int]$vsMajorVersion -lt 16) { "$vsMajorVersion.0" } else { "Current" }
462

463
  $local:BinFolder = Join-Path $vsInstallDir "MSBuild\$msbuildVersionDir\Bin"
464
  $local:Prefer64bit = if (Get-Member -InputObject $vsRequirements -Name 'Prefer64bit') { $vsRequirements.Prefer64bit } else { $false }
465
  if ($local:Prefer64bit -and (Test-Path(Join-Path $local:BinFolder "amd64"))) {
466
    $global:_MSBuildExe = Join-Path $local:BinFolder "amd64\msbuild.exe"
467
  } else {
468
    $global:_MSBuildExe = Join-Path $local:BinFolder "msbuild.exe"
469
  }
470

471
  return $global:_MSBuildExe
472
}
473

474
function InitializeVisualStudioEnvironmentVariables([string] $vsInstallDir, [string] $vsMajorVersion) {
475
  $env:VSINSTALLDIR = $vsInstallDir
476
  Set-Item "env:VS$($vsMajorVersion)0COMNTOOLS" (Join-Path $vsInstallDir "Common7\Tools\")
477

478
  $vsSdkInstallDir = Join-Path $vsInstallDir "VSSDK\"
479
  if (Test-Path $vsSdkInstallDir) {
480
    Set-Item "env:VSSDK$($vsMajorVersion)0Install" $vsSdkInstallDir
481
    $env:VSSDKInstall = $vsSdkInstallDir
482
  }
483
}
484

485
function InstallXCopyMSBuild([string]$packageVersion) {
486
  return InitializeXCopyMSBuild $packageVersion -install $true
487
}
488

489
function InitializeXCopyMSBuild([string]$packageVersion, [bool]$install) {
490
  $packageName = 'RoslynTools.MSBuild'
491
  $packageDir = Join-Path $ToolsDir "msbuild\$packageVersion"
492
  $packagePath = Join-Path $packageDir "$packageName.$packageVersion.nupkg"
493

494
  if (!(Test-Path $packageDir)) {
495
    if (!$install) {
496
      return $null
497
    }
498

499
    Create-Directory $packageDir
500

501
    Write-Host "Downloading $packageName $packageVersion"
502
    $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit
503
    Retry({
504
      Invoke-WebRequest "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/flat2/$packageName/$packageVersion/$packageName.$packageVersion.nupkg" -OutFile $packagePath
505
    })
506

507
    Unzip $packagePath $packageDir
508
  }
509

510
  return Join-Path $packageDir 'tools'
511
}
512

513
#
514
# Locates Visual Studio instance that meets the minimal requirements specified by tools.vs object in global.json.
515
#
516
# The following properties of tools.vs are recognized:
517
#   "version": "{major}.{minor}"
518
#       Two part minimal VS version, e.g. "15.9", "16.0", etc.
519
#   "components": ["componentId1", "componentId2", ...]
520
#       Array of ids of workload components that must be available in the VS instance.
521
#       See e.g. https://docs.microsoft.com/en-us/visualstudio/install/workload-component-id-vs-enterprise?view=vs-2017
522
#
523
# Returns JSON describing the located VS instance (same format as returned by vswhere),
524
# or $null if no instance meeting the requirements is found on the machine.
525
#
526
function LocateVisualStudio([object]$vsRequirements = $null){
527
  if (-not (IsWindowsPlatform)) {
528
    throw "Cannot run vswhere on non-Windows platforms."
529
  }
530

531
  if (Get-Member -InputObject $GlobalJson.tools -Name 'vswhere') {
532
    $vswhereVersion = $GlobalJson.tools.vswhere
533
  } else {
534
    $vswhereVersion = '2.5.2'
535
  }
536

537
  $vsWhereDir = Join-Path $ToolsDir "vswhere\$vswhereVersion"
538
  $vsWhereExe = Join-Path $vsWhereDir 'vswhere.exe'
539

540
  if (!(Test-Path $vsWhereExe)) {
541
    Create-Directory $vsWhereDir
542
    Write-Host 'Downloading vswhere'
543
    Retry({
544
      Invoke-WebRequest "https://netcorenativeassets.blob.core.windows.net/resource-packages/external/windows/vswhere/$vswhereVersion/vswhere.exe" -OutFile $vswhereExe
545
    })
546
  }
547

548
  if (!$vsRequirements) { $vsRequirements = $GlobalJson.tools.vs }
549
  $args = @('-latest', '-format', 'json', '-requires', 'Microsoft.Component.MSBuild', '-products', '*')
550

551
  if (!$excludePrereleaseVS) {
552
    $args += '-prerelease'
553
  }
554

555
  if (Get-Member -InputObject $vsRequirements -Name 'version') {
556
    $args += '-version'
557
    $args += $vsRequirements.version
558
  }
559

560
  if (Get-Member -InputObject $vsRequirements -Name 'components') {
561
    foreach ($component in $vsRequirements.components) {
562
      $args += '-requires'
563
      $args += $component
564
    }
565
  }
566

567
  $vsInfo =& $vsWhereExe $args | ConvertFrom-Json
568

569
  if ($lastExitCode -ne 0) {
570
    return $null
571
  }
572

573
  # use first matching instance
574
  return $vsInfo[0]
575
}
576

577
function InitializeBuildTool() {
578
  if (Test-Path variable:global:_BuildTool) {
579
    # If the requested msbuild parameters do not match, clear the cached variables.
580
    if($global:_BuildTool.Contains('ExcludePrereleaseVS') -and $global:_BuildTool.ExcludePrereleaseVS -ne $excludePrereleaseVS) {
581
      Remove-Item variable:global:_BuildTool
582
      Remove-Item variable:global:_MSBuildExe
583
    } else {
584
      return $global:_BuildTool
585
    }
586
  }
587

588
  if (-not $msbuildEngine) {
589
    $msbuildEngine = GetDefaultMSBuildEngine
590
  }
591

592
  # Initialize dotnet cli if listed in 'tools'
593
  $dotnetRoot = $null
594
  if (Get-Member -InputObject $GlobalJson.tools -Name 'dotnet') {
595
    $dotnetRoot = InitializeDotNetCli -install:$restore
596
  }
597

598
  if ($msbuildEngine -eq 'dotnet') {
599
    if (!$dotnetRoot) {
600
      Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "/global.json must specify 'tools.dotnet'."
601
      ExitWithExitCode 1
602
    }
603
    $dotnetPath = Join-Path $dotnetRoot (GetExecutableFileName 'dotnet')
604

605
    # Use override if it exists - commonly set by source-build
606
    if ($null -eq $env:_OverrideArcadeInitializeBuildToolFramework) {
607
      $initializeBuildToolFramework="net8.0"
608
    } else {
609
      $initializeBuildToolFramework=$env:_OverrideArcadeInitializeBuildToolFramework
610
    }
611

612
    $buildTool = @{ Path = $dotnetPath; Command = 'msbuild'; Tool = 'dotnet'; Framework = $initializeBuildToolFramework }
613
  } elseif ($msbuildEngine -eq "vs") {
614
    try {
615
      $msbuildPath = InitializeVisualStudioMSBuild -install:$restore
616
    } catch {
617
      Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_
618
      ExitWithExitCode 1
619
    }
620

621
    $buildTool = @{ Path = $msbuildPath; Command = ""; Tool = "vs"; Framework = "net472"; ExcludePrereleaseVS = $excludePrereleaseVS }
622
  } else {
623
    Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Unexpected value of -msbuildEngine: '$msbuildEngine'."
624
    ExitWithExitCode 1
625
  }
626

627
  return $global:_BuildTool = $buildTool
628
}
629

630
function GetDefaultMSBuildEngine() {
631
  # Presence of tools.vs indicates the repo needs to build using VS msbuild on Windows.
632
  if (Get-Member -InputObject $GlobalJson.tools -Name 'vs') {
633
    return 'vs'
634
  }
635

636
  if (Get-Member -InputObject $GlobalJson.tools -Name 'dotnet') {
637
    return 'dotnet'
638
  }
639

640
  Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "-msbuildEngine must be specified, or /global.json must specify 'tools.dotnet' or 'tools.vs'."
641
  ExitWithExitCode 1
642
}
643

644
function GetNuGetPackageCachePath() {
645
  if ($env:NUGET_PACKAGES -eq $null) {
646
    # Use local cache on CI to ensure deterministic build.
647
    # Avoid using the http cache as workaround for https://github.com/NuGet/Home/issues/3116
648
    # use global cache in dev builds to avoid cost of downloading packages.
649
    # For directory normalization, see also: https://github.com/NuGet/Home/issues/7968
650
    if ($useGlobalNuGetCache) {
651
      $env:NUGET_PACKAGES = Join-Path $env:UserProfile '.nuget\packages\'
652
    } else {
653
      $env:NUGET_PACKAGES = Join-Path $RepoRoot '.packages\'
654
      $env:RESTORENOCACHE = $true
655
    }
656
  }
657

658
  return $env:NUGET_PACKAGES
659
}
660

661
# Returns a full path to an Arcade SDK task project file.
662
function GetSdkTaskProject([string]$taskName) {
663
  return Join-Path (Split-Path (InitializeToolset) -Parent) "SdkTasks\$taskName.proj"
664
}
665

666
function InitializeNativeTools() {
667
  if (-Not (Test-Path variable:DisableNativeToolsetInstalls) -And (Get-Member -InputObject $GlobalJson -Name "native-tools")) {
668
    $nativeArgs= @{}
669
    if ($ci) {
670
      $nativeArgs = @{
671
        InstallDirectory = "$ToolsDir"
672
      }
673
    }
674
    if ($env:NativeToolsOnMachine) {
675
      Write-Host "Variable NativeToolsOnMachine detected, enabling native tool path promotion..."
676
      $nativeArgs += @{ PathPromotion = $true }
677
    }
678
    & "$PSScriptRoot/init-tools-native.ps1" @nativeArgs
679
  }
680
}
681

682
function Read-ArcadeSdkVersion() {
683
  return $GlobalJson.'msbuild-sdks'.'Microsoft.DotNet.Arcade.Sdk'
684
}
685

686
function InitializeToolset() {
687
  if (Test-Path variable:global:_ToolsetBuildProj) {
688
    return $global:_ToolsetBuildProj
689
  }
690

691
  $nugetCache = GetNuGetPackageCachePath
692

693
  $toolsetVersion = Read-ArcadeSdkVersion
694
  $toolsetLocationFile = Join-Path $ToolsetDir "$toolsetVersion.txt"
695

696
  if (Test-Path $toolsetLocationFile) {
697
    $path = Get-Content $toolsetLocationFile -TotalCount 1
698
    if (Test-Path $path) {
699
      return $global:_ToolsetBuildProj = $path
700
    }
701
  }
702

703
  if (-not $restore) {
704
    Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Toolset version $toolsetVersion has not been restored."
705
    ExitWithExitCode 1
706
  }
707

708
  $buildTool = InitializeBuildTool
709

710
  $proj = Join-Path $ToolsetDir 'restore.proj'
711
  $bl = if ($binaryLog) { '/bl:' + (Join-Path $LogDir 'ToolsetRestore.binlog') } else { '' }
712

713
  '<Project Sdk="Microsoft.DotNet.Arcade.Sdk"/>' | Set-Content $proj
714

715
  MSBuild-Core $proj $bl /t:__WriteToolsetLocation /clp:ErrorsOnly`;NoSummary /p:__ToolsetLocationOutputFile=$toolsetLocationFile
716

717
  $path = Get-Content $toolsetLocationFile -Encoding UTF8 -TotalCount 1
718
  if (!(Test-Path $path)) {
719
    throw "Invalid toolset path: $path"
720
  }
721

722
  return $global:_ToolsetBuildProj = $path
723
}
724

725
function ExitWithExitCode([int] $exitCode) {
726
  if ($ci -and $prepareMachine) {
727
    Stop-Processes
728
  }
729
  exit $exitCode
730
}
731

732
# Check if $LASTEXITCODE is a nonzero exit code (NZEC). If so, print a Azure Pipeline error for
733
# diagnostics, then exit the script with the $LASTEXITCODE.
734
function Exit-IfNZEC([string] $category = "General") {
735
  Write-Host "Exit code $LASTEXITCODE"
736
  if ($LASTEXITCODE -ne 0) {
737
    $message = "Last command failed with exit code $LASTEXITCODE."
738
    Write-PipelineTelemetryError -Force -Category $category -Message $message
739
    ExitWithExitCode $LASTEXITCODE
740
  }
741
}
742

743
function Stop-Processes() {
744
  Write-Host 'Killing running build processes...'
745
  foreach ($processName in $processesToStopOnExit) {
746
    Get-Process -Name $processName -ErrorAction SilentlyContinue | Stop-Process
747
  }
748
}
749

750
#
751
# Executes msbuild (or 'dotnet msbuild') with arguments passed to the function.
752
# The arguments are automatically quoted.
753
# Terminates the script if the build fails.
754
#
755
function MSBuild() {
756
  if ($pipelinesLog) {
757
    $buildTool = InitializeBuildTool
758

759
    if ($ci -and $buildTool.Tool -eq 'dotnet') {
760
      $env:NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS = 20
761
      $env:NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS = 20
762
      Write-PipelineSetVariable -Name 'NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS' -Value '20'
763
      Write-PipelineSetVariable -Name 'NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS' -Value '20'
764
    }
765

766
    Enable-Nuget-EnhancedRetry
767

768
    $toolsetBuildProject = InitializeToolset
769
    $basePath = Split-Path -parent $toolsetBuildProject
770
    $possiblePaths = @(
771
      # new scripts need to work with old packages, so we need to look for the old names/versions
772
      (Join-Path $basePath (Join-Path $buildTool.Framework 'Microsoft.DotNet.ArcadeLogging.dll')),
773
      (Join-Path $basePath (Join-Path $buildTool.Framework 'Microsoft.DotNet.Arcade.Sdk.dll')),
774
      (Join-Path $basePath (Join-Path netcoreapp2.1 'Microsoft.DotNet.ArcadeLogging.dll')),
775
      (Join-Path $basePath (Join-Path netcoreapp2.1 'Microsoft.DotNet.Arcade.Sdk.dll'))
776
      (Join-Path $basePath (Join-Path netcoreapp3.1 'Microsoft.DotNet.ArcadeLogging.dll')),
777
      (Join-Path $basePath (Join-Path netcoreapp3.1 'Microsoft.DotNet.Arcade.Sdk.dll'))
778
      (Join-Path $basePath (Join-Path net7.0 'Microsoft.DotNet.ArcadeLogging.dll')),
779
      (Join-Path $basePath (Join-Path net7.0 'Microsoft.DotNet.Arcade.Sdk.dll'))
780
    )
781
    $selectedPath = $null
782
    foreach ($path in $possiblePaths) {
783
      if (Test-Path $path -PathType Leaf) {
784
        $selectedPath = $path
785
        break
786
      }
787
    }
788
    if (-not $selectedPath) {
789
      Write-PipelineTelemetryError -Category 'Build' -Message 'Unable to find arcade sdk logger assembly.'
790
      ExitWithExitCode 1
791
    }
792
    $args += "/logger:$selectedPath"
793
  }
794

795
  MSBuild-Core @args
796
}
797

798
#
799
# Executes msbuild (or 'dotnet msbuild') with arguments passed to the function.
800
# The arguments are automatically quoted.
801
# Terminates the script if the build fails.
802
#
803
function MSBuild-Core() {
804
  if ($ci) {
805
    if (!$binaryLog -and !$excludeCIBinarylog) {
806
      Write-PipelineTelemetryError -Category 'Build' -Message 'Binary log must be enabled in CI build, or explicitly opted-out from with the -excludeCIBinarylog switch.'
807
      ExitWithExitCode 1
808
    }
809

810
    if ($nodeReuse) {
811
      Write-PipelineTelemetryError -Category 'Build' -Message 'Node reuse must be disabled in CI build.'
812
      ExitWithExitCode 1
813
    }
814
  }
815

816
  Enable-Nuget-EnhancedRetry
817

818
  $buildTool = InitializeBuildTool
819

820
  $cmdArgs = "$($buildTool.Command) /m /nologo /clp:Summary /v:$verbosity /nr:$nodeReuse /p:ContinuousIntegrationBuild=$ci"
821

822
  if ($warnAsError) {
823
    $cmdArgs += ' /warnaserror /p:TreatWarningsAsErrors=true'
824
  }
825
  else {
826
    $cmdArgs += ' /p:TreatWarningsAsErrors=false'
827
  }
828

829
  foreach ($arg in $args) {
830
    if ($null -ne $arg -and $arg.Trim() -ne "") {
831
      if ($arg.EndsWith('\')) {
832
        $arg = $arg + "\"
833
      }
834
      $cmdArgs += " `"$arg`""
835
    }
836
  }
837

838
  $env:ARCADE_BUILD_TOOL_COMMAND = "$($buildTool.Path) $cmdArgs"
839

840
  $exitCode = Exec-Process $buildTool.Path $cmdArgs
841

842
  if ($exitCode -ne 0) {
843
    # We should not Write-PipelineTaskError here because that message shows up in the build summary
844
    # The build already logged an error, that's the reason it failed. Producing an error here only adds noise.
845
    Write-Host "Build failed with exit code $exitCode. Check errors above." -ForegroundColor Red
846

847
    $buildLog = GetMSBuildBinaryLogCommandLineArgument $args
848
    if ($null -ne $buildLog) {
849
      Write-Host "See log: $buildLog" -ForegroundColor DarkGray
850
    }
851

852
    # When running on Azure Pipelines, override the returned exit code to avoid double logging.
853
    if ($ci -and $env:SYSTEM_TEAMPROJECT -ne $null) {
854
      Write-PipelineSetResult -Result "Failed" -Message "msbuild execution failed."
855
      # Exiting with an exit code causes the azure pipelines task to log yet another "noise" error
856
      # The above Write-PipelineSetResult will cause the task to be marked as failure without adding yet another error
857
      ExitWithExitCode 0
858
    } else {
859
      ExitWithExitCode $exitCode
860
    }
861
  }
862
}
863

864
function GetMSBuildBinaryLogCommandLineArgument($arguments) {
865
  foreach ($argument in $arguments) {
866
    if ($argument -ne $null) {
867
      $arg = $argument.Trim()
868
      if ($arg.StartsWith('/bl:', "OrdinalIgnoreCase")) {
869
        return $arg.Substring('/bl:'.Length)
870
      }
871

872
      if ($arg.StartsWith('/binaryLogger:', 'OrdinalIgnoreCase')) {
873
        return $arg.Substring('/binaryLogger:'.Length)
874
      }
875
    }
876
  }
877

878
  return $null
879
}
880

881
function GetExecutableFileName($baseName) {
882
  if (IsWindowsPlatform) {
883
    return "$baseName.exe"
884
  }
885
  else {
886
    return $baseName
887
  }
888
}
889

890
function IsWindowsPlatform() {
891
  return [environment]::OSVersion.Platform -eq [PlatformID]::Win32NT
892
}
893

894
function Get-Darc($version) {
895
  $darcPath  = "$TempDir\darc\$(New-Guid)"
896
  if ($version -ne $null) {
897
    & $PSScriptRoot\darc-init.ps1 -toolpath $darcPath -darcVersion $version | Out-Host
898
  } else {
899
    & $PSScriptRoot\darc-init.ps1 -toolpath $darcPath | Out-Host
900
  }
901
  return "$darcPath\darc.exe"
902
}
903

904
. $PSScriptRoot\pipeline-logging-functions.ps1
905

906
$RepoRoot = Resolve-Path (Join-Path $PSScriptRoot '..\..\')
907
$EngRoot = Resolve-Path (Join-Path $PSScriptRoot '..')
908
$ArtifactsDir = Join-Path $RepoRoot 'artifacts'
909
$ToolsetDir = Join-Path $ArtifactsDir 'toolset'
910
$ToolsDir = Join-Path $RepoRoot '.tools'
911
$LogDir = Join-Path (Join-Path $ArtifactsDir 'log') $configuration
912
$TempDir = Join-Path (Join-Path $ArtifactsDir 'tmp') $configuration
913
$GlobalJson = Get-Content -Raw -Path (Join-Path $RepoRoot 'global.json') | ConvertFrom-Json
914
# true if global.json contains a "runtimes" section
915
$globalJsonHasRuntimes = if ($GlobalJson.tools.PSObject.Properties.Name -Match 'runtimes') { $true } else { $false }
916

917
Create-Directory $ToolsetDir
918
Create-Directory $TempDir
919
Create-Directory $LogDir
920

921
Write-PipelineSetVariable -Name 'Artifacts' -Value $ArtifactsDir
922
Write-PipelineSetVariable -Name 'Artifacts.Toolset' -Value $ToolsetDir
923
Write-PipelineSetVariable -Name 'Artifacts.Log' -Value $LogDir
924
Write-PipelineSetVariable -Name 'TEMP' -Value $TempDir
925
Write-PipelineSetVariable -Name 'TMP' -Value $TempDir
926

927
# Import custom tools configuration, if present in the repo.
928
# Note: Import in global scope so that the script set top-level variables without qualification.
929
if (!$disableConfigureToolsetImport) {
930
  $configureToolsetScript = Join-Path $EngRoot 'configure-toolset.ps1'
931
  if (Test-Path $configureToolsetScript) {
932
    . $configureToolsetScript
933
    if ((Test-Path variable:failOnConfigureToolsetError) -And $failOnConfigureToolsetError) {
934
      if ((Test-Path variable:LastExitCode) -And ($LastExitCode -ne 0)) {
935
        Write-PipelineTelemetryError -Category 'Build' -Message 'configure-toolset.ps1 returned a non-zero exit code'
936
        ExitWithExitCode $LastExitCode
937
      }
938
    }
939
  }
940
}
941

942
#
943
# If $ci flag is set, turn on (and log that we did) special environment variables for improved Nuget client retry logic.
944
#
945
function Enable-Nuget-EnhancedRetry() {
946
    if ($ci) {
947
      Write-Host "Setting NUGET enhanced retry environment variables"
948
      $env:NUGET_ENABLE_ENHANCED_HTTP_RETRY = 'true'
949
      $env:NUGET_ENHANCED_MAX_NETWORK_TRY_COUNT = 6
950
      $env:NUGET_ENHANCED_NETWORK_RETRY_DELAY_MILLISECONDS = 1000
951
      $env:NUGET_RETRY_HTTP_429 = 'true'
952
      Write-PipelineSetVariable -Name 'NUGET_ENABLE_ENHANCED_HTTP_RETRY' -Value 'true'
953
      Write-PipelineSetVariable -Name 'NUGET_ENHANCED_MAX_NETWORK_TRY_COUNT' -Value '6'
954
      Write-PipelineSetVariable -Name 'NUGET_ENHANCED_NETWORK_RETRY_DELAY_MILLISECONDS' -Value '1000'
955
      Write-PipelineSetVariable -Name 'NUGET_RETRY_HTTP_429' -Value 'true'
956
    }
957
}
958