msbuild

1
param(
2
  [Parameter(Mandatory = $true)][string] $InputPath, # Full path to directory where NuGet packages to be checked are stored
3
  [Parameter(Mandatory = $true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation
4
  [Parameter(Mandatory = $true)][string] $DotnetSymbolVersion, # Version of dotnet symbol to use
5
  [Parameter(Mandatory = $false)][switch] $CheckForWindowsPdbs, # If we should check for the existence of windows pdbs in addition to portable PDBs
6
  [Parameter(Mandatory = $false)][switch] $ContinueOnError, # If we should keep checking symbols after an error
7
  [Parameter(Mandatory = $false)][switch] $Clean,           # Clean extracted symbols directory after checking symbols
8
  [Parameter(Mandatory = $false)][string] $SymbolExclusionFile  # Exclude the symbols in the file from publishing to symbol server
9
)
10

11
. $PSScriptRoot\..\tools.ps1
12
# Maximum number of jobs to run in parallel
13
$MaxParallelJobs = 16
14

15
# Max number of retries
16
$MaxRetry = 5
17

18
# Wait time between check for system load
19
$SecondsBetweenLoadChecks = 10
20

21
# Set error codes
22
Set-Variable -Name "ERROR_BADEXTRACT" -Option Constant -Value -1
23
Set-Variable -Name "ERROR_FILEDOESNOTEXIST" -Option Constant -Value -2
24

25
$WindowsPdbVerificationParam = ""
26
if ($CheckForWindowsPdbs) {
27
  $WindowsPdbVerificationParam = "--windows-pdbs"
28
}
29

30
$ExclusionSet = New-Object System.Collections.Generic.HashSet[string];
31

32
if (!$InputPath -or !(Test-Path $InputPath)){
33
  Write-Host "No symbols to validate."
34
  ExitWithExitCode 0
35
}
36

37
#Check if the path exists
38
if ($SymbolExclusionFile -and (Test-Path $SymbolExclusionFile)){
39
  [string[]]$Exclusions = Get-Content "$SymbolExclusionFile"
40
  $Exclusions | foreach { if($_ -and $_.Trim()){$ExclusionSet.Add($_)} }
41
}
42
else{
43
  Write-Host "Symbol Exclusion file does not exists. No symbols to exclude."
44
}
45

46
$CountMissingSymbols = {
47
  param( 
48
    [string] $PackagePath, # Path to a NuGet package
49
    [string] $WindowsPdbVerificationParam # If we should check for the existence of windows pdbs in addition to portable PDBs
50
  )
51

52
  Add-Type -AssemblyName System.IO.Compression.FileSystem
53

54
  Write-Host "Validating $PackagePath "
55

56
  # Ensure input file exist
57
  if (!(Test-Path $PackagePath)) {
58
    Write-PipelineTaskError "Input file does not exist: $PackagePath"
59
    return [pscustomobject]@{
60
      result      = $using:ERROR_FILEDOESNOTEXIST
61
      packagePath = $PackagePath
62
    }
63
  }
64
  
65
  # Extensions for which we'll look for symbols
66
  $RelevantExtensions = @('.dll', '.exe', '.so', '.dylib')
67

68
  # How many files are missing symbol information
69
  $MissingSymbols = 0
70

71
  $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath)
72
  $PackageGuid = New-Guid
73
  $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageGuid
74
  $SymbolsPath = Join-Path -Path $ExtractPath -ChildPath 'Symbols'
75
  
76
  try {
77
    [System.IO.Compression.ZipFile]::ExtractToDirectory($PackagePath, $ExtractPath)
78
  }
79
  catch {
80
    Write-Host "Something went wrong extracting $PackagePath"
81
    Write-Host $_
82
    return [pscustomobject]@{
83
      result      = $using:ERROR_BADEXTRACT
84
      packagePath = $PackagePath
85
    }
86
  }
87

88
  Get-ChildItem -Recurse $ExtractPath |
89
  Where-Object { $RelevantExtensions -contains $_.Extension } |
90
  ForEach-Object {
91
    $FileName = $_.FullName
92
    if ($FileName -Match '\\ref\\') {
93
      Write-Host "`t Ignoring reference assembly file " $FileName
94
      return
95
    }
96

97
    $FirstMatchingSymbolDescriptionOrDefault = {
98
      param( 
99
        [string] $FullPath, # Full path to the module that has to be checked
100
        [string] $TargetServerParam, # Parameter to pass to `Symbol Tool` indicating the server to lookup for symbols
101
        [string] $WindowsPdbVerificationParam, # Parameter to pass to potential check for windows-pdbs.
102
        [string] $SymbolsPath
103
      )
104

105
      $FileName = [System.IO.Path]::GetFileName($FullPath)
106
      $Extension = [System.IO.Path]::GetExtension($FullPath)
107

108
      # Those below are potential symbol files that the `dotnet symbol` might
109
      # return. Which one will be returned depend on the type of file we are
110
      # checking and which type of file was uploaded.
111

112
      # The file itself is returned
113
      $SymbolPath = $SymbolsPath + '\' + $FileName
114

115
      # PDB file for the module
116
      $PdbPath = $SymbolPath.Replace($Extension, '.pdb')
117

118
      # PDB file for R2R module (created by crossgen)
119
      $NGenPdb = $SymbolPath.Replace($Extension, '.ni.pdb')
120

121
      # DBG file for a .so library
122
      $SODbg = $SymbolPath.Replace($Extension, '.so.dbg')
123

124
      # DWARF file for a .dylib
125
      $DylibDwarf = $SymbolPath.Replace($Extension, '.dylib.dwarf')
126

127
      $dotnetSymbolExe = "$env:USERPROFILE\.dotnet\tools"
128
      $dotnetSymbolExe = Resolve-Path "$dotnetSymbolExe\dotnet-symbol.exe"
129

130
      $totalRetries = 0
131

132
      while ($totalRetries -lt $using:MaxRetry) {
133

134
        # Save the output and get diagnostic output
135
        $output = & $dotnetSymbolExe --symbols --modules $WindowsPdbVerificationParam $TargetServerParam $FullPath -o $SymbolsPath --diagnostics | Out-String
136

137
        if ((Test-Path $PdbPath) -and (Test-path $SymbolPath)) {
138
          return 'Module and PDB for Module'
139
        }
140
        elseif ((Test-Path $NGenPdb) -and (Test-Path $PdbPath) -and (Test-Path $SymbolPath)) {
141
          return 'Dll, PDB and NGen PDB'
142
        }
143
        elseif ((Test-Path $SODbg) -and (Test-Path $SymbolPath)) {
144
          return 'So and DBG for SO'
145
        }  
146
        elseif ((Test-Path $DylibDwarf) -and (Test-Path $SymbolPath)) {
147
          return 'Dylib and Dwarf for Dylib'
148
        }  
149
        elseif (Test-Path $SymbolPath) {
150
          return 'Module'
151
        }
152
        else
153
        {
154
          $totalRetries++
155
        }
156
      }
157
      
158
      return $null
159
    }
160

161
    $FileRelativePath = $FileName.Replace("$ExtractPath\", "")
162
    if (($($using:ExclusionSet) -ne $null) -and ($($using:ExclusionSet).Contains($FileRelativePath) -or ($($using:ExclusionSet).Contains($FileRelativePath.Replace("\", "/"))))){
163
      Write-Host "Skipping $FileName from symbol validation"
164
    }
165

166
    else {
167
      $FileGuid = New-Guid
168
      $ExpandedSymbolsPath = Join-Path -Path $SymbolsPath -ChildPath $FileGuid
169

170
      $SymbolsOnMSDL = & $FirstMatchingSymbolDescriptionOrDefault `
171
          -FullPath $FileName `
172
          -TargetServerParam '--microsoft-symbol-server' `
173
          -SymbolsPath "$ExpandedSymbolsPath-msdl" `
174
          -WindowsPdbVerificationParam $WindowsPdbVerificationParam
175
      $SymbolsOnSymWeb = & $FirstMatchingSymbolDescriptionOrDefault `
176
          -FullPath $FileName `
177
          -TargetServerParam '--internal-server' `
178
          -SymbolsPath "$ExpandedSymbolsPath-symweb" `
179
          -WindowsPdbVerificationParam $WindowsPdbVerificationParam
180

181
      Write-Host -NoNewLine "`t Checking file " $FileName "... "
182
  
183
      if ($SymbolsOnMSDL -ne $null -and $SymbolsOnSymWeb -ne $null) {
184
        Write-Host "Symbols found on MSDL ($SymbolsOnMSDL) and SymWeb ($SymbolsOnSymWeb)"
185
      }
186
      else {
187
        $MissingSymbols++
188

189
        if ($SymbolsOnMSDL -eq $null -and $SymbolsOnSymWeb -eq $null) {
190
          Write-Host 'No symbols found on MSDL or SymWeb!'
191
        }
192
        else {
193
          if ($SymbolsOnMSDL -eq $null) {
194
            Write-Host 'No symbols found on MSDL!'
195
          }
196
          else {
197
            Write-Host 'No symbols found on SymWeb!'
198
          }
199
        }
200
      }
201
    }
202
  }
203
  
204
  if ($using:Clean) {
205
    Remove-Item $ExtractPath -Recurse -Force
206
  }
207
  
208
  Pop-Location
209

210
  return [pscustomobject]@{
211
    result      = $MissingSymbols
212
    packagePath = $PackagePath
213
  }
214
}
215

216
function CheckJobResult(
217
  $result, 
218
  $packagePath,
219
  [ref]$DupedSymbols,
220
  [ref]$TotalFailures) {
221
  if ($result -eq $ERROR_BADEXTRACT) {
222
    Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "$packagePath has duplicated symbol files"
223
    $DupedSymbols.Value++
224
  } 
225
  elseif ($result -eq $ERROR_FILEDOESNOTEXIST) {
226
    Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "$packagePath does not exist"
227
    $TotalFailures.Value++
228
  }
229
  elseif ($result -gt '0') {
230
    Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Missing symbols for $result modules in the package $packagePath"
231
    $TotalFailures.Value++
232
  }
233
  else {
234
    Write-Host "All symbols verified for package $packagePath"
235
  }
236
}
237

238
function CheckSymbolsAvailable {
239
  if (Test-Path $ExtractPath) {
240
    Remove-Item $ExtractPath -Force  -Recurse -ErrorAction SilentlyContinue
241
  }
242

243
  $TotalPackages = 0
244
  $TotalFailures = 0
245
  $DupedSymbols = 0
246

247
  Get-ChildItem "$InputPath\*.nupkg" |
248
    ForEach-Object {
249
      $FileName = $_.Name
250
      $FullName = $_.FullName
251

252
      # These packages from Arcade-Services include some native libraries that
253
      # our current symbol uploader can't handle. Below is a workaround until
254
      # we get issue: https://github.com/dotnet/arcade/issues/2457 sorted.
255
      if ($FileName -Match 'Microsoft\.DotNet\.Darc\.') {
256
        Write-Host "Ignoring Arcade-services file: $FileName"
257
        Write-Host
258
        return
259
      }
260
      elseif ($FileName -Match 'Microsoft\.DotNet\.Maestro\.Tasks\.') {
261
        Write-Host "Ignoring Arcade-services file: $FileName"
262
        Write-Host
263
        return
264
      }
265

266
      $TotalPackages++
267

268
      Start-Job -ScriptBlock $CountMissingSymbols -ArgumentList @($FullName,$WindowsPdbVerificationParam) | Out-Null
269

270
      $NumJobs = @(Get-Job -State 'Running').Count
271

272
      while ($NumJobs -ge $MaxParallelJobs) {
273
        Write-Host "There are $NumJobs validation jobs running right now. Waiting $SecondsBetweenLoadChecks seconds to check again."
274
        sleep $SecondsBetweenLoadChecks
275
        $NumJobs = @(Get-Job -State 'Running').Count
276
      }
277

278
      foreach ($Job in @(Get-Job -State 'Completed')) {
279
        $jobResult = Wait-Job -Id $Job.Id | Receive-Job
280
        CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$DupedSymbols) ([ref]$TotalFailures)
281
        Remove-Job -Id $Job.Id
282
      }
283
      Write-Host
284
    }
285

286
  foreach ($Job in @(Get-Job)) {
287
    $jobResult = Wait-Job -Id $Job.Id | Receive-Job
288
    CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$DupedSymbols) ([ref]$TotalFailures)
289
  }
290

291
  if ($TotalFailures -gt 0 -or $DupedSymbols -gt 0) {
292
    if ($TotalFailures -gt 0) {
293
      Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Symbols missing for $TotalFailures/$TotalPackages packages"
294
    }
295

296
    if ($DupedSymbols -gt 0) {
297
      Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "$DupedSymbols/$TotalPackages packages had duplicated symbol files and could not be extracted"
298
    }
299
    
300
    ExitWithExitCode 1
301
  }
302
  else {
303
    Write-Host "All symbols validated!"
304
  }
305
}
306

307
function InstallDotnetSymbol {
308
  $dotnetSymbolPackageName = 'dotnet-symbol'
309

310
  $dotnetRoot = InitializeDotNetCli -install:$true
311
  $dotnet = "$dotnetRoot\dotnet.exe"
312
  $toolList = & "$dotnet" tool list --global
313

314
  if (($toolList -like "*$dotnetSymbolPackageName*") -and ($toolList -like "*$dotnetSymbolVersion*")) {
315
    Write-Host "dotnet-symbol version $dotnetSymbolVersion is already installed."
316
  }
317
  else {
318
    Write-Host "Installing dotnet-symbol version $dotnetSymbolVersion..."
319
    Write-Host 'You may need to restart your command window if this is the first dotnet tool you have installed.'
320
    & "$dotnet" tool install $dotnetSymbolPackageName --version $dotnetSymbolVersion --verbosity "minimal" --global
321
  }
322
}
323

324
try {
325
  . $PSScriptRoot\post-build-utils.ps1
326
  
327
  InstallDotnetSymbol
328

329
  foreach ($Job in @(Get-Job)) {
330
    Remove-Job -Id $Job.Id
331
  }
332

333
  CheckSymbolsAvailable
334
}
335
catch {
336
  Write-Host $_.ScriptStackTrace
337
  Write-PipelineTelemetryError -Category 'CheckSymbols' -Message $_
338
  ExitWithExitCode 1
339
}
340