msbuild

1
param(
2
  [Parameter(Mandatory=$true)][string] $InputPath,              # Full path to directory where Symbols.NuGet packages to be checked are stored
3
  [Parameter(Mandatory=$true)][string] $ExtractPath,            # Full path to directory where the packages will be extracted during validation
4
  [Parameter(Mandatory=$false)][string] $GHRepoName,            # GitHub name of the repo including the Org. E.g., dotnet/arcade
5
  [Parameter(Mandatory=$false)][string] $GHCommit,              # GitHub commit SHA used to build the packages
6
  [Parameter(Mandatory=$true)][string] $SourcelinkCliVersion    # Version of SourceLink CLI to use
7
)
8

9
. $PSScriptRoot\post-build-utils.ps1
10

11
# Cache/HashMap (File -> Exist flag) used to consult whether a file exist 
12
# in the repository at a specific commit point. This is populated by inserting
13
# all files present in the repo at a specific commit point.
14
$global:RepoFiles = @{}
15

16
# Maximum number of jobs to run in parallel
17
$MaxParallelJobs = 16
18

19
$MaxRetries = 5
20
$RetryWaitTimeInSeconds = 30
21

22
# Wait time between check for system load
23
$SecondsBetweenLoadChecks = 10
24

25
if (!$InputPath -or !(Test-Path $InputPath)){
26
  Write-Host "No files to validate."
27
  ExitWithExitCode 0
28
}
29

30
$ValidatePackage = {
31
  param( 
32
    [string] $PackagePath                                 # Full path to a Symbols.NuGet package
33
  )
34

35
  . $using:PSScriptRoot\..\tools.ps1
36

37
  # Ensure input file exist
38
  if (!(Test-Path $PackagePath)) {
39
    Write-Host "Input file does not exist: $PackagePath"
40
    return [pscustomobject]@{
41
      result = 1
42
      packagePath = $PackagePath
43
    }
44
  }
45

46
  # Extensions for which we'll look for SourceLink information
47
  # For now we'll only care about Portable & Embedded PDBs
48
  $RelevantExtensions = @('.dll', '.exe', '.pdb')
49
 
50
  Write-Host -NoNewLine 'Validating ' ([System.IO.Path]::GetFileName($PackagePath)) '...'
51

52
  $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath)
53
  $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageId
54
  $FailedFiles = 0
55

56
  Add-Type -AssemblyName System.IO.Compression.FileSystem
57

58
  [System.IO.Directory]::CreateDirectory($ExtractPath)  | Out-Null
59

60
  try {
61
    $zip = [System.IO.Compression.ZipFile]::OpenRead($PackagePath)
62

63
    $zip.Entries | 
64
      Where-Object {$RelevantExtensions -contains [System.IO.Path]::GetExtension($_.Name)} |
65
        ForEach-Object {
66
          $FileName = $_.FullName
67
          $Extension = [System.IO.Path]::GetExtension($_.Name)
68
          $FakeName = -Join((New-Guid), $Extension)
69
          $TargetFile = Join-Path -Path $ExtractPath -ChildPath $FakeName 
70

71
          # We ignore resource DLLs
72
          if ($FileName.EndsWith('.resources.dll')) {
73
            return [pscustomobject]@{
74
              result = 0
75
              packagePath = $PackagePath
76
            }
77
          }
78

79
          [System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile, $true)
80

81
          $ValidateFile = {
82
            param( 
83
              [string] $FullPath,                                # Full path to the module that has to be checked
84
              [string] $RealPath,
85
              [ref] $FailedFiles
86
            )
87

88
            $sourcelinkExe = "$env:USERPROFILE\.dotnet\tools"
89
            $sourcelinkExe = Resolve-Path "$sourcelinkExe\sourcelink.exe"
90
            $SourceLinkInfos = & $sourcelinkExe print-urls $FullPath | Out-String
91

92
            if ($LASTEXITCODE -eq 0 -and -not ([string]::IsNullOrEmpty($SourceLinkInfos))) {
93
              $NumFailedLinks = 0
94

95
              # We only care about Http addresses
96
              $Matches = (Select-String '(http[s]?)(:\/\/)([^\s,]+)' -Input $SourceLinkInfos -AllMatches).Matches
97

98
              if ($Matches.Count -ne 0) {
99
                $Matches.Value |
100
                  ForEach-Object {
101
                    $Link = $_
102
                    $CommitUrl = "https://raw.githubusercontent.com/${using:GHRepoName}/${using:GHCommit}/"
103
                    
104
                    $FilePath = $Link.Replace($CommitUrl, "")
105
                    $Status = 200
106
                    $Cache = $using:RepoFiles
107

108
                    $attempts = 0
109

110
                    while ($attempts -lt $using:MaxRetries) {
111
                      if ( !($Cache.ContainsKey($FilePath)) ) {
112
                        try {
113
                          $Uri = $Link -as [System.URI]
114
                        
115
                          if ($Link -match "submodules") {
116
                            # Skip submodule links until sourcelink properly handles submodules
117
                            $Status = 200
118
                          }
119
                          elseif ($Uri.AbsoluteURI -ne $null -and ($Uri.Host -match 'github' -or $Uri.Host -match 'githubusercontent')) {
120
                            # Only GitHub links are valid
121
                            $Status = (Invoke-WebRequest -Uri $Link -UseBasicParsing -Method HEAD -TimeoutSec 5).StatusCode
122
                          }
123
                          else {
124
                            # If it's not a github link, we want to break out of the loop and not retry.
125
                            $Status = 0
126
                            $attempts = $using:MaxRetries
127
                          }
128
                        }
129
                        catch {
130
                          Write-Host $_
131
                          $Status = 0
132
                        }
133
                      }
134

135
                      if ($Status -ne 200) {
136
                        $attempts++
137
                        
138
                        if  ($attempts -lt $using:MaxRetries)
139
                        {
140
                          $attemptsLeft = $using:MaxRetries - $attempts
141
                          Write-Warning "Download failed, $attemptsLeft attempts remaining, will retry in $using:RetryWaitTimeInSeconds seconds"
142
                          Start-Sleep -Seconds $using:RetryWaitTimeInSeconds
143
                        }
144
                        else {
145
                          if ($NumFailedLinks -eq 0) {
146
                            if ($FailedFiles.Value -eq 0) {
147
                              Write-Host
148
                            }
149
  
150
                            Write-Host "`tFile $RealPath has broken links:"
151
                          }
152
  
153
                          Write-Host "`t`tFailed to retrieve $Link"
154
  
155
                          $NumFailedLinks++
156
                        }
157
                      }
158
                      else {
159
                        break
160
                      }
161
                    }
162
                  }
163
              }
164

165
              if ($NumFailedLinks -ne 0) {
166
                $FailedFiles.value++
167
                $global:LASTEXITCODE = 1
168
              }
169
            }
170
          }
171
        
172
          &$ValidateFile $TargetFile $FileName ([ref]$FailedFiles)
173
        }
174
  }
175
  catch {
176
    Write-Host $_
177
  }
178
  finally {
179
    $zip.Dispose() 
180
  }
181

182
  if ($FailedFiles -eq 0) {
183
    Write-Host 'Passed.'
184
    return [pscustomobject]@{
185
      result = 0
186
      packagePath = $PackagePath
187
    }
188
  }
189
  else {
190
    Write-PipelineTelemetryError -Category 'SourceLink' -Message "$PackagePath has broken SourceLink links."
191
    return [pscustomobject]@{
192
      result = 1
193
      packagePath = $PackagePath
194
    }
195
  }
196
}
197

198
function CheckJobResult(
199
    $result, 
200
    $packagePath,
201
    [ref]$ValidationFailures,
202
    [switch]$logErrors) {
203
  if ($result -ne '0') {
204
    if ($logErrors) {
205
      Write-PipelineTelemetryError -Category 'SourceLink' -Message "$packagePath has broken SourceLink links."
206
    }
207
    $ValidationFailures.Value++
208
  }
209
}
210

211
function ValidateSourceLinkLinks {
212
  if ($GHRepoName -ne '' -and !($GHRepoName -Match '^[^\s\/]+/[^\s\/]+$')) {
213
    if (!($GHRepoName -Match '^[^\s-]+-[^\s]+$')) {
214
      Write-PipelineTelemetryError -Category 'SourceLink' -Message "GHRepoName should be in the format <org>/<repo> or <org>-<repo>. '$GHRepoName'"
215
      ExitWithExitCode 1
216
    }
217
    else {
218
      $GHRepoName = $GHRepoName -replace '^([^\s-]+)-([^\s]+)$', '$1/$2';
219
    }
220
  }
221

222
  if ($GHCommit -ne '' -and !($GHCommit -Match '^[0-9a-fA-F]{40}$')) {
223
    Write-PipelineTelemetryError -Category 'SourceLink' -Message "GHCommit should be a 40 chars hexadecimal string. '$GHCommit'"
224
    ExitWithExitCode 1
225
  }
226

227
  if ($GHRepoName -ne '' -and $GHCommit -ne '') {
228
    $RepoTreeURL = -Join('http://api.github.com/repos/', $GHRepoName, '/git/trees/', $GHCommit, '?recursive=1')
229
    $CodeExtensions = @('.cs', '.vb', '.fs', '.fsi', '.fsx', '.fsscript')
230

231
    try {
232
      # Retrieve the list of files in the repo at that particular commit point and store them in the RepoFiles hash
233
      $Data = Invoke-WebRequest $RepoTreeURL -UseBasicParsing | ConvertFrom-Json | Select-Object -ExpandProperty tree
234
  
235
      foreach ($file in $Data) {
236
        $Extension = [System.IO.Path]::GetExtension($file.path)
237

238
        if ($CodeExtensions.Contains($Extension)) {
239
          $RepoFiles[$file.path] = 1
240
        }
241
      }
242
    }
243
    catch {
244
      Write-Host "Problems downloading the list of files from the repo. Url used: $RepoTreeURL . Execution will proceed without caching."
245
    }
246
  }
247
  elseif ($GHRepoName -ne '' -or $GHCommit -ne '') {
248
    Write-Host 'For using the http caching mechanism both GHRepoName and GHCommit should be informed.'
249
  }
250
  
251
  if (Test-Path $ExtractPath) {
252
    Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue
253
  }
254

255
  $ValidationFailures = 0
256

257
  # Process each NuGet package in parallel
258
  Get-ChildItem "$InputPath\*.symbols.nupkg" |
259
    ForEach-Object {
260
      Write-Host "Starting $($_.FullName)"
261
      Start-Job -ScriptBlock $ValidatePackage -ArgumentList $_.FullName | Out-Null
262
      $NumJobs = @(Get-Job -State 'Running').Count
263
      
264
      while ($NumJobs -ge $MaxParallelJobs) {
265
        Write-Host "There are $NumJobs validation jobs running right now. Waiting $SecondsBetweenLoadChecks seconds to check again."
266
        sleep $SecondsBetweenLoadChecks
267
        $NumJobs = @(Get-Job -State 'Running').Count
268
      }
269

270
      foreach ($Job in @(Get-Job -State 'Completed')) {
271
        $jobResult = Wait-Job -Id $Job.Id | Receive-Job
272
        CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$ValidationFailures) -LogErrors
273
        Remove-Job -Id $Job.Id
274
      }
275
    }
276

277
  foreach ($Job in @(Get-Job)) {
278
    $jobResult = Wait-Job -Id $Job.Id | Receive-Job
279
    CheckJobResult $jobResult.result $jobResult.packagePath ([ref]$ValidationFailures)
280
    Remove-Job -Id $Job.Id
281
  }
282
  if ($ValidationFailures -gt 0) {
283
    Write-PipelineTelemetryError -Category 'SourceLink' -Message "$ValidationFailures package(s) failed validation."
284
    ExitWithExitCode 1
285
  }
286
}
287

288
function InstallSourcelinkCli {
289
  $sourcelinkCliPackageName = 'sourcelink'
290

291
  $dotnetRoot = InitializeDotNetCli -install:$true
292
  $dotnet = "$dotnetRoot\dotnet.exe"
293
  $toolList = & "$dotnet" tool list --global
294

295
  if (($toolList -like "*$sourcelinkCliPackageName*") -and ($toolList -like "*$sourcelinkCliVersion*")) {
296
    Write-Host "SourceLink CLI version $sourcelinkCliVersion is already installed."
297
  }
298
  else {
299
    Write-Host "Installing SourceLink CLI version $sourcelinkCliVersion..."
300
    Write-Host 'You may need to restart your command window if this is the first dotnet tool you have installed.'
301
    & "$dotnet" tool install $sourcelinkCliPackageName --version $sourcelinkCliVersion --verbosity "minimal" --global 
302
  }
303
}
304

305
try {
306
  InstallSourcelinkCli
307

308
  foreach ($Job in @(Get-Job)) {
309
    Remove-Job -Id $Job.Id
310
  }
311

312
  ValidateSourceLinkLinks 
313
}
314
catch {
315
  Write-Host $_.Exception
316
  Write-Host $_.ScriptStackTrace
317
  Write-PipelineTelemetryError -Category 'SourceLink' -Message $_
318
  ExitWithExitCode 1
319
}
320