diff --git a/eng/build-utils.ps1 b/eng/build-utils.ps1 index 61db1261098d5..b21c0a3872e72 100644 --- a/eng/build-utils.ps1 +++ b/eng/build-utils.ps1 @@ -25,7 +25,7 @@ function GetProjectOutputBinary([string]$fileName, [string]$projectName = "", [s function GetPublishData() { if (Test-Path variable:global:_PublishData) { - return $global:_PublishData + return $global:_PublishData } Write-Host "Downloading $PublishDataUrl" @@ -38,9 +38,9 @@ function GetBranchPublishData([string]$branchName) { $data = GetPublishData if (Get-Member -InputObject $data.branches -Name $branchName) { - return $data.branches.$branchName + return $data.branches.$branchName } else { - return $null + return $null } } @@ -48,9 +48,9 @@ function GetReleasePublishData([string]$releaseName) { $data = GetPublishData if (Get-Member -InputObject $data.releases -Name $releaseName) { - return $data.releases.$releaseName + return $data.releases.$releaseName } else { - return $null + return $null } } @@ -64,67 +64,67 @@ function GetReleasePublishData([string]$releaseName) { # # Original sample came from: http://jameskovacs.com/2010/02/25/the-exec-problem/ function Exec-Block([scriptblock]$cmd) { - & $cmd + & $cmd - # Need to check both of these cases for errors as they represent different items - # - $?: did the powershell script block throw an error - # - $lastexitcode: did a windows command executed by the script block end in error - if ((-not $?) -or ($lastexitcode -ne 0)) { - throw "Command failed to execute: $cmd" - } + # Need to check both of these cases for errors as they represent different items + # - $?: did the powershell script block throw an error + # - $lastexitcode: did a windows command executed by the script block end in error + if ((-not $?) -or ($lastexitcode -ne 0)) { + throw "Command failed to execute: $cmd" + } } function Exec-CommandCore([string]$command, [string]$commandArgs, [switch]$useConsole = $true) { - if ($useConsole) { - $exitCode = Exec-Process $command $commandArgs - if ($exitCode -ne 0) { - throw "Command failed to execute with exit code $($exitCode): $command $commandArgs" - } - return + if ($useConsole) { + $exitCode = Exec-Process $command $commandArgs + if ($exitCode -ne 0) { + throw "Command failed to execute with exit code $($exitCode): $command $commandArgs" } + return + } - $startInfo = New-Object System.Diagnostics.ProcessStartInfo - $startInfo.FileName = $command - $startInfo.Arguments = $commandArgs - - $startInfo.UseShellExecute = $false - $startInfo.WorkingDirectory = Get-Location - $startInfo.RedirectStandardOutput = $true - $startInfo.CreateNoWindow = $true + $startInfo = New-Object System.Diagnostics.ProcessStartInfo + $startInfo.FileName = $command + $startInfo.Arguments = $commandArgs + + $startInfo.UseShellExecute = $false + $startInfo.WorkingDirectory = Get-Location + $startInfo.RedirectStandardOutput = $true + $startInfo.CreateNoWindow = $true + + $process = New-Object System.Diagnostics.Process + $process.StartInfo = $startInfo + $process.Start() | Out-Null + + $finished = $false + try { + # The OutputDataReceived event doesn't fire as events are sent by the + # process in powershell. Possibly due to subtlties of how Powershell + # manages the thread pool that I'm not aware of. Using blocking + # reading here as an alternative which is fine since this blocks + # on completion already. + $out = $process.StandardOutput + while (-not $out.EndOfStream) { + $line = $out.ReadLine() + Write-Output $line + } - $process = New-Object System.Diagnostics.Process - $process.StartInfo = $startInfo - $process.Start() | Out-Null + while (-not $process.WaitForExit(100)) { + # Non-blocking loop done to allow ctr-c interrupts + } - $finished = $false - try { - # The OutputDataReceived event doesn't fire as events are sent by the - # process in powershell. Possibly due to subtlties of how Powershell - # manages the thread pool that I'm not aware of. Using blocking - # reading here as an alternative which is fine since this blocks - # on completion already. - $out = $process.StandardOutput - while (-not $out.EndOfStream) { - $line = $out.ReadLine() - Write-Output $line - } - - while (-not $process.WaitForExit(100)) { - # Non-blocking loop done to allow ctr-c interrupts - } - - $finished = $true - if ($process.ExitCode -ne 0) { - throw "Command failed to execute with exit code $($process.ExitCode): $command $commandArgs" - } + $finished = $true + if ($process.ExitCode -ne 0) { + throw "Command failed to execute with exit code $($process.ExitCode): $command $commandArgs" } - finally { - # If we didn't finish then an error occured or the user hit ctrl-c. Either - # way kill the process - if (-not $finished) { - $process.Kill() - } + } + finally { + # If we didn't finish then an error occured or the user hit ctrl-c. Either + # way kill the process + if (-not $finished) { + $process.Kill() } + } } # Handy function for executing a windows command which needs to go through @@ -137,7 +137,7 @@ function Exec-CommandCore([string]$command, [string]$commandArgs, [switch]$useCo # Exec-Command $msbuild $args # function Exec-Command([string]$command, [string]$commandArgs) { - Exec-CommandCore -command $command -commandArgs $commandargs -useConsole:$false + Exec-CommandCore -command $command -commandArgs $commandargs -useConsole:$false } # Functions exactly like Exec-Command but lets the process re-use the current @@ -147,14 +147,14 @@ function Exec-Command([string]$command, [string]$commandArgs) { # Exec-Command $msbuild $args | Out-Host # function Exec-Console([string]$command, [string]$commandArgs) { - Exec-CommandCore -command $command -commandArgs $commandargs -useConsole:$true + Exec-CommandCore -command $command -commandArgs $commandargs -useConsole:$true } # Handy function for executing a powershell script in a clean environment with # arguments. Prefer this over & sourcing a script as it will both use a clean # environment and do proper error checking function Exec-Script([string]$script, [string]$scriptArgs = "") { - Exec-Command "powershell" "-noprofile -executionPolicy RemoteSigned -file `"$script`" $scriptArgs" + Exec-Command "powershell" "-noprofile -executionPolicy RemoteSigned -file `"$script`" $scriptArgs" } # Ensure the proper .NET Core SDK is available. Returns the location to the dotnet.exe. @@ -163,104 +163,104 @@ function Ensure-DotnetSdk() { } function Get-VersionCore([string]$name, [string]$versionFile) { - $name = $name.Replace(".", "") - $name = $name.Replace("-", "") - $nodeName = "$($name)Version" - $x = [xml](Get-Content -raw $versionFile) - $node = $x.SelectSingleNode("//Project/PropertyGroup/$nodeName") - if ($node -ne $null) { - return $node.InnerText - } + $name = $name.Replace(".", "") + $name = $name.Replace("-", "") + $nodeName = "$($name)Version" + $x = [xml](Get-Content -raw $versionFile) + $node = $x.SelectSingleNode("//Project/PropertyGroup/$nodeName") + if ($node -ne $null) { + return $node.InnerText + } - throw "Cannot find package $name in $versionFile" + throw "Cannot find package $name in $versionFile" } # Return the version of the NuGet package as used in this repo function Get-PackageVersion([string]$name) { - return Get-VersionCore $name (Join-Path $EngRoot "Versions.props") + return Get-VersionCore $name (Join-Path $EngRoot "Versions.props") } # Locate the directory where our NuGet packages will be deployed. Needs to be kept in sync # with the logic in Version.props function Get-PackagesDir() { - $d = $null - if ($env:NUGET_PACKAGES -ne $null) { - $d = $env:NUGET_PACKAGES - } - else { - $d = Join-Path $env:UserProfile ".nuget\packages\" - } + $d = $null + if ($env:NUGET_PACKAGES -ne $null) { + $d = $env:NUGET_PACKAGES + } + else { + $d = Join-Path $env:UserProfile ".nuget\packages\" + } - Create-Directory $d - return $d + Create-Directory $d + return $d } # Locate the directory of a specific NuGet package which is restored via our main # toolset values. function Get-PackageDir([string]$name, [string]$version = "") { - if ($version -eq "") { - $version = Get-PackageVersion $name - } + if ($version -eq "") { + $version = Get-PackageVersion $name + } - $p = Get-PackagesDir - $p = Join-Path $p $name.ToLowerInvariant() - $p = Join-Path $p $version - return $p + $p = Get-PackagesDir + $p = Join-Path $p $name.ToLowerInvariant() + $p = Join-Path $p $version + return $p } function Run-MSBuild([string]$projectFilePath, [string]$buildArgs = "", [string]$logFileName = "", [switch]$parallel = $true, [switch]$summary = $true, [switch]$warnAsError = $true, [string]$configuration = $script:configuration) { - # Because we override the C#/VB toolset to build against our LKG package, it is important - # that we do not reuse MSBuild nodes from other jobs/builds on the machine. Otherwise, - # we'll run into issues such as https://github.com/dotnet/roslyn/issues/6211. - # MSBuildAdditionalCommandLineArgs= - $args = "/p:TreatWarningsAsErrors=true /nologo /nodeReuse:false /p:Configuration=$configuration "; - - if ($warnAsError) { - $args += " /warnaserror" - } + # Because we override the C#/VB toolset to build against our LKG package, it is important + # that we do not reuse MSBuild nodes from other jobs/builds on the machine. Otherwise, + # we'll run into issues such as https://github.com/dotnet/roslyn/issues/6211. + # MSBuildAdditionalCommandLineArgs= + $args = "/p:TreatWarningsAsErrors=true /nologo /nodeReuse:false /p:Configuration=$configuration "; + + if ($warnAsError) { + $args += " /warnaserror" + } - if ($summary) { - $args += " /consoleloggerparameters:Verbosity=minimal;summary" - } else { - $args += " /consoleloggerparameters:Verbosity=minimal" - } + if ($summary) { + $args += " /consoleloggerparameters:Verbosity=minimal;summary" + } else { + $args += " /consoleloggerparameters:Verbosity=minimal" + } - if ($parallel) { - $args += " /m" - } + if ($parallel) { + $args += " /m" + } - if ($skipAnalyzers) { - $args += " /p:UseRoslynAnalyzers=false" - } + if ($skipAnalyzers) { + $args += " /p:UseRoslynAnalyzers=false" + } - if ($binaryLog) { - if ($logFileName -eq "") { - $logFileName = [IO.Path]::GetFileNameWithoutExtension($projectFilePath) - } - $logFileName = [IO.Path]::ChangeExtension($logFileName, ".binlog") - $logFilePath = Join-Path $LogDir $logFileName - $args += " /bl:$logFilePath" + if ($binaryLog) { + if ($logFileName -eq "") { + $logFileName = [IO.Path]::GetFileNameWithoutExtension($projectFilePath) } + $logFileName = [IO.Path]::ChangeExtension($logFileName, ".binlog") + $logFilePath = Join-Path $LogDir $logFileName + $args += " /bl:$logFilePath" + } - if ($officialBuildId) { - $args += " /p:OfficialBuildId=" + $officialBuildId - } + if ($officialBuildId) { + $args += " /p:OfficialBuildId=" + $officialBuildId + } - if ($ci) { - $args += " /p:ContinuousIntegrationBuild=true" - } + if ($ci) { + $args += " /p:ContinuousIntegrationBuild=true" + } - if ($bootstrapDir -ne "") { - $args += " /p:BootstrapBuildPath=$bootstrapDir" - } + if ($bootstrapDir -ne "") { + $args += " /p:BootstrapBuildPath=$bootstrapDir" + } - $args += " $buildArgs" - $args += " $projectFilePath" - $args += " $properties" + $args += " $buildArgs" + $args += " $projectFilePath" + $args += " $properties" - $buildTool = InitializeBuildTool - Exec-Console $buildTool.Path "$($buildTool.Command) $args" + $buildTool = InitializeBuildTool + Exec-Console $buildTool.Path "$($buildTool.Command) $args" } # Create a bootstrap build of the compiler. Returns the directory where the bootstrap build @@ -269,48 +269,48 @@ function Run-MSBuild([string]$projectFilePath, [string]$buildArgs = "", [string] # Important to not set $script:bootstrapDir here yet as we're actually in the process of # building the bootstrap. function Make-BootstrapBuild() { - Write-Host "Building bootstrap compiler" + Write-Host "Building bootstrap compiler" - $dir = Join-Path $ArtifactsDir "Bootstrap" - Remove-Item -re $dir -ErrorAction SilentlyContinue - Create-Directory $dir + $dir = Join-Path $ArtifactsDir "Bootstrap" + Remove-Item -re $dir -ErrorAction SilentlyContinue + Create-Directory $dir - $packageName = "Microsoft.Net.Compilers.Toolset" - $projectPath = "src\NuGet\$packageName\$packageName.Package.csproj" + $packageName = "Microsoft.Net.Compilers.Toolset" + $projectPath = "src\NuGet\$packageName\$packageName.Package.csproj" - Run-MSBuild $projectPath "/restore /t:Pack /p:DotNetUseShippingVersions=true /p:InitialDefineConstants=BOOTSTRAP /p:PackageOutputPath=`"$dir`" /p:EnableNgenOptimization=false" -logFileName "Bootstrap" -configuration $bootstrapConfiguration - $packageFile = Get-ChildItem -Path $dir -Filter "$packageName.*.nupkg" - Unzip "$dir\$packageFile" $dir + Run-MSBuild $projectPath "/restore /t:Pack /p:DotNetUseShippingVersions=true /p:InitialDefineConstants=BOOTSTRAP /p:PackageOutputPath=`"$dir`" /p:EnableNgenOptimization=false" -logFileName "Bootstrap" -configuration $bootstrapConfiguration + $packageFile = Get-ChildItem -Path $dir -Filter "$packageName.*.nupkg" + Unzip "$dir\$packageFile" $dir - Write-Host "Cleaning Bootstrap compiler artifacts" - Run-MSBuild $projectPath "/t:Clean" -logFileName "BootstrapClean" + Write-Host "Cleaning Bootstrap compiler artifacts" + Run-MSBuild $projectPath "/t:Clean" -logFileName "BootstrapClean" - return $dir + return $dir } Add-Type -AssemblyName 'System.Drawing' Add-Type -AssemblyName 'System.Windows.Forms' function Capture-Screenshot($path) { - $width = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds.Width - $height = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds.Height + $width = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds.Width + $height = [System.Windows.Forms.Screen]::PrimaryScreen.Bounds.Height - $bitmap = New-Object System.Drawing.Bitmap $width, $height + $bitmap = New-Object System.Drawing.Bitmap $width, $height + try { + $graphics = [System.Drawing.Graphics]::FromImage($bitmap) try { - $graphics = [System.Drawing.Graphics]::FromImage($bitmap) - try { - $graphics.CopyFromScreen( ` - [System.Windows.Forms.Screen]::PrimaryScreen.Bounds.X, ` - [System.Windows.Forms.Screen]::PrimaryScreen.Bounds.Y, ` - 0, ` - 0, ` - $bitmap.Size, ` - [System.Drawing.CopyPixelOperation]::SourceCopy) - } finally { - $graphics.Dispose() - } - - $bitmap.Save($path, [System.Drawing.Imaging.ImageFormat]::Png) + $graphics.CopyFromScreen( ` + [System.Windows.Forms.Screen]::PrimaryScreen.Bounds.X, ` + [System.Windows.Forms.Screen]::PrimaryScreen.Bounds.Y, ` + 0, ` + 0, ` + $bitmap.Size, ` + [System.Drawing.CopyPixelOperation]::SourceCopy) } finally { - $bitmap.Dispose() + $graphics.Dispose() } + + $bitmap.Save($path, [System.Drawing.Imaging.ImageFormat]::Png) + } finally { + $bitmap.Dispose() + } } diff --git a/eng/build.ps1 b/eng/build.ps1 index df154e741c970..fd3ab1cad6d31 100644 --- a/eng/build.ps1 +++ b/eng/build.ps1 @@ -16,104 +16,104 @@ [CmdletBinding(PositionalBinding=$false)] param ( - [string][Alias('c')]$configuration = "Debug", - [string][Alias('v')]$verbosity = "m", - [string]$msbuildEngine = "vs", - - # Actions - [switch][Alias('r')]$restore, - [switch][Alias('b')]$build, - [switch]$rebuild, - [switch]$sign, - [switch]$pack, - [switch]$publish, - [switch]$launch, - [switch]$help, - - # Options - [switch]$bootstrap, - [string]$bootstrapConfiguration = "Release", - [switch][Alias('bl')]$binaryLog, - [switch]$buildServerLog, - [switch]$ci, - [switch]$procdump, - [switch]$skipAnalyzers, - [switch][Alias('d')]$deployExtensions, - [switch]$prepareMachine, - [switch]$useGlobalNuGetCache = $true, - [switch]$warnAsError = $false, - - # official build settings - [string]$officialBuildId = "", - [string]$officialSkipApplyOptimizationData = "", - [string]$officialSkipTests = "", - [string]$officialSourceBranchName = "", - [string]$officialIbcSourceBranchName = "", - [string]$officialIbcDropId = "", - - # Test actions - [switch]$test32, - [switch]$test64, - [switch]$testVsi, - [switch][Alias('test')]$testDesktop, - [switch]$testCoreClr, - [switch]$testIOperation, - [switch]$testLegacyCompletion, - - [parameter(ValueFromRemainingArguments=$true)][string[]]$properties) + [string][Alias('c')]$configuration = "Debug", + [string][Alias('v')]$verbosity = "m", + [string]$msbuildEngine = "vs", + + # Actions + [switch][Alias('r')]$restore, + [switch][Alias('b')]$build, + [switch]$rebuild, + [switch]$sign, + [switch]$pack, + [switch]$publish, + [switch]$launch, + [switch]$help, + + # Options + [switch]$bootstrap, + [string]$bootstrapConfiguration = "Release", + [switch][Alias('bl')]$binaryLog, + [switch]$buildServerLog, + [switch]$ci, + [switch]$procdump, + [switch]$skipAnalyzers, + [switch][Alias('d')]$deployExtensions, + [switch]$prepareMachine, + [switch]$useGlobalNuGetCache = $true, + [switch]$warnAsError = $false, + + # official build settings + [string]$officialBuildId = "", + [string]$officialSkipApplyOptimizationData = "", + [string]$officialSkipTests = "", + [string]$officialSourceBranchName = "", + [string]$officialIbcSourceBranchName = "", + [string]$officialIbcDropId = "", + + # Test actions + [switch]$test32, + [switch]$test64, + [switch]$testVsi, + [switch][Alias('test')]$testDesktop, + [switch]$testCoreClr, + [switch]$testIOperation, + [switch]$testLegacyCompletion, + + [parameter(ValueFromRemainingArguments=$true)][string[]]$properties) Set-StrictMode -version 2.0 $ErrorActionPreference = "Stop" function Print-Usage() { - Write-Host "Common settings:" - Write-Host " -configuration Build configuration: 'Debug' or 'Release' (short: -c)" - Write-Host " -verbosity Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic]" - Write-Host " -deployExtensions Deploy built vsixes (short: -d)" - Write-Host " -binaryLog Create MSBuild binary log (short: -bl)" - Write-Host " -buildServerLog Create Roslyn build server log" - Write-Host "" - Write-Host "Actions:" - Write-Host " -restore Restore packages (short: -r)" - Write-Host " -build Build main solution (short: -b)" - Write-Host " -rebuild Rebuild main solution" - Write-Host " -pack Build NuGet packages, VS insertion manifests and installer" - Write-Host " -sign Sign our binaries" - Write-Host " -publish Publish build artifacts (e.g. symbols)" - Write-Host " -launch Launch Visual Studio in developer hive" - Write-Host " -help Print help and exit" - Write-Host "" - Write-Host "Test actions" - Write-Host " -test32 Run unit tests in the 32-bit runner" - Write-Host " -test64 Run units tests in the 64-bit runner" - Write-Host " -testDesktop Run Desktop unit tests (short: -test)" - Write-Host " -testCoreClr Run CoreClr unit tests" - Write-Host " -testVsi Run all integration tests" - Write-Host " -testIOperation Run extra checks to validate IOperations" - Write-Host " -testLegacyCompletion Run integration tests with legacy completion" - Write-Host "" - Write-Host "Advanced settings:" - Write-Host " -ci Set when running on CI server" - Write-Host " -bootstrap Build using a bootstrap compilers" - Write-Host " -bootstrapConfiguration Build configuration for bootstrap compiler: 'Debug' or 'Release'" - Write-Host " -msbuildEngine Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)." - Write-Host " -procdump Monitor test runs with procdump" - Write-Host " -skipAnalyzers Do not run analyzers during build operations" - Write-Host " -prepareMachine Prepare machine for CI run, clean up processes after build" - Write-Host " -useGlobalNuGetCache Use global NuGet cache." - Write-Host " -warnAsError Treat all warnings as errors" - Write-Host "" - Write-Host "Official build settings:" - Write-Host " -officialBuildId An official build id, e.g. 20190102.3" - Write-Host " -officialSkipTests Pass 'true' to not run tests" - Write-Host " -officialSkipApplyOptimizationData Pass 'true' to not apply optimization data" - Write-Host " -officialSourceBranchName The source branch name" - Write-Host " -officialIbcDropId IBC data drop to use (e.g. '20190210.1/935479/1')." - Write-Host " 'default' for the most recent available for the branch." - Write-Host " -officialIbcSourceBranchName IBC source branch (e.g. 'master-vs-deps')" - Write-Host " 'default' to select branch based on eng/config/PublishData.json." - Write-Host "" - Write-Host "Command line arguments starting with '/p:' are passed through to MSBuild." + Write-Host "Common settings:" + Write-Host " -configuration Build configuration: 'Debug' or 'Release' (short: -c)" + Write-Host " -verbosity Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic]" + Write-Host " -deployExtensions Deploy built vsixes (short: -d)" + Write-Host " -binaryLog Create MSBuild binary log (short: -bl)" + Write-Host " -buildServerLog Create Roslyn build server log" + Write-Host "" + Write-Host "Actions:" + Write-Host " -restore Restore packages (short: -r)" + Write-Host " -build Build main solution (short: -b)" + Write-Host " -rebuild Rebuild main solution" + Write-Host " -pack Build NuGet packages, VS insertion manifests and installer" + Write-Host " -sign Sign our binaries" + Write-Host " -publish Publish build artifacts (e.g. symbols)" + Write-Host " -launch Launch Visual Studio in developer hive" + Write-Host " -help Print help and exit" + Write-Host "" + Write-Host "Test actions" + Write-Host " -test32 Run unit tests in the 32-bit runner" + Write-Host " -test64 Run units tests in the 64-bit runner" + Write-Host " -testDesktop Run Desktop unit tests (short: -test)" + Write-Host " -testCoreClr Run CoreClr unit tests" + Write-Host " -testVsi Run all integration tests" + Write-Host " -testIOperation Run extra checks to validate IOperations" + Write-Host " -testLegacyCompletion Run integration tests with legacy completion" + Write-Host "" + Write-Host "Advanced settings:" + Write-Host " -ci Set when running on CI server" + Write-Host " -bootstrap Build using a bootstrap compilers" + Write-Host " -bootstrapConfiguration Build configuration for bootstrap compiler: 'Debug' or 'Release'" + Write-Host " -msbuildEngine Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)." + Write-Host " -procdump Monitor test runs with procdump" + Write-Host " -skipAnalyzers Do not run analyzers during build operations" + Write-Host " -prepareMachine Prepare machine for CI run, clean up processes after build" + Write-Host " -useGlobalNuGetCache Use global NuGet cache." + Write-Host " -warnAsError Treat all warnings as errors" + Write-Host "" + Write-Host "Official build settings:" + Write-Host " -officialBuildId An official build id, e.g. 20190102.3" + Write-Host " -officialSkipTests Pass 'true' to not run tests" + Write-Host " -officialSkipApplyOptimizationData Pass 'true' to not apply optimization data" + Write-Host " -officialSourceBranchName The source branch name" + Write-Host " -officialIbcDropId IBC data drop to use (e.g. '20190210.1/935479/1')." + Write-Host " 'default' for the most recent available for the branch." + Write-Host " -officialIbcSourceBranchName IBC source branch (e.g. 'master-vs-deps')" + Write-Host " 'default' to select branch based on eng/config/PublishData.json." + Write-Host "" + Write-Host "Command line arguments starting with '/p:' are passed through to MSBuild." } # Process the command line arguments and establish defaults for the values which are not @@ -124,507 +124,507 @@ function Print-Usage() { # $build based on say $testDesktop. It's possible the developer wanted only for testing # to execute, not any build. function Process-Arguments() { - function OfficialBuildOnly([string]$argName) { - if ((Get-Variable $argName -Scope Script).Value) { - if (!$officialBuildId) { - Write-Host "$argName can only be specified for official builds" - exit 1 - } - } else { - if ($officialBuildId) { - Write-Host "$argName must be specified in official builds" - exit 1 - } - } + function OfficialBuildOnly([string]$argName) { + if ((Get-Variable $argName -Scope Script).Value) { + if (!$officialBuildId) { + Write-Host "$argName can only be specified for official builds" + exit 1 + } + } else { + if ($officialBuildId) { + Write-Host "$argName must be specified in official builds" + exit 1 + } } + } - if ($help -or (($properties -ne $null) -and ($properties.Contains("/help") -or $properties.Contains("/?")))) { + if ($help -or (($properties -ne $null) -and ($properties.Contains("/help") -or $properties.Contains("/?")))) { Print-Usage exit 0 + } + + OfficialBuildOnly "officialSkipTests" + OfficialBuildOnly "officialSkipApplyOptimizationData" + OfficialBuildOnly "officialSourceBranchName" + OfficialBuildOnly "officialIbcDropId" + OfficialBuildOnly "officialIbcSourceBranchName" + + if ($officialBuildId) { + $script:useGlobalNuGetCache = $false + $script:procdump = $true + $script:testDesktop = ![System.Boolean]::Parse($officialSkipTests) + $script:applyOptimizationData = ![System.Boolean]::Parse($officialSkipApplyOptimizationData) + } else { + $script:applyOptimizationData = $false + } + + if ($ci) { + $script:binaryLog = $true + if ($bootstrap) { + $script:buildServerLog = $true } + } - OfficialBuildOnly "officialSkipTests" - OfficialBuildOnly "officialSkipApplyOptimizationData" - OfficialBuildOnly "officialSourceBranchName" - OfficialBuildOnly "officialIbcDropId" - OfficialBuildOnly "officialIbcSourceBranchName" - - if ($officialBuildId) { - $script:useGlobalNuGetCache = $false - $script:procdump = $true - $script:testDesktop = ![System.Boolean]::Parse($officialSkipTests) - $script:applyOptimizationData = ![System.Boolean]::Parse($officialSkipApplyOptimizationData) - } else { - $script:applyOptimizationData = $false - } - - if ($ci) { - $script:binaryLog = $true - if ($bootstrap) { - $script:buildServerLog = $true - } - } - - if ($test32 -and $test64) { - Write-Host "Cannot combine -test32 and -test64" - exit 1 - } + if ($test32 -and $test64) { + Write-Host "Cannot combine -test32 and -test64" + exit 1 + } - $anyUnit = $testDesktop -or $testCoreClr - if ($anyUnit -and $testVsi) { - Write-Host "Cannot combine unit and VSI testing" - exit 1 - } + $anyUnit = $testDesktop -or $testCoreClr + if ($anyUnit -and $testVsi) { + Write-Host "Cannot combine unit and VSI testing" + exit 1 + } - if ($testVsi) { - # Avoid spending time in analyzers when requested, and also in the slowest integration test builds - $script:skipAnalyzers = $true - $script:bootstrap = $false - } + if ($testVsi) { + # Avoid spending time in analyzers when requested, and also in the slowest integration test builds + $script:skipAnalyzers = $true + $script:bootstrap = $false + } - if ($build -and $launch -and -not $deployExtensions) { - Write-Host -ForegroundColor Red "Cannot combine -build and -launch without -deployExtensions" - exit 1 - } + if ($build -and $launch -and -not $deployExtensions) { + Write-Host -ForegroundColor Red "Cannot combine -build and -launch without -deployExtensions" + exit 1 + } - $script:test32 = -not $test64 + $script:test32 = -not $test64 - foreach ($property in $properties) { - if (!$property.StartsWith("/p:", "InvariantCultureIgnoreCase")) { - Write-Host "Invalid argument: $property" - Print-Usage - exit 1 - } + foreach ($property in $properties) { + if (!$property.StartsWith("/p:", "InvariantCultureIgnoreCase")) { + Write-Host "Invalid argument: $property" + Print-Usage + exit 1 } + } } function BuildSolution() { - # Roslyn.sln can't be built with dotnet due to WPF and VSIX build task dependencies - $solution = if ($msbuildEngine -eq 'dotnet') { "Compilers.sln" } else { "Roslyn.sln" } - - Write-Host "$($solution):" - - $bl = if ($binaryLog) { "/bl:" + (Join-Path $LogDir "Build.binlog") } else { "" } - - if ($buildServerLog) { - ${env:ROSLYNCOMMANDLINELOGFILE} = Join-Path $LogDir "Build.Server.log" - } - - $projects = Join-Path $RepoRoot $solution - $enableAnalyzers = !$skipAnalyzers - $toolsetBuildProj = InitializeToolset - - # Have to disable quiet restore during bootstrap builds to work around - # an arcade bug - # https://github.com/dotnet/arcade/issues/2220 - $quietRestore = !($ci -or ($bootstrapDir -ne "")) - $testTargetFrameworks = if ($testCoreClr) { "netcoreapp3.0%3Bnetcoreapp2.1" } else { "" } - - $ibcSourceBranchName = GetIbcSourceBranchName - $ibcDropId = if ($officialIbcDropId -ne "default") { $officialIbcDropId } else { "" } - - # Do not set this property to true explicitly, since that would override values set in projects. - $suppressExtensionDeployment = if (!$deployExtensions) { "/p:DeployExtension=false" } else { "" } - - # Workaround for some machines in the AzDO pool not allowing long paths (%5c is msbuild escaped backslash) - $ibcDir = Join-Path $RepoRoot ".o%5c" - - try { - # Setting /p:TreatWarningsAsErrors=true is a workaround for https://github.com/Microsoft/msbuild/issues/3062. - # We don't pass /warnaserror to msbuild ($warnAsError is set to $false by default above), but set - # /p:TreatWarningsAsErrors=true so that compiler reported warnings, other than IDE0055 are treated as errors. - # Warnings reported from other msbuild tasks are not treated as errors for now. - MSBuild $toolsetBuildProj ` - $bl ` - /p:Configuration=$configuration ` - /p:Projects=$projects ` - /p:RepoRoot=$RepoRoot ` - /p:Restore=$restore ` - /p:Build=$build ` - /p:Test=$testCoreClr ` - /p:Rebuild=$rebuild ` - /p:Pack=$pack ` - /p:Sign=$sign ` - /p:Publish=$publish ` - /p:ContinuousIntegrationBuild=$ci ` - /p:OfficialBuildId=$officialBuildId ` - /p:UseRoslynAnalyzers=$enableAnalyzers ` - /p:BootstrapBuildPath=$bootstrapDir ` - /p:QuietRestore=$quietRestore ` - /p:QuietRestoreBinaryLog=$binaryLog ` - /p:TestTargetFrameworks=$testTargetFrameworks ` - /p:TreatWarningsAsErrors=true ` - /p:VisualStudioIbcSourceBranchName=$ibcSourceBranchName ` - /p:VisualStudioIbcDropId=$ibcDropId ` - /p:EnableNgenOptimization=$applyOptimizationData ` - /p:IbcOptimizationDataDir=$ibcDir ` - $suppressExtensionDeployment ` - @properties - } - finally { - ${env:ROSLYNCOMMANDLINELOGFILE} = $null - } + # Roslyn.sln can't be built with dotnet due to WPF and VSIX build task dependencies + $solution = if ($msbuildEngine -eq 'dotnet') { "Compilers.sln" } else { "Roslyn.sln" } + + Write-Host "$($solution):" + + $bl = if ($binaryLog) { "/bl:" + (Join-Path $LogDir "Build.binlog") } else { "" } + + if ($buildServerLog) { + ${env:ROSLYNCOMMANDLINELOGFILE} = Join-Path $LogDir "Build.Server.log" + } + + $projects = Join-Path $RepoRoot $solution + $enableAnalyzers = !$skipAnalyzers + $toolsetBuildProj = InitializeToolset + + # Have to disable quiet restore during bootstrap builds to work around + # an arcade bug + # https://github.com/dotnet/arcade/issues/2220 + $quietRestore = !($ci -or ($bootstrapDir -ne "")) + $testTargetFrameworks = if ($testCoreClr) { "netcoreapp3.0%3Bnetcoreapp2.1" } else { "" } + + $ibcSourceBranchName = GetIbcSourceBranchName + $ibcDropId = if ($officialIbcDropId -ne "default") { $officialIbcDropId } else { "" } + + # Do not set this property to true explicitly, since that would override values set in projects. + $suppressExtensionDeployment = if (!$deployExtensions) { "/p:DeployExtension=false" } else { "" } + + # Workaround for some machines in the AzDO pool not allowing long paths (%5c is msbuild escaped backslash) + $ibcDir = Join-Path $RepoRoot ".o%5c" + + try { + # Setting /p:TreatWarningsAsErrors=true is a workaround for https://github.com/Microsoft/msbuild/issues/3062. + # We don't pass /warnaserror to msbuild ($warnAsError is set to $false by default above), but set + # /p:TreatWarningsAsErrors=true so that compiler reported warnings, other than IDE0055 are treated as errors. + # Warnings reported from other msbuild tasks are not treated as errors for now. + MSBuild $toolsetBuildProj ` + $bl ` + /p:Configuration=$configuration ` + /p:Projects=$projects ` + /p:RepoRoot=$RepoRoot ` + /p:Restore=$restore ` + /p:Build=$build ` + /p:Test=$testCoreClr ` + /p:Rebuild=$rebuild ` + /p:Pack=$pack ` + /p:Sign=$sign ` + /p:Publish=$publish ` + /p:ContinuousIntegrationBuild=$ci ` + /p:OfficialBuildId=$officialBuildId ` + /p:UseRoslynAnalyzers=$enableAnalyzers ` + /p:BootstrapBuildPath=$bootstrapDir ` + /p:QuietRestore=$quietRestore ` + /p:QuietRestoreBinaryLog=$binaryLog ` + /p:TestTargetFrameworks=$testTargetFrameworks ` + /p:TreatWarningsAsErrors=true ` + /p:VisualStudioIbcSourceBranchName=$ibcSourceBranchName ` + /p:VisualStudioIbcDropId=$ibcDropId ` + /p:EnableNgenOptimization=$applyOptimizationData ` + /p:IbcOptimizationDataDir=$ibcDir ` + $suppressExtensionDeployment ` + @properties + } + finally { + ${env:ROSLYNCOMMANDLINELOGFILE} = $null + } } # Get the branch that produced the IBC data this build is going to consume. # IBC data are only merged in official built, but we want to test some of the logic in CI builds as well. function GetIbcSourceBranchName() { - if (Test-Path variable:global:_IbcSourceBranchName) { + if (Test-Path variable:global:_IbcSourceBranchName) { return $global:_IbcSourceBranchName - } + } - function calculate { - $fallback = "master-vs-deps" + function calculate { + $fallback = "master-vs-deps" - if (!$officialIbcSourceBranchName) { - return $fallback - } + if (!$officialIbcSourceBranchName) { + return $fallback + } - if ($officialIbcSourceBranchName -ne "default") { - return $officialIbcSourceBranchName - } - - $branchData = GetBranchPublishData $officialSourceBranchName - if ($branchData -eq $null) { - Write-Host "Warning: Branch $officialSourceBranchName is not listed in PublishData.json. Using IBC data from '$fallback'." -ForegroundColor Yellow - Write-Host "Override by setting IbcSourceBranchName build variable." -ForegroundColor Yellow - return $fallback - } + if ($officialIbcSourceBranchName -ne "default") { + return $officialIbcSourceBranchName + } - if (Get-Member -InputObject $branchData -Name "ibcSourceBranch") { - return $branchData.ibcSourceBranch - } + $branchData = GetBranchPublishData $officialSourceBranchName + if ($branchData -eq $null) { + Write-Host "Warning: Branch $officialSourceBranchName is not listed in PublishData.json. Using IBC data from '$fallback'." -ForegroundColor Yellow + Write-Host "Override by setting IbcSourceBranchName build variable." -ForegroundColor Yellow + return $fallback + } - return $officialSourceBranchName + if (Get-Member -InputObject $branchData -Name "ibcSourceBranch") { + return $branchData.ibcSourceBranch } - return $global:_IbcSourceBranchName = calculate + return $officialSourceBranchName + } + + return $global:_IbcSourceBranchName = calculate } # Set VSO variables used by MicroBuildBuildVSBootstrapper pipeline task function SetVisualStudioBootstrapperBuildArgs() { - $fallbackBranch = "master-vs-deps" + $fallbackBranch = "master-vs-deps" - $branchName = if ($officialSourceBranchName) { $officialSourceBranchName } else { $fallbackBranch } - $branchData = GetBranchPublishData $branchName + $branchName = if ($officialSourceBranchName) { $officialSourceBranchName } else { $fallbackBranch } + $branchData = GetBranchPublishData $branchName - if ($branchData -eq $null) { - Write-Host "Warning: Branch $officialSourceBranchName is not listed in PublishData.json. Using VS bootstrapper for branch '$fallbackBranch'. " -ForegroundColor Yellow - $branchData = GetBranchPublishData $fallbackBranch - } + if ($branchData -eq $null) { + Write-Host "Warning: Branch $officialSourceBranchName is not listed in PublishData.json. Using VS bootstrapper for branch '$fallbackBranch'. " -ForegroundColor Yellow + $branchData = GetBranchPublishData $fallbackBranch + } - # VS branch name is e.g. "lab/d16.0stg", "rel/d15.9", "lab/ml", etc. - $vsBranchSimpleName = $branchData.vsBranch.Split('/')[-1] - $vsMajorVersion = $branchData.vsMajorVersion - $vsChannel = "int.$vsBranchSimpleName" + # VS branch name is e.g. "lab/d16.0stg", "rel/d15.9", "lab/ml", etc. + $vsBranchSimpleName = $branchData.vsBranch.Split('/')[-1] + $vsMajorVersion = $branchData.vsMajorVersion + $vsChannel = "int.$vsBranchSimpleName" - Write-Host "##vso[task.setvariable variable=VisualStudio.MajorVersion;]$vsMajorVersion" - Write-Host "##vso[task.setvariable variable=VisualStudio.ChannelName;]$vsChannel" + Write-Host "##vso[task.setvariable variable=VisualStudio.MajorVersion;]$vsMajorVersion" + Write-Host "##vso[task.setvariable variable=VisualStudio.ChannelName;]$vsChannel" - $insertionDir = Join-Path $VSSetupDir "Insertion" - $manifestList = [string]::Join(',', (Get-ChildItem "$insertionDir\*.vsman")) - Write-Host "##vso[task.setvariable variable=VisualStudio.SetupManifestList;]$manifestList" + $insertionDir = Join-Path $VSSetupDir "Insertion" + $manifestList = [string]::Join(',', (Get-ChildItem "$insertionDir\*.vsman")) + Write-Host "##vso[task.setvariable variable=VisualStudio.SetupManifestList;]$manifestList" } # Core function for running our unit / integration tests tests function TestUsingOptimizedRunner() { - # Tests need to locate .NET Core SDK - $dotnet = InitializeDotNetCli + # Tests need to locate .NET Core SDK + $dotnet = InitializeDotNetCli - if ($testVsi) { - Deploy-VsixViaTool - - if ($ci) { - # Minimize all windows to avoid interference during integration test runs - $shell = New-Object -ComObject "Shell.Application" - $shell.MinimizeAll() - } - } - - if ($testIOperation) { - $env:ROSLYN_TEST_IOPERATION = "true" - } + if ($testVsi) { + Deploy-VsixViaTool - if ($testLegacyCompletion) { - $env:ROSLYN_TEST_LEGACY_COMPLETION = "true" + if ($ci) { + # Minimize all windows to avoid interference during integration test runs + $shell = New-Object -ComObject "Shell.Application" + $shell.MinimizeAll() } + } - $testResultsDir = Join-Path $ArtifactsDir "TestResults\$configuration" - $binDir = Join-Path $ArtifactsDir "bin" - $runTests = GetProjectOutputBinary "RunTests.exe" + if ($testIOperation) { + $env:ROSLYN_TEST_IOPERATION = "true" + } - if (!(Test-Path $runTests)) { - Write-Host "Test runner not found: '$runTests'. Run Build.cmd first." -ForegroundColor Red - ExitWithExitCode 1 - } + if ($testLegacyCompletion) { + $env:ROSLYN_TEST_LEGACY_COMPLETION = "true" + } - $xunitDir = Join-Path (Get-PackageDir "xunit.runner.console") "tools\net472" - $args = "`"$xunitDir`"" - $args += " `"-out:$testResultsDir`"" - $args += " `"-logs:$LogDir`"" - $args += " -nocache" - $args += " -tfm:net472" - - if ($testDesktop -or $testIOperation) { - if ($test32) { - $dlls = Get-ChildItem -Recurse -Include "*.UnitTests.dll" $binDir - } else { - $dlls = Get-ChildItem -Recurse -Include "*.UnitTests.dll" -Exclude "*InteractiveHost*" $binDir - } - } elseif ($testVsi) { - # Since they require Visual Studio to be installed, ensure that the MSBuildWorkspace tests run along with our VS - # integration tests in CI. - if ($ci) { - $dlls += @(Get-Item (GetProjectOutputBinary "Microsoft.CodeAnalysis.Workspaces.MSBuild.UnitTests.dll")) - } + $testResultsDir = Join-Path $ArtifactsDir "TestResults\$configuration" + $binDir = Join-Path $ArtifactsDir "bin" + $runTests = GetProjectOutputBinary "RunTests.exe" - $dlls += @(Get-ChildItem -Recurse -Include "*.IntegrationTests.dll" $binDir) + if (!(Test-Path $runTests)) { + Write-Host "Test runner not found: '$runTests'. Run Build.cmd first." -ForegroundColor Red + ExitWithExitCode 1 + } + + $xunitDir = Join-Path (Get-PackageDir "xunit.runner.console") "tools\net472" + $args = "`"$xunitDir`"" + $args += " `"-out:$testResultsDir`"" + $args += " `"-logs:$LogDir`"" + $args += " -nocache" + $args += " -tfm:net472" + + if ($testDesktop -or $testIOperation) { + if ($test32) { + $dlls = Get-ChildItem -Recurse -Include "*.UnitTests.dll" $binDir } else { - $dlls = Get-ChildItem -Recurse -Include "*.IntegrationTests.dll" $binDir - $args += " -trait:Feature=NetCore" + $dlls = Get-ChildItem -Recurse -Include "*.UnitTests.dll" -Exclude "*InteractiveHost*" $binDir + } + } elseif ($testVsi) { + # Since they require Visual Studio to be installed, ensure that the MSBuildWorkspace tests run along with our VS + # integration tests in CI. + if ($ci) { + $dlls += @(Get-Item (GetProjectOutputBinary "Microsoft.CodeAnalysis.Workspaces.MSBuild.UnitTests.dll")) } - # Exclude out the multi-targetted netcore app projects - $dlls = $dlls | ?{ -not ($_.FullName -match ".*netcoreapp.*") } + $dlls += @(Get-ChildItem -Recurse -Include "*.IntegrationTests.dll" $binDir) + } else { + $dlls = Get-ChildItem -Recurse -Include "*.IntegrationTests.dll" $binDir + $args += " -trait:Feature=NetCore" + } - # Exclude out the ref assemblies - $dlls = $dlls | ?{ -not ($_.FullName -match ".*\\ref\\.*") } - $dlls = $dlls | ?{ -not ($_.FullName -match ".*/ref/.*") } + # Exclude out the multi-targetted netcore app projects + $dlls = $dlls | ?{ -not ($_.FullName -match ".*netcoreapp.*") } - if ($ci) { - $args += " -xml" - if ($testVsi) { - $args += " -timeout:110" - } else { - $args += " -timeout:65" - } - } + # Exclude out the ref assemblies + $dlls = $dlls | ?{ -not ($_.FullName -match ".*\\ref\\.*") } + $dlls = $dlls | ?{ -not ($_.FullName -match ".*/ref/.*") } - $procdumpPath = Ensure-ProcDump - $args += " -procdumppath:$procDumpPath" - if ($procdump) { - $args += " -useprocdump"; + if ($ci) { + $args += " -xml" + if ($testVsi) { + $args += " -timeout:110" + } else { + $args += " -timeout:65" } + } - if ($test64) { - $args += " -test64" - } + $procdumpPath = Ensure-ProcDump + $args += " -procdumppath:$procDumpPath" + if ($procdump) { + $args += " -useprocdump"; + } - foreach ($dll in $dlls) { - $args += " $dll" - } + if ($test64) { + $args += " -test64" + } - try { - Exec-Console $runTests $args - } finally { - Get-Process "xunit*" -ErrorAction SilentlyContinue | Stop-Process - if ($testIOperation) { - Remove-Item env:\ROSLYN_TEST_IOPERATION - } - if ($testLegacyCompletion) { - Remove-Item env:\ROSLYN_TEST_LEGACY_COMPLETION - } + foreach ($dll in $dlls) { + $args += " $dll" + } + + try { + Exec-Console $runTests $args + } finally { + Get-Process "xunit*" -ErrorAction SilentlyContinue | Stop-Process + if ($testIOperation) { + Remove-Item env:\ROSLYN_TEST_IOPERATION + } + if ($testLegacyCompletion) { + Remove-Item env:\ROSLYN_TEST_LEGACY_COMPLETION } + } } # Deploy our core VSIX libraries to Visual Studio via the Roslyn VSIX tool. This is an alternative to # deploying at build time. function Deploy-VsixViaTool() { - $vsixDir = Get-PackageDir "RoslynTools.VSIXExpInstaller" - $vsixExe = Join-Path $vsixDir "tools\VsixExpInstaller.exe" - - $vsInfo = LocateVisualStudio - if ($vsInfo -eq $null) { - throw "Unable to locate required Visual Studio installation" - } - - $vsDir = $vsInfo.installationPath.TrimEnd("\") - $vsId = $vsInfo.instanceId - $vsMajorVersion = $vsInfo.installationVersion.Split('.')[0] - - $hive = "RoslynDev" - Write-Host "Using VS Instance $vsId at `"$vsDir`"" - $baseArgs = "/rootSuffix:$hive /vsInstallDir:`"$vsDir`"" - - Write-Host "Uninstalling old Roslyn VSIX" - - # Actual uninstall is failing at the moment using the uninstall options. Temporarily using - # wildfire to uninstall our VSIX extensions - $extDir = Join-Path ${env:USERPROFILE} "AppData\Local\Microsoft\VisualStudio\$vsMajorVersion.0_$vsid$hive" - if (Test-Path $extDir) { - foreach ($dir in Get-ChildItem -Directory $extDir) { - $name = Split-Path -leaf $dir - Write-Host "`tUninstalling $name" - } - Remove-Item -re -fo $extDir - } - - Write-Host "Installing all Roslyn VSIX" - - # VSIX files need to be installed in this specific order: - $orderedVsixFileNames = @( - "Roslyn.Compilers.Extension.vsix", - "Roslyn.VisualStudio.Setup.vsix", - "Roslyn.VisualStudio.Setup.Dependencies.vsix", - "Roslyn.VisualStudio.InteractiveComponents.vsix", - "ExpressionEvaluatorPackage.vsix", - "Roslyn.VisualStudio.DiagnosticsWindow.vsix", - "Microsoft.VisualStudio.IntegrationTest.Setup.vsix") - - foreach ($vsixFileName in $orderedVsixFileNames) { - $vsixFile = Join-Path $VSSetupDir $vsixFileName - $fullArg = "$baseArgs $vsixFile" - Write-Host "`tInstalling $vsixFileName" - Exec-Console $vsixExe $fullArg - } + $vsixDir = Get-PackageDir "RoslynTools.VSIXExpInstaller" + $vsixExe = Join-Path $vsixDir "tools\VsixExpInstaller.exe" + + $vsInfo = LocateVisualStudio + if ($vsInfo -eq $null) { + throw "Unable to locate required Visual Studio installation" + } + + $vsDir = $vsInfo.installationPath.TrimEnd("\") + $vsId = $vsInfo.instanceId + $vsMajorVersion = $vsInfo.installationVersion.Split('.')[0] + + $hive = "RoslynDev" + Write-Host "Using VS Instance $vsId at `"$vsDir`"" + $baseArgs = "/rootSuffix:$hive /vsInstallDir:`"$vsDir`"" + + Write-Host "Uninstalling old Roslyn VSIX" + + # Actual uninstall is failing at the moment using the uninstall options. Temporarily using + # wildfire to uninstall our VSIX extensions + $extDir = Join-Path ${env:USERPROFILE} "AppData\Local\Microsoft\VisualStudio\$vsMajorVersion.0_$vsid$hive" + if (Test-Path $extDir) { + foreach ($dir in Get-ChildItem -Directory $extDir) { + $name = Split-Path -leaf $dir + Write-Host "`tUninstalling $name" + } + Remove-Item -re -fo $extDir + } + + Write-Host "Installing all Roslyn VSIX" + + # VSIX files need to be installed in this specific order: + $orderedVsixFileNames = @( + "Roslyn.Compilers.Extension.vsix", + "Roslyn.VisualStudio.Setup.vsix", + "Roslyn.VisualStudio.Setup.Dependencies.vsix", + "Roslyn.VisualStudio.InteractiveComponents.vsix", + "ExpressionEvaluatorPackage.vsix", + "Roslyn.VisualStudio.DiagnosticsWindow.vsix", + "Microsoft.VisualStudio.IntegrationTest.Setup.vsix") + + foreach ($vsixFileName in $orderedVsixFileNames) { + $vsixFile = Join-Path $VSSetupDir $vsixFileName + $fullArg = "$baseArgs $vsixFile" + Write-Host "`tInstalling $vsixFileName" + Exec-Console $vsixExe $fullArg + } } # Ensure that procdump is available on the machine. Returns the path to the directory that contains # the procdump binaries (both 32 and 64 bit) function Ensure-ProcDump() { - # Jenkins images default to having procdump installed in the root. Use that if available to avoid - # an unnecessary download. - if (Test-Path "C:\SysInternals\procdump.exe") { - return "C:\SysInternals" - } - - $outDir = Join-Path $ToolsDir "ProcDump" - $filePath = Join-Path $outDir "procdump.exe" - if (-not (Test-Path $filePath)) { - Remove-Item -Re $filePath -ErrorAction SilentlyContinue - Create-Directory $outDir - $zipFilePath = Join-Path $toolsDir "procdump.zip" - Invoke-WebRequest "https://download.sysinternals.com/files/Procdump.zip" -UseBasicParsing -outfile $zipFilePath | Out-Null - Unzip $zipFilePath $outDir - } - - return $outDir + # Jenkins images default to having procdump installed in the root. Use that if available to avoid + # an unnecessary download. + if (Test-Path "C:\SysInternals\procdump.exe") { + return "C:\SysInternals" + } + + $outDir = Join-Path $ToolsDir "ProcDump" + $filePath = Join-Path $outDir "procdump.exe" + if (-not (Test-Path $filePath)) { + Remove-Item -Re $filePath -ErrorAction SilentlyContinue + Create-Directory $outDir + $zipFilePath = Join-Path $toolsDir "procdump.zip" + Invoke-WebRequest "https://download.sysinternals.com/files/Procdump.zip" -UseBasicParsing -outfile $zipFilePath | Out-Null + Unzip $zipFilePath $outDir + } + + return $outDir } function Prepare-TempDir() { - Copy-Item (Join-Path $RepoRoot "src\Workspaces\MSBuildTest\Resources\.editorconfig") $TempDir - Copy-Item (Join-Path $RepoRoot "src\Workspaces\MSBuildTest\Resources\Directory.Build.props") $TempDir - Copy-Item (Join-Path $RepoRoot "src\Workspaces\MSBuildTest\Resources\Directory.Build.targets") $TempDir - Copy-Item (Join-Path $RepoRoot "src\Workspaces\MSBuildTest\Resources\Directory.Build.rsp") $TempDir - Copy-Item (Join-Path $RepoRoot "src\Workspaces\MSBuildTest\Resources\NuGet.Config") $TempDir + Copy-Item (Join-Path $RepoRoot "src\Workspaces\MSBuildTest\Resources\.editorconfig") $TempDir + Copy-Item (Join-Path $RepoRoot "src\Workspaces\MSBuildTest\Resources\Directory.Build.props") $TempDir + Copy-Item (Join-Path $RepoRoot "src\Workspaces\MSBuildTest\Resources\Directory.Build.targets") $TempDir + Copy-Item (Join-Path $RepoRoot "src\Workspaces\MSBuildTest\Resources\Directory.Build.rsp") $TempDir + Copy-Item (Join-Path $RepoRoot "src\Workspaces\MSBuildTest\Resources\NuGet.Config") $TempDir } function List-Processes() { - Write-Host "Listing running build processes..." - Get-Process -Name "msbuild" -ErrorAction SilentlyContinue | Out-Host - Get-Process -Name "vbcscompiler" -ErrorAction SilentlyContinue | Out-Host - Get-Process -Name "dotnet" -ErrorAction SilentlyContinue | where { $_.Modules | select { $_.ModuleName -eq "VBCSCompiler.dll" } } | Out-Host - Get-Process -Name "devenv" -ErrorAction SilentlyContinue | Out-Host + Write-Host "Listing running build processes..." + Get-Process -Name "msbuild" -ErrorAction SilentlyContinue | Out-Host + Get-Process -Name "vbcscompiler" -ErrorAction SilentlyContinue | Out-Host + Get-Process -Name "dotnet" -ErrorAction SilentlyContinue | where { $_.Modules | select { $_.ModuleName -eq "VBCSCompiler.dll" } } | Out-Host + Get-Process -Name "devenv" -ErrorAction SilentlyContinue | Out-Host } try { - if ($PSVersionTable.PSVersion.Major -lt "5") { - Write-Host "PowerShell version must be 5 or greater (version $($PSVersionTable.PSVersion) detected)" - exit 1 - } + if ($PSVersionTable.PSVersion.Major -lt "5") { + Write-Host "PowerShell version must be 5 or greater (version $($PSVersionTable.PSVersion) detected)" + exit 1 + } - $regKeyProperty = Get-ItemProperty -Path HKLM:\SYSTEM\CurrentControlSet\Control\FileSystem -Name "LongPathsEnabled" -ErrorAction Ignore - if (($null -eq $regKeyProperty) -or ($regKeyProperty.LongPathsEnabled -ne 1)) { - Write-Host "LongPath is not enabled, you may experience build errors. You can avoid these by enabling LongPath with ``reg ADD HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\FileSystem /v LongPathsEnabled /t REG_DWORD /d 1``" - } + $regKeyProperty = Get-ItemProperty -Path HKLM:\SYSTEM\CurrentControlSet\Control\FileSystem -Name "LongPathsEnabled" -ErrorAction Ignore + if (($null -eq $regKeyProperty) -or ($regKeyProperty.LongPathsEnabled -ne 1)) { + Write-Host "LongPath is not enabled, you may experience build errors. You can avoid these by enabling LongPath with ``reg ADD HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\FileSystem /v LongPathsEnabled /t REG_DWORD /d 1``" + } - Process-Arguments + Process-Arguments - . (Join-Path $PSScriptRoot "build-utils.ps1") + . (Join-Path $PSScriptRoot "build-utils.ps1") - if ($testVsi) { - $processesToStopOnExit += "devenv" - } + if ($testVsi) { + $processesToStopOnExit += "devenv" + } - Push-Location $RepoRoot + Push-Location $RepoRoot - if ($ci) { - List-Processes - Prepare-TempDir - - if ($testVsi) { - $screenshotPath = (Join-Path $LogDir "StartingBuild.png") - try { - Capture-Screenshot $screenshotPath - } - catch { - Write-Host "Screenshot failed; attempting to connect to the console" - - # Keep the session open so we have a UI to interact with - $quserItems = ((quser $env:USERNAME | select -Skip 1) -split '\s+') - $sessionid = $quserItems[2] - if ($sessionid -eq 'Disc') { - # When the session isn't connected, the third value is 'Disc' instead of the ID - $sessionid = $quserItems[1] - } - - if ($quserItems[1] -eq 'console') { - Write-Host "Disconnecting from console before attempting reconnection" - try { - tsdiscon - } catch { - # ignore - } - - # Disconnection is asynchronous, so wait a few seconds for it to complete - Start-Sleep -Seconds 3 - query user - } - - Write-Host "tscon $sessionid /dest:console" - tscon $sessionid /dest:console - - # Connection is asynchronous, so wait a few seconds for it to complete - Start-Sleep 3 - query user - - # Make sure we can capture a screenshot. An exception at this point will fail-fast the build. - Capture-Screenshot $screenshotPath - } + if ($ci) { + List-Processes + Prepare-TempDir + + if ($testVsi) { + $screenshotPath = (Join-Path $LogDir "StartingBuild.png") + try { + Capture-Screenshot $screenshotPath + } + catch { + Write-Host "Screenshot failed; attempting to connect to the console" + + # Keep the session open so we have a UI to interact with + $quserItems = ((quser $env:USERNAME | select -Skip 1) -split '\s+') + $sessionid = $quserItems[2] + if ($sessionid -eq 'Disc') { + # When the session isn't connected, the third value is 'Disc' instead of the ID + $sessionid = $quserItems[1] } - } - if ($ci) { - $global:_DotNetInstallDir = Join-Path $RepoRoot ".dotnet" - InstallDotNetSdk $global:_DotNetInstallDir $GlobalJson.tools.dotnet + if ($quserItems[1] -eq 'console') { + Write-Host "Disconnecting from console before attempting reconnection" + try { + tsdiscon + } catch { + # ignore + } + + # Disconnection is asynchronous, so wait a few seconds for it to complete + Start-Sleep -Seconds 3 + query user + } - # Make sure a 2.1 runtime is installed so we can run our tests. Most of them still - # target netcoreapp2.1. - InstallDotNetSdk $global:_DotNetInstallDir "2.1.503" - } + Write-Host "tscon $sessionid /dest:console" + tscon $sessionid /dest:console - if ($bootstrap) { - $bootstrapDir = Make-BootstrapBuild - } + # Connection is asynchronous, so wait a few seconds for it to complete + Start-Sleep 3 + query user - if ($restore -or $build -or $rebuild -or $pack -or $sign -or $publish -or $testCoreClr) { - BuildSolution + # Make sure we can capture a screenshot. An exception at this point will fail-fast the build. + Capture-Screenshot $screenshotPath + } } + } - if ($ci -and $build -and $msbuildEngine -eq "vs") { - SetVisualStudioBootstrapperBuildArgs - } + if ($ci) { + $global:_DotNetInstallDir = Join-Path $RepoRoot ".dotnet" + InstallDotNetSdk $global:_DotNetInstallDir $GlobalJson.tools.dotnet - if ($testDesktop -or $testVsi -or $testIOperation) { - TestUsingOptimizedRunner - } + # Make sure a 2.1 runtime is installed so we can run our tests. Most of them still + # target netcoreapp2.1. + InstallDotNetSdk $global:_DotNetInstallDir "2.1.503" + } - if ($launch) { - $devenvExe = Join-Path $env:VSINSTALLDIR 'Common7\IDE\devenv.exe' - &$devenvExe /rootSuffix RoslynDev - } + if ($bootstrap) { + $bootstrapDir = Make-BootstrapBuild + } + + if ($restore -or $build -or $rebuild -or $pack -or $sign -or $publish -or $testCoreClr) { + BuildSolution + } + + if ($ci -and $build -and $msbuildEngine -eq "vs") { + SetVisualStudioBootstrapperBuildArgs + } - ExitWithExitCode 0 + if ($testDesktop -or $testVsi -or $testIOperation) { + TestUsingOptimizedRunner + } + + if ($launch) { + $devenvExe = Join-Path $env:VSINSTALLDIR 'Common7\IDE\devenv.exe' + &$devenvExe /rootSuffix RoslynDev + } + + ExitWithExitCode 0 } catch { - Write-Host $_ - Write-Host $_.Exception - Write-Host $_.ScriptStackTrace - ExitWithExitCode 1 + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + ExitWithExitCode 1 } finally { - Pop-Location + Pop-Location } diff --git a/eng/findibc.ps1 b/eng/findibc.ps1 index f962065278636..0d84c23f5071c 100644 --- a/eng/findibc.ps1 +++ b/eng/findibc.ps1 @@ -1,39 +1,39 @@ -[CmdletBinding(PositionalBinding=$false)] +[CmdletBinding(PositionalBinding=$false)] param ( - [string]$intermediateAssembly, - [string]$ibcRootFolder) + [string]$intermediateAssembly, + [string]$ibcRootFolder) try { - $assemblyName = [System.IO.Path]::GetFileName($intermediateAssembly) - $fullPath = [System.IO.Path]::GetFullPath($ibcRootFolder) - if(![System.IO.Directory]::Exists($fullPath)){ - # There is no product data directory return - return "" - } - $root = (New-Object -TypeName System.IO.DirectoryInfo -ArgumentList $fullPath) - $dllEntry = [System.Linq.Enumerable]::SingleOrDefault($root.EnumerateFiles($assemblyName,[System.IO.SearchOption]::AllDirectories)) - if (!$dllEntry) - { - return ""; - } + $assemblyName = [System.IO.Path]::GetFileName($intermediateAssembly) + $fullPath = [System.IO.Path]::GetFullPath($ibcRootFolder) + if(![System.IO.Directory]::Exists($fullPath)){ + # There is no product data directory return + return "" + } + $root = (New-Object -TypeName System.IO.DirectoryInfo -ArgumentList $fullPath) + $dllEntry = [System.Linq.Enumerable]::SingleOrDefault($root.EnumerateFiles($assemblyName,[System.IO.SearchOption]::AllDirectories)) + if (!$dllEntry) + { + return ""; + } - $ibcFileInfos = $dllEntry.Directory.EnumerateFiles("*.ibc") - $strings = (New-Object "System.Collections.Generic.List[System.String]") - foreach ($ibcFileInfo in $ibcFileInfos) - { - $name = $ibcFileInfo.FullName - $strings.Add($name) - } - $ibcFiles = $strings -join ' ' + $ibcFileInfos = $dllEntry.Directory.EnumerateFiles("*.ibc") + $strings = (New-Object "System.Collections.Generic.List[System.String]") + foreach ($ibcFileInfo in $ibcFileInfos) + { + $name = $ibcFileInfo.FullName + $strings.Add($name) + } + $ibcFiles = $strings -join ' ' - return $ibcFiles + return $ibcFiles } catch { - Write-Host $_ - Write-Host $_.Exception - Write-Host $_.ScriptStackTrace - exit 1 + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + exit 1 } finally { - Pop-Location -} \ No newline at end of file + Pop-Location +} diff --git a/eng/generate-compiler-code.ps1 b/eng/generate-compiler-code.ps1 index 39fc891612412..bc3a919a14924 100644 --- a/eng/generate-compiler-code.ps1 +++ b/eng/generate-compiler-code.ps1 @@ -9,115 +9,115 @@ Set-StrictMode -version 2.0 $ErrorActionPreference="Stop" function Run-Tool($projectFilePath, $toolArgs) { - $toolName = Split-Path -leaf $projectFilePath - Write-Host "Running $toolName $toolArgs" - Exec-Console $dotnet "run -p $projectFilePath $toolArgs" + $toolName = Split-Path -leaf $projectFilePath + Write-Host "Running $toolName $toolArgs" + Exec-Console $dotnet "run -p $projectFilePath $toolArgs" } function Run-LanguageCore($language, $languageSuffix, $languageDir, $syntaxProject, $errorFactsProject, $generatedDir, $generatedTestDir) { - $syntaxFilePath = Join-Path $languageDir "Syntax\Syntax.xml" - $syntaxTestFilePath = Join-Path $generatedTestDir "Syntax.Test.xml.Generated.$($languageSuffix)" - $boundFilePath = Join-Path $languageDir "BoundTree\BoundNodes.xml" - $boundGeneratedFilePath = Join-Path $generatedDir "BoundNodes.xml.Generated.$($languageSuffix)" - $errorFileName = if ($language -eq "CSharp") { "ErrorCode.cs" } else { "Errors.vb" } - $errorFilePath = Join-Path $languageDir "Errors\$errorFileName" - $errorGeneratedFilePath = Join-Path $generatedDir "ErrorFacts.Generated.$($languageSuffix)" - - Create-Directory $generatedDir - Create-Directory $generatedTestDir - Run-Tool $syntaxProject "`"$syntaxFilePath`" `"$generatedDir`"" - Run-Tool $syntaxProject "`"$syntaxFilePath`" `"$syntaxTestFilePath`" /test" - Run-Tool $boundTreeGenProject "$language `"$boundFilePath`" `"$boundGeneratedFilePath`"" - Run-Tool $errorFactsProject "`"$errorFilePath`" `"$errorGeneratedFilePath`"" + $syntaxFilePath = Join-Path $languageDir "Syntax\Syntax.xml" + $syntaxTestFilePath = Join-Path $generatedTestDir "Syntax.Test.xml.Generated.$($languageSuffix)" + $boundFilePath = Join-Path $languageDir "BoundTree\BoundNodes.xml" + $boundGeneratedFilePath = Join-Path $generatedDir "BoundNodes.xml.Generated.$($languageSuffix)" + $errorFileName = if ($language -eq "CSharp") { "ErrorCode.cs" } else { "Errors.vb" } + $errorFilePath = Join-Path $languageDir "Errors\$errorFileName" + $errorGeneratedFilePath = Join-Path $generatedDir "ErrorFacts.Generated.$($languageSuffix)" + + Create-Directory $generatedDir + Create-Directory $generatedTestDir + Run-Tool $syntaxProject "`"$syntaxFilePath`" `"$generatedDir`"" + Run-Tool $syntaxProject "`"$syntaxFilePath`" `"$syntaxTestFilePath`" /test" + Run-Tool $boundTreeGenProject "$language `"$boundFilePath`" `"$boundGeneratedFilePath`"" + Run-Tool $errorFactsProject "`"$errorFilePath`" `"$errorGeneratedFilePath`"" } # Test the contents of our generated files to ensure they are equal. Compares our checked # in code with the freshly generated code. function Test-GeneratedContent($generatedDir, $scratchDir) { - $algo = "MD5" - foreach ($fileName in (Get-ChildItem $scratchDir)) { - Write-Host "Checking $fileName" - $realFilePath = Join-Path $generatedDir $fileName - $scratchFilePath = Join-Path $scratchDir $fileName - $scratchHash = (Get-FileHash $scratchFilePath -algorithm $algo).Hash - $realHash = (Get-FileHash $realFilePath -algorithm $algo).Hash - if ($scratchHash -ne $realHash) { - Write-Host "Files are out of date" - Write-Host "Run $(Join-Path $PSScriptRoot generate-compiler-code.ps1) to refresh" - throw "Files are out of date" - } + $algo = "MD5" + foreach ($fileName in (Get-ChildItem $scratchDir)) { + Write-Host "Checking $fileName" + $realFilePath = Join-Path $generatedDir $fileName + $scratchFilePath = Join-Path $scratchDir $fileName + $scratchHash = (Get-FileHash $scratchFilePath -algorithm $algo).Hash + $realHash = (Get-FileHash $realFilePath -algorithm $algo).Hash + if ($scratchHash -ne $realHash) { + Write-Host "Files are out of date" + Write-Host "Run $(Join-Path $PSScriptRoot generate-compiler-code.ps1) to refresh" + throw "Files are out of date" } + } } function Run-Language($language, $languageSuffix, $languageDir, $languageTestDir, $syntaxTool, $errorFactsTool) { - $generatedDir = Join-Path $languageDir "Generated" - $generatedTestDir = Join-Path $languageTestDir "Generated" - if (-not $test) { - Run-LanguageCore $language $languageSuffix $languageDir $syntaxTool $errorFactsTool $generatedDir $generatedTestDir - } - else { - $scratchDir = Join-Path $generationTempDir "$language\Src" - $scratchTestDir = Join-Path $generationTempDir "$language\Test" - Run-LanguageCore $language $languageSuffix $languageDir $syntaxTool $errorFactsTool $scratchDir $scratchTestDir - Test-GeneratedContent $generatedDir $scratchDir - Test-GeneratedContent $generatedTestDir $scratchTestDir - } + $generatedDir = Join-Path $languageDir "Generated" + $generatedTestDir = Join-Path $languageTestDir "Generated" + if (-not $test) { + Run-LanguageCore $language $languageSuffix $languageDir $syntaxTool $errorFactsTool $generatedDir $generatedTestDir + } + else { + $scratchDir = Join-Path $generationTempDir "$language\Src" + $scratchTestDir = Join-Path $generationTempDir "$language\Test" + Run-LanguageCore $language $languageSuffix $languageDir $syntaxTool $errorFactsTool $scratchDir $scratchTestDir + Test-GeneratedContent $generatedDir $scratchDir + Test-GeneratedContent $generatedTestDir $scratchTestDir + } } function Run-GetTextCore($generatedDir) { - $syntaxFilePath = Join-Path $basicDir "Syntax\Syntax.xml" - $syntaxTextFilePath = Join-Path $generatedDir "Syntax.xml.GetText.Generated.vb" + $syntaxFilePath = Join-Path $basicDir "Syntax\Syntax.xml" + $syntaxTextFilePath = Join-Path $generatedDir "Syntax.xml.GetText.Generated.vb" - Create-Directory $generatedDir - Run-Tool $basicSyntaxProject "`"$syntaxFilePath`" `"$syntaxTextFilePath`" /gettext" + Create-Directory $generatedDir + Run-Tool $basicSyntaxProject "`"$syntaxFilePath`" `"$syntaxTextFilePath`" /gettext" } function Run-GetText() { - $generatedDir = Join-Path $RepoRoot "src\ExpressionEvaluator\VisualBasic\Source\ResultProvider\Generated" - if (-not $test) { - Run-GetTextCore $generatedDir - } - else { - $scratchDir = Join-Path $generationTempDir "VB\GetText" - Run-GetTextCore $scratchDir - Test-GeneratedContent $generatedDir $scratchDir - } + $generatedDir = Join-Path $RepoRoot "src\ExpressionEvaluator\VisualBasic\Source\ResultProvider\Generated" + if (-not $test) { + Run-GetTextCore $generatedDir + } + else { + $scratchDir = Join-Path $generationTempDir "VB\GetText" + Run-GetTextCore $scratchDir + Test-GeneratedContent $generatedDir $scratchDir + } } function Get-ToolPath($projectRelativePath) { - $p = Join-Path 'src\Tools\Source\CompilerGeneratorTools\Source' $projectRelativePath - $p = Join-Path $RepoRoot $p - return $p + $p = Join-Path 'src\Tools\Source\CompilerGeneratorTools\Source' $projectRelativePath + $p = Join-Path $RepoRoot $p + return $p } try { - . (Join-Path $PSScriptRoot "build-utils.ps1") - Push-Location $RepoRoot + . (Join-Path $PSScriptRoot "build-utils.ps1") + Push-Location $RepoRoot - $dotnet = Ensure-DotnetSdk - $boundTreeGenProject = Get-ToolPath 'BoundTreeGenerator\CompilersBoundTreeGenerator.csproj' + $dotnet = Ensure-DotnetSdk + $boundTreeGenProject = Get-ToolPath 'BoundTreeGenerator\CompilersBoundTreeGenerator.csproj' - $csharpDir = Join-Path $RepoRoot "src\Compilers\CSharp\Portable" - $csharpTestDir = Join-Path $RepoRoot "src\Compilers\CSharp\Test\Syntax" - $csharpSyntaxProject = Get-ToolPath 'CSharpSyntaxGenerator\CSharpSyntaxGenerator.csproj' - $csharpErrorFactsProject = Get-ToolPath 'CSharpErrorFactsGenerator\CSharpErrorFactsGenerator.csproj' - $basicDir = Join-Path $RepoRoot "src\Compilers\VisualBasic\Portable" - $basicTestDir = Join-Path $RepoRoot "src\Compilers\VisualBasic\Test\Syntax" - $basicSyntaxProject = Get-ToolPath 'VisualBasicSyntaxGenerator\VisualBasicSyntaxGenerator.vbproj' - $basicErrorFactsProject = Get-ToolPath 'VisualBasicErrorFactsGenerator\VisualBasicErrorFactsGenerator.vbproj' - $generationTempDir = Join-Path $RepoRoot "artifacts\log\$configuration\Generated" + $csharpDir = Join-Path $RepoRoot "src\Compilers\CSharp\Portable" + $csharpTestDir = Join-Path $RepoRoot "src\Compilers\CSharp\Test\Syntax" + $csharpSyntaxProject = Get-ToolPath 'CSharpSyntaxGenerator\CSharpSyntaxGenerator.csproj' + $csharpErrorFactsProject = Get-ToolPath 'CSharpErrorFactsGenerator\CSharpErrorFactsGenerator.csproj' + $basicDir = Join-Path $RepoRoot "src\Compilers\VisualBasic\Portable" + $basicTestDir = Join-Path $RepoRoot "src\Compilers\VisualBasic\Test\Syntax" + $basicSyntaxProject = Get-ToolPath 'VisualBasicSyntaxGenerator\VisualBasicSyntaxGenerator.vbproj' + $basicErrorFactsProject = Get-ToolPath 'VisualBasicErrorFactsGenerator\VisualBasicErrorFactsGenerator.vbproj' + $generationTempDir = Join-Path $RepoRoot "artifacts\log\$configuration\Generated" - Run-Language "CSharp" "cs" $csharpDir $csharpTestDir $csharpSyntaxProject $csharpErrorFactsProject - Run-Language "VB" "vb" $basicDir $basicTestDir $basicSyntaxProject $basicErrorFactsProject - Run-GetText + Run-Language "CSharp" "cs" $csharpDir $csharpTestDir $csharpSyntaxProject $csharpErrorFactsProject + Run-Language "VB" "vb" $basicDir $basicTestDir $basicSyntaxProject $basicErrorFactsProject + Run-GetText - exit 0 + exit 0 } catch { - Write-Host $_ - exit 1 + Write-Host $_ + exit 1 } finally { - Pop-Location + Pop-Location } diff --git a/eng/publish-assets.ps1 b/eng/publish-assets.ps1 index 0344f0cfda5b7..5cbce726296c6 100644 --- a/eng/publish-assets.ps1 +++ b/eng/publish-assets.ps1 @@ -8,165 +8,165 @@ # degree of regularity. [CmdletBinding(PositionalBinding=$false)] Param( - # Standard options - [string]$configuration = "", - [string]$branchName = "", - [string]$releaseName = "", - [switch]$test, - - # Credentials - [string]$gitHubUserName = "", - [string]$gitHubToken = "", - [string]$gitHubEmail = "", - [string]$nugetApiKey = "", - [string]$myGetApiKey = "" + # Standard options + [string]$configuration = "", + [string]$branchName = "", + [string]$releaseName = "", + [switch]$test, + + # Credentials + [string]$gitHubUserName = "", + [string]$gitHubToken = "", + [string]$gitHubEmail = "", + [string]$nugetApiKey = "", + [string]$myGetApiKey = "" ) Set-StrictMode -version 2.0 $ErrorActionPreference="Stop" [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 function Get-PublishKey([string]$uploadUrl) { - $url = New-Object Uri $uploadUrl - switch ($url.Host) { - "dotnet.myget.org" { return $myGetApiKey } - "api.nuget.org" { return $nugetApiKey } - default { throw "Cannot determine publish key for $uploadUrl" } - } + $url = New-Object Uri $uploadUrl + switch ($url.Host) { + "dotnet.myget.org" { return $myGetApiKey } + "api.nuget.org" { return $nugetApiKey } + default { throw "Cannot determine publish key for $uploadUrl" } + } } # Publish the NuGet packages to the specified URL function Publish-NuGet([string]$packageDir, [string]$uploadUrl) { - Push-Location $packageDir - try { - Write-Host "Publishing $(Split-Path -leaf $packageDir) to $uploadUrl" - $apiKey = Get-PublishKey $uploadUrl - foreach ($package in Get-ChildItem *.nupkg) { - $nupkg = Split-Path -Leaf $package - Write-Host " Publishing $nupkg" - if (-not (Test-Path $nupkg)) { - throw "$nupkg does not exist" - } - - if (-not $test) { - Exec-Console $dotnet "nuget push $nupkg --source $uploadUrl --api-key $apiKey" - } - } - } - finally { - Pop-Location + Push-Location $packageDir + try { + Write-Host "Publishing $(Split-Path -leaf $packageDir) to $uploadUrl" + $apiKey = Get-PublishKey $uploadUrl + foreach ($package in Get-ChildItem *.nupkg) { + $nupkg = Split-Path -Leaf $package + Write-Host " Publishing $nupkg" + if (-not (Test-Path $nupkg)) { + throw "$nupkg does not exist" + } + + if (-not $test) { + Exec-Console $dotnet "nuget push $nupkg --source $uploadUrl --api-key $apiKey" + } } + } + finally { + Pop-Location + } } function Publish-Vsix([string]$uploadUrl) { - Write-Host "Publishing VSIX to $uploadUrl" - $apiKey = Get-PublishKey $uploadUrl - $extensions = [xml](Get-Content (Join-Path $EngRoot "config\PublishVsix.MyGet.config")) - foreach ($extension in $extensions.extensions.extension) { - $vsix = Join-Path $VSSetupDir ($extension.id + ".vsix") - if (-not (Test-Path $vsix)) { - throw "VSIX $vsix does not exist" - } - - Write-Host " Publishing '$vsix'" - if (-not $test) { - $response = Invoke-WebRequest -Uri $uploadUrl -Headers @{"X-NuGet-ApiKey"=$apiKey} -ContentType 'multipart/form-data' -InFile $vsix -Method Post -UseBasicParsing - if ($response.StatusCode -ne 201) { - throw "Failed to upload VSIX extension: $vsix. Upload failed with Status code: $response.StatusCode" - } - } + Write-Host "Publishing VSIX to $uploadUrl" + $apiKey = Get-PublishKey $uploadUrl + $extensions = [xml](Get-Content (Join-Path $EngRoot "config\PublishVsix.MyGet.config")) + foreach ($extension in $extensions.extensions.extension) { + $vsix = Join-Path $VSSetupDir ($extension.id + ".vsix") + if (-not (Test-Path $vsix)) { + throw "VSIX $vsix does not exist" } + + Write-Host " Publishing '$vsix'" + if (-not $test) { + $response = Invoke-WebRequest -Uri $uploadUrl -Headers @{"X-NuGet-ApiKey"=$apiKey} -ContentType 'multipart/form-data' -InFile $vsix -Method Post -UseBasicParsing + if ($response.StatusCode -ne 201) { + throw "Failed to upload VSIX extension: $vsix. Upload failed with Status code: $response.StatusCode" + } + } + } } function Publish-Channel([string]$packageDir, [string]$name) { - $publish = GetProjectOutputBinary "RoslynPublish.exe" - $args = "-nugetDir `"$packageDir`" -channel $name -gu $gitHubUserName -gt $gitHubToken -ge $githubEmail" - Write-Host "Publishing $packageDir to channel $name" - if (-not $test) { - Exec-Console $publish $args - } + $publish = GetProjectOutputBinary "RoslynPublish.exe" + $args = "-nugetDir `"$packageDir`" -channel $name -gu $gitHubUserName -gt $gitHubToken -ge $githubEmail" + Write-Host "Publishing $packageDir to channel $name" + if (-not $test) { + Exec-Console $publish $args + } } # Do basic verification on the values provided in the publish configuration function Test-Entry($publishData, [switch]$isBranch) { - if ($isBranch) { - if ($publishData.nuget -ne $null) { - foreach ($nugetKind in $publishData.nugetKind) { - if ($nugetKind -ne "PerBuildPreRelease" -and $nugetKind -ne "Shipping" -and $nugetKind -ne "NonShipping") { + if ($isBranch) { + if ($publishData.nuget -ne $null) { + foreach ($nugetKind in $publishData.nugetKind) { + if ($nugetKind -ne "PerBuildPreRelease" -and $nugetKind -ne "Shipping" -and $nugetKind -ne "NonShipping") { throw "Branches are only allowed to publish Shipping, NonShipping, or PerBuildPreRelease" - } - } } + } } + } } # Publish a given entry: branch or release. function Publish-Entry($publishData, [switch]$isBranch) { - Test-Entry $publishData -isBranch:$isBranch - - # First publish the NuGet packages to the specified feeds - foreach ($url in $publishData.nuget) { - foreach ($nugetKind in $publishData.nugetKind) { - Publish-NuGet (Join-Path $PackagesDir $nugetKind) $url - } - } + Test-Entry $publishData -isBranch:$isBranch - # Next publish the VSIX to the specified feeds - $vsixData = $publishData.vsix - if ($vsixData -ne $null) { - Publish-Vsix $vsixData + # First publish the NuGet packages to the specified feeds + foreach ($url in $publishData.nuget) { + foreach ($nugetKind in $publishData.nugetKind) { + Publish-NuGet (Join-Path $PackagesDir $nugetKind) $url } - - # Finally get our channels uploaded to versions - foreach ($channel in $publishData.channels) { - foreach ($nugetKind in $publishData.nugetKind) { - Publish-Channel (Join-Path $PackagesDir $nugetKind) $channel - } + } + + # Next publish the VSIX to the specified feeds + $vsixData = $publishData.vsix + if ($vsixData -ne $null) { + Publish-Vsix $vsixData + } + + # Finally get our channels uploaded to versions + foreach ($channel in $publishData.channels) { + foreach ($nugetKind in $publishData.nugetKind) { + Publish-Channel (Join-Path $PackagesDir $nugetKind) $channel } + } - exit 0 + exit 0 } try { - if ($configuration -eq "") { - Write-Host "Must provide the build configuration with -configuration" - exit 1 - } - - . (Join-Path $PSScriptRoot "build-utils.ps1") + if ($configuration -eq "") { + Write-Host "Must provide the build configuration with -configuration" + exit 1 + } - $dotnet = Ensure-DotnetSdk + . (Join-Path $PSScriptRoot "build-utils.ps1") - if ($branchName -ne "" -and $releaseName -ne "") { - Write-Host "Can only specify -branchName or -releaseName, not both" - exit 1 - } + $dotnet = Ensure-DotnetSdk - if ($branchName -ne "") { - $data = GetBranchPublishData $branchName - if ($data -eq $null) { - Write-Host "Branch $branchName not listed for publishing." - exit 0 - } + if ($branchName -ne "" -and $releaseName -ne "") { + Write-Host "Can only specify -branchName or -releaseName, not both" + exit 1 + } - Publish-Entry $data -isBranch:$true + if ($branchName -ne "") { + $data = GetBranchPublishData $branchName + if ($data -eq $null) { + Write-Host "Branch $branchName not listed for publishing." + exit 0 } - elseif ($releaseName -ne "") { - $data = GetReleasePublishData $releaseName - if ($data -eq $null) { - Write-Host "Release $releaseName not listed for publishing." - exit 1 - } - Publish-Entry $data -isBranch:$false - } - else { - Write-Host "Need to specify -branchName or -releaseName" - exit 1 + Publish-Entry $data -isBranch:$true + } + elseif ($releaseName -ne "") { + $data = GetReleasePublishData $releaseName + if ($data -eq $null) { + Write-Host "Release $releaseName not listed for publishing." + exit 1 } + + Publish-Entry $data -isBranch:$false + } + else { + Write-Host "Need to specify -branchName or -releaseName" + exit 1 + } } catch { - Write-Host $_ - Write-Host $_.Exception - Write-Host $_.ScriptStackTrace - exit 1 + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + exit 1 } diff --git a/eng/test-build-correctness.ps1 b/eng/test-build-correctness.ps1 index f9b856242d8a9..fd975b30456c7 100644 --- a/eng/test-build-correctness.ps1 +++ b/eng/test-build-correctness.ps1 @@ -1,70 +1,70 @@ <# - This script drives the Jenkins verification that our build is correct. In particular: + This script drives the Jenkins verification that our build is correct. In particular: - - Our build has no double writes - - Our project.json files are consistent - - Our build files are well structured - - Our solution states are consistent - - Our generated files are consistent + - Our build has no double writes + - Our project.json files are consistent + - Our build files are well structured + - Our solution states are consistent + - Our generated files are consistent #> [CmdletBinding(PositionalBinding=$false)] param( - [string]$configuration = "Debug", - [switch]$help) + [string]$configuration = "Debug", + [switch]$help) Set-StrictMode -version 2.0 $ErrorActionPreference="Stop" function Print-Usage() { - Write-Host "Usage: test-build-correctness.ps1" - Write-Host " -configuration Build configuration ('Debug' or 'Release')" + Write-Host "Usage: test-build-correctness.ps1" + Write-Host " -configuration Build configuration ('Debug' or 'Release')" } try { - if ($help) { - Print-Usage - exit 0 - } + if ($help) { + Print-Usage + exit 0 + } - $ci = $true + $ci = $true - . (Join-Path $PSScriptRoot "build-utils.ps1") - Push-Location $RepoRoot + . (Join-Path $PSScriptRoot "build-utils.ps1") + Push-Location $RepoRoot - Write-Host "Building Roslyn" - Exec-Block { & (Join-Path $PSScriptRoot "build.ps1") -restore -build -ci:$ci -configuration:$configuration -pack -binaryLog -useGlobalNuGetCache:$false -warnAsError:$false -properties "/p:RoslynEnforceCodeStyle=true"} + Write-Host "Building Roslyn" + Exec-Block { & (Join-Path $PSScriptRoot "build.ps1") -restore -build -ci:$ci -configuration:$configuration -pack -binaryLog -useGlobalNuGetCache:$false -warnAsError:$false -properties "/p:RoslynEnforceCodeStyle=true"} - # Verify the state of our various build artifacts - Write-Host "Running BuildBoss" - $buildBossPath = GetProjectOutputBinary "BuildBoss.exe" - Exec-Console $buildBossPath "-r `"$RepoRoot`" -c $configuration" - Write-Host "" + # Verify the state of our various build artifacts + Write-Host "Running BuildBoss" + $buildBossPath = GetProjectOutputBinary "BuildBoss.exe" + Exec-Console $buildBossPath "-r `"$RepoRoot`" -c $configuration" + Write-Host "" - # Verify the state of our generated syntax files - Write-Host "Checking generated compiler files" - Exec-Block { & (Join-Path $PSScriptRoot "generate-compiler-code.ps1") -test -configuration:$configuration } - Write-Host "" - - # Verify the state of creating run settings for OptProf - Write-Host "Checking OptProf run settings generation" + # Verify the state of our generated syntax files + Write-Host "Checking generated compiler files" + Exec-Block { & (Join-Path $PSScriptRoot "generate-compiler-code.ps1") -test -configuration:$configuration } + Write-Host "" + + # Verify the state of creating run settings for OptProf + Write-Host "Checking OptProf run settings generation" - # create a fake BootstrapperInfo.json file - $bootstrapperInfoPath = Join-Path $TempDir "BootstrapperInfo.json" - $bootstrapperInfoContent = "[{""BuildDrop"": ""https://vsdrop.corp.microsoft.com/file/v1/Products/42.42.42.42/42.42.42.42""}]" - $bootstrapperInfoContent | Set-Content $bootstrapperInfoPath + # create a fake BootstrapperInfo.json file + $bootstrapperInfoPath = Join-Path $TempDir "BootstrapperInfo.json" + $bootstrapperInfoContent = "[{""BuildDrop"": ""https://vsdrop.corp.microsoft.com/file/v1/Products/42.42.42.42/42.42.42.42""}]" + $bootstrapperInfoContent | Set-Content $bootstrapperInfoPath - # generate run settings - Exec-Block { & (Join-Path $PSScriptRoot "common\sdk-task.ps1") -configuration:$configuration -task VisualStudio.BuildIbcTrainingSettings /p:VisualStudioDropName="Products/DummyDrop" /p:BootstrapperInfoPath=$bootstrapperInfoPath } - - exit 0 + # generate run settings + Exec-Block { & (Join-Path $PSScriptRoot "common\sdk-task.ps1") -configuration:$configuration -task VisualStudio.BuildIbcTrainingSettings /p:VisualStudioDropName="Products/DummyDrop" /p:BootstrapperInfoPath=$bootstrapperInfoPath } + + exit 0 } catch [exception] { - Write-Host $_ - Write-Host $_.Exception - exit 1 + Write-Host $_ + Write-Host $_.Exception + exit 1 } finally { - Pop-Location + Pop-Location } diff --git a/eng/test-determinism.ps1 b/eng/test-determinism.ps1 index 42203901fea19..65d3b062e9c9f 100644 --- a/eng/test-determinism.ps1 +++ b/eng/test-determinism.ps1 @@ -10,16 +10,16 @@ Set-StrictMode -version 2.0 $ErrorActionPreference = "Stop" function Print-Usage() { - Write-Host "Usage: test-determinism.ps1" - Write-Host " -configuration Build configuration ('Debug' or 'Release')" - Write-Host " -msbuildEngine Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)." - Write-Host " -bootstrapDir Directory containing the bootstrap compiler" - Write-Host " -altRootDrive The drive we build on (via subst) for verifying pathmap implementation" + Write-Host "Usage: test-determinism.ps1" + Write-Host " -configuration Build configuration ('Debug' or 'Release')" + Write-Host " -msbuildEngine Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)." + Write-Host " -bootstrapDir Directory containing the bootstrap compiler" + Write-Host " -altRootDrive The drive we build on (via subst) for verifying pathmap implementation" } if ($help) { - Print-Usage - exit 0 + Print-Usage + exit 0 } # List of binary names that should be skipped because they have a known issue that @@ -27,83 +27,83 @@ if ($help) { $script:skipList = @() function Run-Build([string]$rootDir, [string]$logFileName) { - # Clean out the previous run - Write-Host "Cleaning binaries" - $stopWatch = [System.Diagnostics.StopWatch]::StartNew() - Remove-Item -Recurse (Get-BinDir $rootDir) - Remove-Item -Recurse (Get-ObjDir $rootDir) - $stopWatch.Stop() - Write-Host "Cleaning took $($stopWatch.Elapsed)" + # Clean out the previous run + Write-Host "Cleaning binaries" + $stopWatch = [System.Diagnostics.StopWatch]::StartNew() + Remove-Item -Recurse (Get-BinDir $rootDir) + Remove-Item -Recurse (Get-ObjDir $rootDir) + $stopWatch.Stop() + Write-Host "Cleaning took $($stopWatch.Elapsed)" - $solution = Join-Path $rootDir "Roslyn.sln" + $solution = Join-Path $rootDir "Roslyn.sln" - Write-Host "Restoring $solution" - Run-MSBuild $solution "/t:Restore" -logFileName:"Restore-$logFileName" + Write-Host "Restoring $solution" + Run-MSBuild $solution "/t:Restore" -logFileName:"Restore-$logFileName" - Write-Host "Building $solution" - Run-MSBuild $solution "/p:DebugDeterminism=true /p:Features=`"debug-determinism`" /p:DeployExtension=false" -logFileName:$logFileName + Write-Host "Building $solution" + Run-MSBuild $solution "/p:DebugDeterminism=true /p:Features=`"debug-determinism`" /p:DeployExtension=false" -logFileName:$logFileName } function Get-ObjDir([string]$rootDir) { - return Join-Path $rootDir "artifacts\obj" + return Join-Path $rootDir "artifacts\obj" } function Get-BinDir([string]$rootDir) { - return Join-Path $rootDir "artifacts\bin" + return Join-Path $rootDir "artifacts\bin" } # Return all of the files that need to be processed for determinism under the given # directory. function Get-FilesToProcess([string]$rootDir) { - $objDir = Get-ObjDir $rootDir - foreach ($item in Get-ChildItem -re -in *.dll,*.exe,*.pdb,*.sourcelink.json $objDir) { - $filePath = $item.FullName - $fileName = Split-Path -leaf $filePath - - if ($skipList.Contains($fileName)) { - continue; - } - - $fileId = $filePath.Substring($objDir.Length).Replace("\", ".") - $fileHash = (Get-FileHash $filePath -algorithm MD5).Hash - - $data = @{} - $data.Hash = $fileHash - $data.Content = [IO.File]::ReadAllBytes($filePath) - $data.FileId = $fileId - $data.FileName = $fileName - $data.FilePath = $filePath - - $keyFilePath = $filePath + ".key" - $keyFileName = Split-Path -leaf $keyFilePath - if (Test-Path $keyFilePath) { - $data.KeyFileName = $keyFileName - $data.KeyFilePath = $keyFilePath - $data.KeyFileContent = [IO.File]::ReadAllBytes($keyFilePath) - } - else { - $data.KeyFileName = "" - $data.KeyFilePath = "" - $data.KeyFileContent = $null - } - - Write-Output $data + $objDir = Get-ObjDir $rootDir + foreach ($item in Get-ChildItem -re -in *.dll,*.exe,*.pdb,*.sourcelink.json $objDir) { + $filePath = $item.FullName + $fileName = Split-Path -leaf $filePath + + if ($skipList.Contains($fileName)) { + continue; + } + + $fileId = $filePath.Substring($objDir.Length).Replace("\", ".") + $fileHash = (Get-FileHash $filePath -algorithm MD5).Hash + + $data = @{} + $data.Hash = $fileHash + $data.Content = [IO.File]::ReadAllBytes($filePath) + $data.FileId = $fileId + $data.FileName = $fileName + $data.FilePath = $filePath + + $keyFilePath = $filePath + ".key" + $keyFileName = Split-Path -leaf $keyFilePath + if (Test-Path $keyFilePath) { + $data.KeyFileName = $keyFileName + $data.KeyFilePath = $keyFilePath + $data.KeyFileContent = [IO.File]::ReadAllBytes($keyFilePath) } + else { + $data.KeyFileName = "" + $data.KeyFilePath = "" + $data.KeyFileContent = $null + } + + Write-Output $data + } } # This will build up the map of all of the binaries and their respective hashes. function Record-Binaries([string]$rootDir) { - $stopWatch = [System.Diagnostics.StopWatch]::StartNew() - Write-Host "Recording file hashes" - - $map = @{ } - foreach ($fileData in Get-FilesToProcess $rootDir) { - Write-Host "`t$($fileData.FileName) = $($fileData.Hash)" - $map[$fileData.FileId] = $fileData - } - $stopWatch.Stop() - Write-Host "Recording took $($stopWatch.Elapsed)" - return $map + $stopWatch = [System.Diagnostics.StopWatch]::StartNew() + Write-Host "Recording file hashes" + + $map = @{ } + foreach ($fileData in Get-FilesToProcess $rootDir) { + Write-Host "`t$($fileData.FileName) = $($fileData.Hash)" + $map[$fileData.FileId] = $fileData + } + $stopWatch.Stop() + Write-Host "Recording took $($stopWatch.Elapsed)" + return $map } # This is a sanity check to ensure that we're actually putting the right entries into @@ -111,155 +111,155 @@ function Record-Binaries([string]$rootDir) { # that this test fails beacuse we didn't record the binaries we intended to record. function Test-MapContents($dataMap) { - # Sanity check to ensure we didn't return a false positive because we failed - # to examine any binaries. - if ($dataMap.Count -lt 40) { - throw "Didn't find the expected count of binaries" + # Sanity check to ensure we didn't return a false positive because we failed + # to examine any binaries. + if ($dataMap.Count -lt 40) { + throw "Didn't find the expected count of binaries" + } + + # Test for some well known binaries + $list = @( + "Microsoft.CodeAnalysis.dll", + "Microsoft.CodeAnalysis.CSharp.dll", + "Microsoft.CodeAnalysis.Workspaces.dll", + "Microsoft.VisualStudio.LanguageServices.Implementation.dll") + + foreach ($fileName in $list) { + $found = $false + foreach ($value in $dataMap.Values) { + if ($value.FileName -eq $fileName) { + $found = $true + break; + } } - # Test for some well known binaries - $list = @( - "Microsoft.CodeAnalysis.dll", - "Microsoft.CodeAnalysis.CSharp.dll", - "Microsoft.CodeAnalysis.Workspaces.dll", - "Microsoft.VisualStudio.LanguageServices.Implementation.dll") - - foreach ($fileName in $list) { - $found = $false - foreach ($value in $dataMap.Values) { - if ($value.FileName -eq $fileName) { - $found = $true - break; - } - } - - if (-not $found) { - throw "Did not find the expected binary $fileName" - } + if (-not $found) { + throw "Did not find the expected binary $fileName" } + } } function Test-Build([string]$rootDir, $dataMap, [string]$logFileName) { - Run-Build $rootDir -logFile $logFileName - - $errorList = @() - $allGood = $true - - Write-Host "Testing the binaries" - $stopWatch = [System.Diagnostics.StopWatch]::StartNew() - foreach ($fileData in Get-FilesToProcess $rootDir) { - $fileId = $fileData.FileId - $fileName = $fileData.FileName - $filePath = $fileData.FilePath - - if (-not $dataMap.Contains($fileId)) { - Write-Host "ERROR! Missing entry in map $fileId->$filePath" - $allGood = $false - continue - } - - $oldfileData = $datamap[$fileId] - if ($fileData.Hash -ne $oldFileData.Hash) { - Write-Host "`tERROR! $fileName contents don't match" - $allGood = $false - $errorList += $fileName - - # Save out the original and baseline for investigation - [IO.File]::WriteAllBytes((Join-Path $errorDirLeft $fileName), $oldFileData.Content) - Copy-Item $filePath (Join-Path $errorDirRight $fileName) - - # Copy the key files if available too - $keyFileName = $oldFileData.KeyFileName - if ($keyFileName -ne "") { - [IO.File]::WriteAllBytes((Join-Path $errorDirLeft $keyFileName), $oldFileData.KeyFileContent) - Copy-Item $fileData.KeyFilePath (Join-Path $errorDirRight $keyFileName) - } - - continue - } - - Write-Host "`tVerified $fileName" + Run-Build $rootDir -logFile $logFileName + + $errorList = @() + $allGood = $true + + Write-Host "Testing the binaries" + $stopWatch = [System.Diagnostics.StopWatch]::StartNew() + foreach ($fileData in Get-FilesToProcess $rootDir) { + $fileId = $fileData.FileId + $fileName = $fileData.FileName + $filePath = $fileData.FilePath + + if (-not $dataMap.Contains($fileId)) { + Write-Host "ERROR! Missing entry in map $fileId->$filePath" + $allGood = $false + continue } - if (-not $allGood) { - Write-Host "Determinism failed for the following binaries:" - foreach ($name in $errorList) { - Write-Host "`t$name" - } + $oldfileData = $datamap[$fileId] + if ($fileData.Hash -ne $oldFileData.Hash) { + Write-Host "`tERROR! $fileName contents don't match" + $allGood = $false + $errorList += $fileName + + # Save out the original and baseline for investigation + [IO.File]::WriteAllBytes((Join-Path $errorDirLeft $fileName), $oldFileData.Content) + Copy-Item $filePath (Join-Path $errorDirRight $fileName) - Write-Host "Archiving failure information" - $zipFile = Join-Path $LogDir "determinism.zip" - Add-Type -Assembly "System.IO.Compression.FileSystem"; - [System.IO.Compression.ZipFile]::CreateFromDirectory($script:errorDir, $zipFile, "Fastest", $true); + # Copy the key files if available too + $keyFileName = $oldFileData.KeyFileName + if ($keyFileName -ne "") { + [IO.File]::WriteAllBytes((Join-Path $errorDirLeft $keyFileName), $oldFileData.KeyFileContent) + Copy-Item $fileData.KeyFilePath (Join-Path $errorDirRight $keyFileName) + } - Write-Host "Please send $zipFile to compiler team for analysis" - exit 1 + continue } - $stopWatch.Stop() - Write-Host "Testing took $($stopWatch.Elapsed)" + Write-Host "`tVerified $fileName" + } + + if (-not $allGood) { + Write-Host "Determinism failed for the following binaries:" + foreach ($name in $errorList) { + Write-Host "`t$name" + } + + Write-Host "Archiving failure information" + $zipFile = Join-Path $LogDir "determinism.zip" + Add-Type -Assembly "System.IO.Compression.FileSystem"; + [System.IO.Compression.ZipFile]::CreateFromDirectory($script:errorDir, $zipFile, "Fastest", $true); + + Write-Host "Please send $zipFile to compiler team for analysis" + exit 1 + } + + $stopWatch.Stop() + Write-Host "Testing took $($stopWatch.Elapsed)" } function Run-Test() { - # Run the initial build so that we can populate the maps - Run-Build $RepoRoot -logFileName "Initial" - $dataMap = Record-Binaries $RepoRoot - Test-MapContents $dataMap - - # Run a test against the source in the same directory location - Test-Build -rootDir $RepoRoot -dataMap $dataMap -logFileName "test1" - - # Run another build in a different source location and verify that path mapping - # allows the build to be identical. To do this we'll copy the entire source - # tree under the artifacts\q directory and run a build from there. - Write-Host "Building in a different directory" - Exec-Command "subst" "$altRootDrive $(Split-Path -parent $RepoRoot)" - try { - $altRootDir = Join-Path "$($altRootDrive)\" (Split-Path -leaf $RepoRoot) - Test-Build -rootDir $altRootDir -dataMap $dataMap -logFileName "test2" - } - finally { - Exec-Command "subst" "$altRootDrive /d" - } + # Run the initial build so that we can populate the maps + Run-Build $RepoRoot -logFileName "Initial" + $dataMap = Record-Binaries $RepoRoot + Test-MapContents $dataMap + + # Run a test against the source in the same directory location + Test-Build -rootDir $RepoRoot -dataMap $dataMap -logFileName "test1" + + # Run another build in a different source location and verify that path mapping + # allows the build to be identical. To do this we'll copy the entire source + # tree under the artifacts\q directory and run a build from there. + Write-Host "Building in a different directory" + Exec-Command "subst" "$altRootDrive $(Split-Path -parent $RepoRoot)" + try { + $altRootDir = Join-Path "$($altRootDrive)\" (Split-Path -leaf $RepoRoot) + Test-Build -rootDir $altRootDir -dataMap $dataMap -logFileName "test2" + } + finally { + Exec-Command "subst" "$altRootDrive /d" + } } try { - . (Join-Path $PSScriptRoot "build-utils.ps1") - - # Create all of the logging directories - $errorDir = Join-Path $LogDir "DeterminismFailures" - $errorDirLeft = Join-Path $errorDir "Left" - $errorDirRight = Join-Path $errorDir "Right" - - Create-Directory $LogDir - Create-Directory $errorDirLeft - Create-Directory $errorDirRight - - $skipAnalyzers = $true - $binaryLog = $true - $officialBuildId = "" - $ci = $true - $properties = @() - - if ($bootstrapDir -eq "") { - $bootstrapDir = Make-BootstrapBuild - } elseif (![IO.Path]::IsPathRooted($script:bootstrapDir)) { - Write-Host "The bootstrap build path must be absolute" - exit 1 - } + . (Join-Path $PSScriptRoot "build-utils.ps1") + + # Create all of the logging directories + $errorDir = Join-Path $LogDir "DeterminismFailures" + $errorDirLeft = Join-Path $errorDir "Left" + $errorDirRight = Join-Path $errorDir "Right" + + Create-Directory $LogDir + Create-Directory $errorDirLeft + Create-Directory $errorDirRight + + $skipAnalyzers = $true + $binaryLog = $true + $officialBuildId = "" + $ci = $true + $properties = @() + + if ($bootstrapDir -eq "") { + $bootstrapDir = Make-BootstrapBuild + } elseif (![IO.Path]::IsPathRooted($script:bootstrapDir)) { + Write-Host "The bootstrap build path must be absolute" + exit 1 + } - Run-Test - exit 0 + Run-Test + exit 0 } catch { - Write-Host $_ - Write-Host $_.Exception - Write-Host $_.ScriptStackTrace - exit 1 + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + exit 1 } finally { - Write-Host "Stopping VBCSCompiler" - Get-Process VBCSCompiler -ErrorAction SilentlyContinue | Stop-Process - Write-Host "Stopped VBCSCompiler" + Write-Host "Stopping VBCSCompiler" + Get-Process VBCSCompiler -ErrorAction SilentlyContinue | Stop-Process + Write-Host "Stopped VBCSCompiler" }