Skip to content

Commit

Permalink
only run bootsrap jobs on compiler changes
Browse files Browse the repository at this point in the history
  • Loading branch information
jmarolf committed Aug 10, 2022
1 parent 44647d8 commit d6037b3
Show file tree
Hide file tree
Showing 4 changed files with 325 additions and 75 deletions.
160 changes: 85 additions & 75 deletions azure-pipelines.yml
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,16 @@ jobs:

# Build Correctness Jobs

- template: eng/pipelines/evaluate-changed-files.yml
parameters:
jobName: Determine_Changes
vmImageName: ubuntu-latest
paths:
- subset: compilers
include:
- src/Compilers/*
- src/Dependencies/*

- template: eng/pipelines/build-windows-job.yml
parameters:
jobName: Correctness_Code_Analysis
Expand Down Expand Up @@ -229,55 +239,55 @@ jobs:
jobName: Correctness_Build_Artifacts
configuration: Release

- ${{ if ne(variables['Build.Reason'], 'PullRequest') }}:
- job: Correctness_Determinism
pool:
name: NetCore1ESPool-Public
demands: ImageOverride -equals Build.Windows.Amd64.VS2022.Pre.Open
timeoutInMinutes: 90
steps:
- template: eng/pipelines/checkout-windows-task.yml

- script: eng/test-determinism.cmd -configuration Debug
displayName: Build - Validate determinism

- template: eng/pipelines/publish-logs.yml
parameters:
jobName: Correctness_Determinism
configuration: Debug
- job: Correctness_Determinism
dependsOn: Determine_Changes
condition: eq(dependencies.Determine_Changes.outputs['SetPathVars_compilers.containsChange'], 'true')
pool:
name: NetCore1ESPool-Public
demands: ImageOverride -equals Build.Windows.Amd64.VS2022.Pre.Open
timeoutInMinutes: 90
steps:
- template: eng/pipelines/checkout-windows-task.yml

- script: eng/test-determinism.cmd -configuration Debug
displayName: Build - Validate determinism

- ${{ if ne(variables['Build.Reason'], 'PullRequest') }}:
- job: Correctness_Bootstrap_Build
pool:
name: NetCore1ESPool-Public
demands: ImageOverride -equals Build.Windows.Amd64.VS2022.Pre.Open
timeoutInMinutes: 90
steps:
- template: eng/pipelines/checkout-windows-task.yml

- script: eng/test-build-correctness.cmd -configuration Release -enableDumps
displayName: Build - Validate correctness

- template: eng/pipelines/publish-logs.yml
parameters:
jobName: Correctness_Bootstrap_Build
configuration: Release

- task: PublishBuildArtifacts@1
displayName: Publish Artifact Packages
inputs:
PathtoPublish: '$(Build.SourcesDirectory)\artifacts\packages\Release\PreRelease'
ArtifactName: 'Bootstrap Packages - PreRelease'
publishLocation: Container
condition: succeeded()

- task: PublishBuildArtifacts@1
displayName: Publish VSIX Packages
inputs:
PathtoPublish: '$(Build.SourcesDirectory)\artifacts\VSSetup\Release\Installer'
ArtifactName: 'Bootstrap VSIX - PreRelease'
publishLocation: Container
condition: succeeded()
- template: eng/pipelines/publish-logs.yml
parameters:
jobName: Correctness_Determinism
configuration: Debug

- job: Correctness_Bootstrap_Build
dependsOn: Determine_Changes
condition: eq(dependencies.Determine_Changes.outputs['SetPathVars_compilers.containsChange'], 'true')
pool:
name: NetCore1ESPool-Public
demands: ImageOverride -equals Build.Windows.Amd64.VS2022.Pre.Open
timeoutInMinutes: 90
steps:
- template: eng/pipelines/checkout-windows-task.yml

- script: eng/test-build-correctness.cmd -configuration Release -enableDumps
displayName: Build - Validate correctness

- template: eng/pipelines/publish-logs.yml
parameters:
jobName: Correctness_Bootstrap_Build
configuration: Release

- task: PublishBuildArtifacts@1
displayName: Publish Artifact Packages
inputs:
PathtoPublish: '$(Build.SourcesDirectory)\artifacts\packages\Release\PreRelease'
ArtifactName: 'Bootstrap Packages - PreRelease'
publishLocation: Container

- task: PublishBuildArtifacts@1
displayName: Publish VSIX Packages
inputs:
PathtoPublish: '$(Build.SourcesDirectory)\artifacts\VSSetup\Release\Installer'
ArtifactName: 'Bootstrap VSIX - PreRelease'
publishLocation: Container

- job: Correctness_TodoCheck
pool:
Expand All @@ -289,34 +299,34 @@ jobs:
- pwsh: eng/todo-check.ps1
displayName: Validate TODO/PROTOTYPE comments are not present

- ${{ if ne(variables['Build.Reason'], 'PullRequest') }}:
- job: Correctness_Rebuild
pool:
name: NetCore1ESPool-Public
demands: ImageOverride -equals Build.Windows.Amd64.VS2022.Pre.Open
timeoutInMinutes: 90
steps:
- template: eng/pipelines/checkout-windows-task.yml

- task: PowerShell@2
displayName: Restore
inputs:
filePath: eng/build.ps1
arguments: -configuration Debug -prepareMachine -ci -restore -binaryLog

- powershell: .\eng\test-rebuild.ps1 -ci -configuration Release
displayName: Run BuildValidator

- task: PublishBuildArtifacts@1
displayName: Publish BuildValidator debug outputs
inputs:
PathtoPublish: '$(Build.SourcesDirectory)/artifacts/BuildValidator'
ArtifactName: 'BuildValidator_DebugOut'
publishLocation: Container
continueOnError: true
condition: failed()
- job: Correctness_Rebuild
dependsOn: Determine_Changes
condition: eq(dependencies.Determine_Changes.outputs['SetPathVars_compilers.containsChange'], 'true')
pool:
name: NetCore1ESPool-Public
demands: ImageOverride -equals Build.Windows.Amd64.VS2022.Pre.Open
timeoutInMinutes: 90
steps:
- template: eng/pipelines/checkout-windows-task.yml

- task: PowerShell@2
displayName: Restore
inputs:
filePath: eng/build.ps1
arguments: -configuration Debug -prepareMachine -ci -restore -binaryLog

- powershell: .\eng\test-rebuild.ps1 -ci -configuration Release
displayName: Run BuildValidator

- task: PublishBuildArtifacts@1
displayName: Publish BuildValidator debug outputs
inputs:
PathtoPublish: '$(Build.SourcesDirectory)/artifacts/BuildValidator'
ArtifactName: 'BuildValidator_DebugOut'
publishLocation: Container
continueOnError: true

- template: eng/pipelines/publish-logs.yml
parameters:
jobName: Correctness_Rebuild
configuration: Release
configuration: Release
168 changes: 168 additions & 0 deletions eng/evaluate-changed-paths.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,168 @@
#!/usr/bin/env bash

# Disable globbing in this bash script since we iterate over path patterns
set -f

# Stop script if unbound variable found (use ${var:-} if intentional)
set -u

# Stop script if command returns non-zero exit code.
# Prevents hidden errors caused by missing error code propagation.
set -e

usage()
{
echo "Script that evaluates changed paths and emits an azure devops variable if the changes contained in the current HEAD against the difftarget meet the includepahts/excludepaths filters:"
echo " --difftarget <value> SHA or branch to diff against. (i.e: HEAD^1, origin/main, 0f4hd36, etc.)"
echo " --excludepaths <value> Escaped list of paths to exclude from diff separated by '+'. (i.e: 'src/libraries/*+'src/installer/*')"
echo " --includepaths <value> Escaped list of paths to include on diff separated by '+'. (i.e: 'src/libraries/System.Private.CoreLib/*')"
echo " --subset Subset name for which we're evaluating in order to include it in logs"
echo " --azurevariable Name of azure devops variable to create if change meets filter criteria"
echo ""

echo "Arguments can also be passed in with a single hyphen."
}

source="${BASH_SOURCE[0]}"

# resolve $source until the file is no longer a symlink
while [[ -h "$source" ]]; do
scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
source="$(readlink "$source")"
# if $source was a relative symlink, we need to resolve it relative to the path where the
# symlink file was located
[[ $source != /* ]] && source="$scriptroot/$source"
done

scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
eng_root=`cd -P "$scriptroot" && pwd`

exclude_paths=()
include_paths=()
subset_name=''
azure_variable=''
diff_target=''

while [[ $# > 0 ]]; do
opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
case "$opt" in
-help|-h)
usage
exit 0
;;
-difftarget)
diff_target=$2
shift
;;
-excludepaths)
IFS='+' read -r -a tmp <<< $2
exclude_paths+=(${tmp[@]})
shift
;;
-includepaths)
IFS='+' read -r -a tmp <<< $2
include_paths+=(${tmp[@]})
shift
;;
-subset)
subset_name=$2
shift
;;
-azurevariable)
azure_variable=$2
shift
;;
esac

shift
done

ci=true # Needed in order to use pipeline-logging-functions.sh
. "$eng_root/common/pipeline-logging-functions.sh"

# -- expected args --
# $@: git diff arguments
customGitDiff() {
(
set -x
git diff -M -C -b --ignore-cr-at-eol --ignore-space-at-eol "$@"
)
}

# runs git diff with supplied filter.
# -- exit codes --
# 0: No match was found
# 1: At least 1 match was found
#
# -- expected args --
# $@: filter string
probePathsWithExitCode() {
local _filter=$@
echo ""
customGitDiff --exit-code --quiet $diff_target -- $_filter
}

# -- expected args --
# $@: filter string
printMatchedPaths() {
local _subset=$subset_name
local _filter=$@
echo ""
echo "----- Matching files for $_subset -----"
customGitDiff --name-only $diff_target -- $_filter
}

probePaths() {
local _subset=$subset_name
local _azure_devops_var_name=$azure_variable
local exclude_path_string=""
local include_path_string=""
local found_applying_changes=false

if [[ ${#exclude_paths[@]} -gt 0 ]]; then
echo ""
echo "******* Probing $_subset exclude paths *******";
for _path in "${exclude_paths[@]}"; do
echo "$_path"
if [[ -z "$exclude_path_string" ]]; then
exclude_path_string=":!$_path"
else
exclude_path_string="$exclude_path_string :!$_path"
fi
done

if ! probePathsWithExitCode $exclude_path_string; then
found_applying_changes=true
printMatchedPaths $exclude_path_string
fi
fi

if [[ $found_applying_changes != true && ${#include_paths[@]} -gt 0 ]]; then
echo ""
echo "******* Probing $_subset include paths *******";
for _path in "${include_paths[@]}"; do
echo "$_path"
if [[ -z "$include_path_string" ]]; then
include_path_string=":$_path"
else
include_path_string="$include_path_string :$_path"
fi
done

if ! probePathsWithExitCode $include_path_string; then
found_applying_changes=true
printMatchedPaths $include_path_string
fi
fi

if [[ $found_applying_changes == true ]]; then
echo ""
echo "Setting pipeline variable $_azure_devops_var_name=true"
Write-PipelineSetVariable -name $_azure_devops_var_name -value true -is_multi_job_variable true
else
echo ""
echo "No changed files for $_subset"
fi
}

probePaths
19 changes: 19 additions & 0 deletions eng/pipelines/evaluate-changed-files-group.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
# This step template evaluates git changes using git based on a include/exclude path filter.
# For more information on how the path evaluation works look at evaluate-changed-paths.sh docs
# at the beginning of that file.

parameters:
# Name for the subset that we're evaluating changes for.
# It is required to name the step correctly and so the variable created can be consumable.
subsetName: ''
# Array containing the arguments that are to be passed down to evaluate-changed-paths.sh
# Note that --azurevariable is always set to containschange, no need to pass it down.
arguments: []

steps:
- ${{ if ne(parameters.arguments[0], '') }}:
- script: eng/evaluate-changed-paths.sh
--azurevariable containsChange
${{ join(' ', parameters.arguments) }}
displayName: Evaluate paths for ${{ parameters.subsetName }}
name: ${{ format('SetPathVars_{0}', parameters.subsetName) }} # need a name to access output variable
Loading

0 comments on commit d6037b3

Please sign in to comment.