From d6037b3b212ae3af71f5216513fbb097a5cb1e9e Mon Sep 17 00:00:00 2001 From: Jonathon Marolf Date: Thu, 21 Jul 2022 16:16:27 -0700 Subject: [PATCH] only run bootsrap jobs on compiler changes --- azure-pipelines.yml | 160 +++++++++-------- eng/evaluate-changed-paths.sh | 168 ++++++++++++++++++ .../evaluate-changed-files-group.yml | 19 ++ eng/pipelines/evaluate-changed-files.yml | 53 ++++++ 4 files changed, 325 insertions(+), 75 deletions(-) create mode 100755 eng/evaluate-changed-paths.sh create mode 100644 eng/pipelines/evaluate-changed-files-group.yml create mode 100644 eng/pipelines/evaluate-changed-files.yml diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 8f0466b0e9997..d6c3a11b0100c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -184,6 +184,16 @@ jobs: # Build Correctness Jobs +- template: eng/pipelines/evaluate-changed-files.yml + parameters: + jobName: Determine_Changes + vmImageName: ubuntu-latest + paths: + - subset: compilers + include: + - src/Compilers/* + - src/Dependencies/* + - template: eng/pipelines/build-windows-job.yml parameters: jobName: Correctness_Code_Analysis @@ -229,55 +239,55 @@ jobs: jobName: Correctness_Build_Artifacts configuration: Release -- ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: - - job: Correctness_Determinism - pool: - name: NetCore1ESPool-Public - demands: ImageOverride -equals Build.Windows.Amd64.VS2022.Pre.Open - timeoutInMinutes: 90 - steps: - - template: eng/pipelines/checkout-windows-task.yml - - - script: eng/test-determinism.cmd -configuration Debug - displayName: Build - Validate determinism - - - template: eng/pipelines/publish-logs.yml - parameters: - jobName: Correctness_Determinism - configuration: Debug +- job: Correctness_Determinism + dependsOn: Determine_Changes + condition: eq(dependencies.Determine_Changes.outputs['SetPathVars_compilers.containsChange'], 'true') + pool: + name: NetCore1ESPool-Public + demands: ImageOverride -equals Build.Windows.Amd64.VS2022.Pre.Open + timeoutInMinutes: 90 + steps: + - template: eng/pipelines/checkout-windows-task.yml + + - script: eng/test-determinism.cmd -configuration Debug + displayName: Build - Validate determinism -- ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: - - job: Correctness_Bootstrap_Build - pool: - name: NetCore1ESPool-Public - demands: ImageOverride -equals Build.Windows.Amd64.VS2022.Pre.Open - timeoutInMinutes: 90 - steps: - - template: eng/pipelines/checkout-windows-task.yml - - - script: eng/test-build-correctness.cmd -configuration Release -enableDumps - displayName: Build - Validate correctness - - - template: eng/pipelines/publish-logs.yml - parameters: - jobName: Correctness_Bootstrap_Build - configuration: Release - - - task: PublishBuildArtifacts@1 - displayName: Publish Artifact Packages - inputs: - PathtoPublish: '$(Build.SourcesDirectory)\artifacts\packages\Release\PreRelease' - ArtifactName: 'Bootstrap Packages - PreRelease' - publishLocation: Container - condition: succeeded() - - - task: PublishBuildArtifacts@1 - displayName: Publish VSIX Packages - inputs: - PathtoPublish: '$(Build.SourcesDirectory)\artifacts\VSSetup\Release\Installer' - ArtifactName: 'Bootstrap VSIX - PreRelease' - publishLocation: Container - condition: succeeded() + - template: eng/pipelines/publish-logs.yml + parameters: + jobName: Correctness_Determinism + configuration: Debug + +- job: Correctness_Bootstrap_Build + dependsOn: Determine_Changes + condition: eq(dependencies.Determine_Changes.outputs['SetPathVars_compilers.containsChange'], 'true') + pool: + name: NetCore1ESPool-Public + demands: ImageOverride -equals Build.Windows.Amd64.VS2022.Pre.Open + timeoutInMinutes: 90 + steps: + - template: eng/pipelines/checkout-windows-task.yml + + - script: eng/test-build-correctness.cmd -configuration Release -enableDumps + displayName: Build - Validate correctness + + - template: eng/pipelines/publish-logs.yml + parameters: + jobName: Correctness_Bootstrap_Build + configuration: Release + + - task: PublishBuildArtifacts@1 + displayName: Publish Artifact Packages + inputs: + PathtoPublish: '$(Build.SourcesDirectory)\artifacts\packages\Release\PreRelease' + ArtifactName: 'Bootstrap Packages - PreRelease' + publishLocation: Container + + - task: PublishBuildArtifacts@1 + displayName: Publish VSIX Packages + inputs: + PathtoPublish: '$(Build.SourcesDirectory)\artifacts\VSSetup\Release\Installer' + ArtifactName: 'Bootstrap VSIX - PreRelease' + publishLocation: Container - job: Correctness_TodoCheck pool: @@ -289,34 +299,34 @@ jobs: - pwsh: eng/todo-check.ps1 displayName: Validate TODO/PROTOTYPE comments are not present -- ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: - - job: Correctness_Rebuild - pool: - name: NetCore1ESPool-Public - demands: ImageOverride -equals Build.Windows.Amd64.VS2022.Pre.Open - timeoutInMinutes: 90 - steps: - - template: eng/pipelines/checkout-windows-task.yml - - - task: PowerShell@2 - displayName: Restore - inputs: - filePath: eng/build.ps1 - arguments: -configuration Debug -prepareMachine -ci -restore -binaryLog - - - powershell: .\eng\test-rebuild.ps1 -ci -configuration Release - displayName: Run BuildValidator - - - task: PublishBuildArtifacts@1 - displayName: Publish BuildValidator debug outputs - inputs: - PathtoPublish: '$(Build.SourcesDirectory)/artifacts/BuildValidator' - ArtifactName: 'BuildValidator_DebugOut' - publishLocation: Container - continueOnError: true - condition: failed() +- job: Correctness_Rebuild + dependsOn: Determine_Changes + condition: eq(dependencies.Determine_Changes.outputs['SetPathVars_compilers.containsChange'], 'true') + pool: + name: NetCore1ESPool-Public + demands: ImageOverride -equals Build.Windows.Amd64.VS2022.Pre.Open + timeoutInMinutes: 90 + steps: + - template: eng/pipelines/checkout-windows-task.yml + + - task: PowerShell@2 + displayName: Restore + inputs: + filePath: eng/build.ps1 + arguments: -configuration Debug -prepareMachine -ci -restore -binaryLog + + - powershell: .\eng\test-rebuild.ps1 -ci -configuration Release + displayName: Run BuildValidator + + - task: PublishBuildArtifacts@1 + displayName: Publish BuildValidator debug outputs + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/artifacts/BuildValidator' + ArtifactName: 'BuildValidator_DebugOut' + publishLocation: Container + continueOnError: true - template: eng/pipelines/publish-logs.yml parameters: jobName: Correctness_Rebuild - configuration: Release + configuration: Release \ No newline at end of file diff --git a/eng/evaluate-changed-paths.sh b/eng/evaluate-changed-paths.sh new file mode 100755 index 0000000000000..5ad7583301035 --- /dev/null +++ b/eng/evaluate-changed-paths.sh @@ -0,0 +1,168 @@ +#!/usr/bin/env bash + +# Disable globbing in this bash script since we iterate over path patterns +set -f + +# Stop script if unbound variable found (use ${var:-} if intentional) +set -u + +# Stop script if command returns non-zero exit code. +# Prevents hidden errors caused by missing error code propagation. +set -e + +usage() +{ + echo "Script that evaluates changed paths and emits an azure devops variable if the changes contained in the current HEAD against the difftarget meet the includepahts/excludepaths filters:" + echo " --difftarget SHA or branch to diff against. (i.e: HEAD^1, origin/main, 0f4hd36, etc.)" + echo " --excludepaths Escaped list of paths to exclude from diff separated by '+'. (i.e: 'src/libraries/*+'src/installer/*')" + echo " --includepaths Escaped list of paths to include on diff separated by '+'. (i.e: 'src/libraries/System.Private.CoreLib/*')" + echo " --subset Subset name for which we're evaluating in order to include it in logs" + echo " --azurevariable Name of azure devops variable to create if change meets filter criteria" + echo "" + + echo "Arguments can also be passed in with a single hyphen." +} + +source="${BASH_SOURCE[0]}" + +# resolve $source until the file is no longer a symlink +while [[ -h "$source" ]]; do + scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + source="$(readlink "$source")" + # if $source was a relative symlink, we need to resolve it relative to the path where the + # symlink file was located + [[ $source != /* ]] && source="$scriptroot/$source" +done + +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" +eng_root=`cd -P "$scriptroot" && pwd` + +exclude_paths=() +include_paths=() +subset_name='' +azure_variable='' +diff_target='' + +while [[ $# > 0 ]]; do + opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")" + case "$opt" in + -help|-h) + usage + exit 0 + ;; + -difftarget) + diff_target=$2 + shift + ;; + -excludepaths) + IFS='+' read -r -a tmp <<< $2 + exclude_paths+=(${tmp[@]}) + shift + ;; + -includepaths) + IFS='+' read -r -a tmp <<< $2 + include_paths+=(${tmp[@]}) + shift + ;; + -subset) + subset_name=$2 + shift + ;; + -azurevariable) + azure_variable=$2 + shift + ;; + esac + + shift +done + +ci=true # Needed in order to use pipeline-logging-functions.sh +. "$eng_root/common/pipeline-logging-functions.sh" + +# -- expected args -- +# $@: git diff arguments +customGitDiff() { + ( + set -x + git diff -M -C -b --ignore-cr-at-eol --ignore-space-at-eol "$@" + ) +} + +# runs git diff with supplied filter. +# -- exit codes -- +# 0: No match was found +# 1: At least 1 match was found +# +# -- expected args -- +# $@: filter string +probePathsWithExitCode() { + local _filter=$@ + echo "" + customGitDiff --exit-code --quiet $diff_target -- $_filter +} + +# -- expected args -- +# $@: filter string +printMatchedPaths() { + local _subset=$subset_name + local _filter=$@ + echo "" + echo "----- Matching files for $_subset -----" + customGitDiff --name-only $diff_target -- $_filter +} + +probePaths() { + local _subset=$subset_name + local _azure_devops_var_name=$azure_variable + local exclude_path_string="" + local include_path_string="" + local found_applying_changes=false + + if [[ ${#exclude_paths[@]} -gt 0 ]]; then + echo "" + echo "******* Probing $_subset exclude paths *******"; + for _path in "${exclude_paths[@]}"; do + echo "$_path" + if [[ -z "$exclude_path_string" ]]; then + exclude_path_string=":!$_path" + else + exclude_path_string="$exclude_path_string :!$_path" + fi + done + + if ! probePathsWithExitCode $exclude_path_string; then + found_applying_changes=true + printMatchedPaths $exclude_path_string + fi + fi + + if [[ $found_applying_changes != true && ${#include_paths[@]} -gt 0 ]]; then + echo "" + echo "******* Probing $_subset include paths *******"; + for _path in "${include_paths[@]}"; do + echo "$_path" + if [[ -z "$include_path_string" ]]; then + include_path_string=":$_path" + else + include_path_string="$include_path_string :$_path" + fi + done + + if ! probePathsWithExitCode $include_path_string; then + found_applying_changes=true + printMatchedPaths $include_path_string + fi + fi + + if [[ $found_applying_changes == true ]]; then + echo "" + echo "Setting pipeline variable $_azure_devops_var_name=true" + Write-PipelineSetVariable -name $_azure_devops_var_name -value true -is_multi_job_variable true + else + echo "" + echo "No changed files for $_subset" + fi +} + +probePaths \ No newline at end of file diff --git a/eng/pipelines/evaluate-changed-files-group.yml b/eng/pipelines/evaluate-changed-files-group.yml new file mode 100644 index 0000000000000..820b0f7f4e0b8 --- /dev/null +++ b/eng/pipelines/evaluate-changed-files-group.yml @@ -0,0 +1,19 @@ +# This step template evaluates git changes using git based on a include/exclude path filter. +# For more information on how the path evaluation works look at evaluate-changed-paths.sh docs +# at the beginning of that file. + +parameters: + # Name for the subset that we're evaluating changes for. + # It is required to name the step correctly and so the variable created can be consumable. + subsetName: '' + # Array containing the arguments that are to be passed down to evaluate-changed-paths.sh + # Note that --azurevariable is always set to containschange, no need to pass it down. + arguments: [] + +steps: + - ${{ if ne(parameters.arguments[0], '') }}: + - script: eng/evaluate-changed-paths.sh + --azurevariable containsChange + ${{ join(' ', parameters.arguments) }} + displayName: Evaluate paths for ${{ parameters.subsetName }} + name: ${{ format('SetPathVars_{0}', parameters.subsetName) }} # need a name to access output variable \ No newline at end of file diff --git a/eng/pipelines/evaluate-changed-files.yml b/eng/pipelines/evaluate-changed-files.yml new file mode 100644 index 0000000000000..b48f0c65c7b2c --- /dev/null +++ b/eng/pipelines/evaluate-changed-files.yml @@ -0,0 +1,53 @@ +# Template to evaluate common paths in different pipelines. +parameters: +- name: jobName + type: string + default: '' +- name: queueName + type: string + default: '' +- name: vmImageName + type: string + default: '' +- name: paths + type: object + default: [] + +jobs: +- job: ${{ parameters.jobName }} + pool: + ${{ if ne(parameters.queueName, '') }}: + name: NetCore1ESPool-Public + demands: ImageOverride -equals ${{ parameters.queueName }} + + ${{ if ne(parameters.vmImageName, '') }}: + vmImage: ${{ parameters.vmImageName }} + timeoutInMinutes: 10 + + steps: + - checkout: none + + - script: | + set -x + git init + git config --local checkout.workers 0 + git config --local fetch.parallel 0 + git remote add origin "$(Build.Repository.Uri)" + git fetch --no-tags --no-auto-maintenance --depth=2 origin "$(Build.SourceVersion)" + git checkout "$(Build.SourceVersion)" + displayName: Shallow Checkout + + - ${{ if ne(parameters.paths[0], '') }}: + - ${{ each path in parameters.paths }}: + - template: evaluate-changed-files-group.yml + parameters: + subsetName: ${{ path.subset }} + arguments: + # The commit that we're building is always a merge commit that is merging into the target branch. + # So the first parent of the commit is on the target branch and the second parent is on the source branch. + - --difftarget HEAD^1 + - --subset ${{ path.subset }} + - ${{ if ne(path.include[0], '') }}: + - --includepaths '${{ join('+', path.include) }}' + - ${{ if ne(path.exclude[0], '') }}: + - --excludepaths '${{ join('+', path.exclude) }}' \ No newline at end of file