diff --git a/.github/workflows/openvmm-ci.json b/.github/workflows/openvmm-ci.json index 1339944d7..eb5b4cf64 100644 --- a/.github/workflows/openvmm-ci.json +++ b/.github/workflows/openvmm-ci.json @@ -3250,78 +3250,6 @@ "flowey_lib_hvlite::init_openvmm_cargo_config_deny_warnings": [ "{\"DenyWarnings\":true}" ] - }, - "20": { - "flowey_lib_common::cfg_cargo_common_flags": [ - "{\"SetLocked\":true}", - "{\"SetVerbose\":{\"backing_var\":{\"RuntimeVar\":\"param0\"},\"is_secret\":false}}" - ], - "flowey_lib_common::download_azcopy": [ - "{\"Version\":\"10.26.0-20240731\"}" - ], - "flowey_lib_common::download_cargo_fuzz": [ - "{\"Version\":\"0.12.0\"}" - ], - "flowey_lib_common::download_cargo_nextest": [ - "{\"Version\":\"0.9.74\"}" - ], - "flowey_lib_common::download_gh_cli": [ - "{\"Version\":\"2.52.0\"}" - ], - "flowey_lib_common::download_mdbook": [ - "{\"Version\":\"0.4.6\"}" - ], - "flowey_lib_common::download_protoc": [ - "{\"Version\":\"27.1\"}" - ], - "flowey_lib_common::git_checkout": [ - "{\"RegisterRepo\":{\"allow_persist_credentials\":false,\"depth\":1,\"pre_run_deps\":[],\"repo_id\":\"hvlite\",\"repo_src\":\"GithubSelf\"}}" - ], - "flowey_lib_common::install_azure_cli": [ - "{\"AutoInstall\":true}", - "{\"Version\":\"2.56.0\"}" - ], - "flowey_lib_common::install_nodejs": [ - "{\"Version\":\"18.x\"}" - ], - "flowey_lib_common::install_rust": [ - "{\"AutoInstall\":true}", - "{\"IgnoreVersion\":false}", - "{\"Version\":\"1.82.0\"}" - ], - "flowey_lib_common::use_gh_cli": [ - "{\"WithAuth\":{\"AuthToken\":{\"backing_var\":{\"RuntimeVar\":\"flowey_lib_hvlite::_jobs::cfg_common:0:flowey_lib_hvlite/src/_jobs/cfg_common.rs:92:36\"},\"is_secret\":true}}}" - ], - "flowey_lib_hvlite::_jobs::cfg_common": [ - "{\"deny_warnings\":true,\"local_only\":null,\"locked\":true,\"verbose\":{\"backing_var\":{\"RuntimeVar\":\"param0\"},\"is_secret\":false}}" - ], - "flowey_lib_hvlite::_jobs::cfg_hvlite_reposource": [ - "{\"hvlite_repo_source\":\"GithubSelf\"}" - ], - "flowey_lib_hvlite::_jobs::cfg_versions": [ - "{}" - ], - "flowey_lib_hvlite::download_lxutil": [ - "{\"Version\":\"10.0.26100.1-240331-1435.ge-release\"}" - ], - "flowey_lib_hvlite::download_openhcl_kernel_package": [ - "{\"Version\":[\"Cvm\",\"6.6.51.2\"]}", - "{\"Version\":[\"CvmDev\",\"6.6.51.6\"]}", - "{\"Version\":[\"Dev\",\"6.6.51.6\"]}", - "{\"Version\":[\"Main\",\"6.6.51.2\"]}" - ], - "flowey_lib_hvlite::download_openvmm_deps": [ - "{\"Version\":\"0.1.0-20241014.2\"}" - ], - "flowey_lib_hvlite::download_uefi_mu_msvm": [ - "{\"Version\":\"24.0.2\"}" - ], - "flowey_lib_hvlite::git_checkout_openvmm_repo": [ - "{\"SetRepoId\":{\"backing_var\":{\"Inline\":\"hvlite\"},\"is_secret\":false}}" - ], - "flowey_lib_hvlite::init_openvmm_cargo_config_deny_warnings": [ - "{\"DenyWarnings\":true}" - ] } } } \ No newline at end of file diff --git a/.github/workflows/openvmm-ci.yaml b/.github/workflows/openvmm-ci.yaml index 68b6b091a..d5008dad4 100644 --- a/.github/workflows/openvmm-ci.yaml +++ b/.github/workflows/openvmm-ci.yaml @@ -4573,57 +4573,6 @@ jobs: with: name: x64-linux-rustdoc path: ${{ runner.temp }}/publish_artifacts/x64-linux-rustdoc/ - job20: - name: openvmm checkin gates - runs-on: windows-latest - permissions: - contents: read - id-token: write - needs: - - job19 - - job18 - - job17 - - job16 - - job15 - - job14 - - job13 - - job12 - - job11 - - job10 - - job9 - - job8 - - job7 - - job6 - - job5 - - job4 - - job3 - - job2 - - job1 - - job0 - steps: - - name: ๐ŸŒผ๐Ÿฅพ Download bootstrapped flowey - uses: actions/download-artifact@v4 - with: - name: _internal-flowey-bootstrap-x86_64-windows-uid-9 - path: ${{ runner.temp }}/bootstrapped-flowey - - run: echo "${{ runner.temp }}/bootstrapped-flowey" >> $GITHUB_PATH - shell: bash - name: ๐ŸŒผ๐Ÿ“ฆ Add flowey to PATH - - name: ๐ŸŒผ๐Ÿ›ซ Initialize job - run: | - AgentTempDirNormal="${{ runner.temp }}" - AgentTempDirNormal=$(echo "$AgentTempDirNormal" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') - echo "AgentTempDirNormal=$AgentTempDirNormal" >> $GITHUB_ENV - - chmod +x $AgentTempDirNormal/bootstrapped-flowey/flowey.exe - - echo '"debug"' | flowey.exe v 20 'FLOWEY_LOG' --update-from-stdin - echo "${{ runner.temp }}/work" | flowey.exe v 20 '_internal_WORKING_DIR' --update-from-stdin --is-raw-string - - cat <<'EOF' | flowey.exe v 20 'param0' --update-from-stdin - ${{ inputs.param0 != '' && inputs.param0 || 'false' }} - EOF - shell: bash job3: name: xtask fmt (windows) runs-on: windows-latest diff --git a/.github/workflows/openvmm-pr.json b/.github/workflows/openvmm-pr.json index 75c2ff40b..5dd206d01 100644 --- a/.github/workflows/openvmm-pr.json +++ b/.github/workflows/openvmm-pr.json @@ -2845,6 +2845,9 @@ "flowey_lib_common::use_gh_cli": [ "{\"WithAuth\":{\"AuthToken\":{\"backing_var\":{\"RuntimeVar\":\"flowey_lib_hvlite::_jobs::cfg_common:0:flowey_lib_hvlite/src/_jobs/cfg_common.rs:92:36\"},\"is_secret\":true}}}" ], + "flowey_lib_hvlite::_jobs::all_good_job": [ + "{\"did_fail_env_var\":\"ANY_JOBS_FAILED\",\"done\":{\"backing_var\":\"start48\",\"is_secret\":false}}" + ], "flowey_lib_hvlite::_jobs::cfg_common": [ "{\"deny_warnings\":true,\"local_only\":null,\"locked\":true,\"verbose\":{\"backing_var\":{\"RuntimeVar\":\"param0\"},\"is_secret\":false}}" ], diff --git a/.github/workflows/openvmm-pr.yaml b/.github/workflows/openvmm-pr.yaml index 85a523957..becd822cf 100644 --- a/.github/workflows/openvmm-pr.yaml +++ b/.github/workflows/openvmm-pr.yaml @@ -2307,14 +2307,6 @@ jobs: - name: 'validate cache entry: gh-release-download' run: flowey.exe e 13 flowey_lib_common::cache 11 shell: bash - - name: ๐ŸŒผ๐Ÿงผ Redact bootstrap var db - run: rm $AgentTempDirNormal/bootstrapped-flowey/job13.json - shell: bash - - name: ๐ŸŒผ๐Ÿฅพ Publish bootstrapped flowey - uses: actions/upload-artifact@v4 - with: - name: _internal-flowey-bootstrap-x86_64-windows-uid-4 - path: ${{ runner.temp }}/bootstrapped-flowey job14: name: clippy [linux, macos], unit tests [x64-linux] runs-on: @@ -3223,9 +3215,17 @@ jobs: - name: test cargo xflowey build-igvm x64 --install-missing-deps run: flowey e 16 flowey_lib_hvlite::_jobs::test_local_flowey_build_igvm 1 shell: bash + - name: ๐ŸŒผ๐Ÿงผ Redact bootstrap var db + run: rm $AgentTempDirNormal/bootstrapped-flowey/job16.json + shell: bash + - name: ๐ŸŒผ๐Ÿฅพ Publish bootstrapped flowey + uses: actions/upload-artifact@v4 + with: + name: _internal-flowey-bootstrap-x86_64-linux-uid-1 + path: ${{ runner.temp }}/bootstrapped-flowey job17: name: openvmm checkin gates - runs-on: windows-latest + runs-on: ubuntu-latest permissions: contents: read id-token: write @@ -3247,11 +3247,14 @@ jobs: - job2 - job1 - job0 + if: ${{ always() }} + env: + ANY_JOBS_FAILED: ${{ contains(needs.*.result, 'cancelled') || contains(needs.*.result, 'failure') }} steps: - name: ๐ŸŒผ๐Ÿฅพ Download bootstrapped flowey uses: actions/download-artifact@v4 with: - name: _internal-flowey-bootstrap-x86_64-windows-uid-4 + name: _internal-flowey-bootstrap-x86_64-linux-uid-1 path: ${{ runner.temp }}/bootstrapped-flowey - run: echo "${{ runner.temp }}/bootstrapped-flowey" >> $GITHUB_PATH shell: bash @@ -3262,15 +3265,18 @@ jobs: AgentTempDirNormal=$(echo "$AgentTempDirNormal" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') echo "AgentTempDirNormal=$AgentTempDirNormal" >> $GITHUB_ENV - chmod +x $AgentTempDirNormal/bootstrapped-flowey/flowey.exe + chmod +x $AgentTempDirNormal/bootstrapped-flowey/flowey - echo '"debug"' | flowey.exe v 17 'FLOWEY_LOG' --update-from-stdin - echo "${{ runner.temp }}/work" | flowey.exe v 17 '_internal_WORKING_DIR' --update-from-stdin --is-raw-string + echo '"debug"' | flowey v 17 'FLOWEY_LOG' --update-from-stdin + echo "${{ runner.temp }}/work" | flowey v 17 '_internal_WORKING_DIR' --update-from-stdin --is-raw-string - cat <<'EOF' | flowey.exe v 17 'param0' --update-from-stdin + cat <<'EOF' | flowey v 17 'param0' --update-from-stdin ${{ inputs.param0 != '' && inputs.param0 || 'false' }} EOF shell: bash + - name: Check if any jobs failed + run: flowey e 17 flowey_lib_hvlite::_jobs::all_good_job 0 + shell: bash job2: name: build and check docs [x64-linux] runs-on: diff --git a/flowey/flowey_cli/src/pipeline_resolver/ado_yaml.rs b/flowey/flowey_cli/src/pipeline_resolver/ado_yaml.rs index 8a9a7b9d8..d3134d4a6 100644 --- a/flowey/flowey_cli/src/pipeline_resolver/ado_yaml.rs +++ b/flowey/flowey_cli/src/pipeline_resolver/ado_yaml.rs @@ -87,6 +87,8 @@ pub fn ado_yaml( arch, cond_param_idx, ref ado_pool, + gh_override_if: _, + gh_global_env: _, gh_pool: _, gh_permissions: _, ref external_read_vars, diff --git a/flowey/flowey_cli/src/pipeline_resolver/direct_run.rs b/flowey/flowey_cli/src/pipeline_resolver/direct_run.rs index 5255f5151..dcfd822d1 100644 --- a/flowey/flowey_cli/src/pipeline_resolver/direct_run.rs +++ b/flowey/flowey_cli/src/pipeline_resolver/direct_run.rs @@ -93,6 +93,8 @@ fn direct_run_do_work( cond_param_idx, ado_pool: _, ado_variables: _, + gh_override_if: _, + gh_global_env: _, gh_pool: _, gh_permissions: _, ref external_read_vars, diff --git a/flowey/flowey_cli/src/pipeline_resolver/generic.rs b/flowey/flowey_cli/src/pipeline_resolver/generic.rs index 8ecdfe02b..0ee7bc166 100644 --- a/flowey/flowey_cli/src/pipeline_resolver/generic.rs +++ b/flowey/flowey_cli/src/pipeline_resolver/generic.rs @@ -66,6 +66,8 @@ pub struct ResolvedPipelineJob { pub arch: FlowArch, pub ado_pool: Option, pub ado_variables: BTreeMap, + pub gh_override_if: Option, + pub gh_global_env: BTreeMap, pub gh_pool: Option, pub gh_permissions: BTreeMap>, pub external_read_vars: BTreeSet, @@ -166,6 +168,8 @@ pub fn resolve_pipeline(pipeline: Pipeline) -> anyhow::Result cond_param_idx, ado_pool, ado_variables, + gh_override_if, + gh_global_env, gh_pool, gh_permissions, }, @@ -217,6 +221,8 @@ pub fn resolve_pipeline(pipeline: Pipeline) -> anyhow::Result label, ado_pool, ado_variables, + gh_override_if, + gh_global_env, gh_pool, gh_permissions, platform, diff --git a/flowey/flowey_cli/src/pipeline_resolver/github_yaml/github_yaml_defs.rs b/flowey/flowey_cli/src/pipeline_resolver/github_yaml/github_yaml_defs.rs index bf42a2741..38e1fbac1 100644 --- a/flowey/flowey_cli/src/pipeline_resolver/github_yaml/github_yaml_defs.rs +++ b/flowey/flowey_cli/src/pipeline_resolver/github_yaml/github_yaml_defs.rs @@ -268,6 +268,10 @@ pub struct Job { pub permissions: BTreeMap, #[serde(skip_serializing_if = "Vec::is_empty")] pub needs: Vec, + #[serde(rename = "if", skip_serializing_if = "Option::is_none")] + pub r#if: Option, + #[serde(skip_serializing_if = "BTreeMap::is_empty")] + pub env: BTreeMap, pub steps: Vec, } diff --git a/flowey/flowey_cli/src/pipeline_resolver/github_yaml/mod.rs b/flowey/flowey_cli/src/pipeline_resolver/github_yaml/mod.rs index 19a36e1c4..65e981c9b 100644 --- a/flowey/flowey_cli/src/pipeline_resolver/github_yaml/mod.rs +++ b/flowey/flowey_cli/src/pipeline_resolver/github_yaml/mod.rs @@ -86,6 +86,8 @@ pub fn github_yaml( arch, ref external_read_vars, ado_pool: _, + ref gh_override_if, + ref gh_global_env, ref gh_pool, ref gh_permissions, cond_param_idx: _, @@ -565,6 +567,8 @@ EOF }) .collect() }, + r#if: gh_override_if.clone(), + env: gh_global_env.clone(), steps: gh_steps, }, ); diff --git a/flowey/flowey_cli/src/pipeline_resolver/viz.rs b/flowey/flowey_cli/src/pipeline_resolver/viz.rs index 90f3bbd52..d07b92c72 100644 --- a/flowey/flowey_cli/src/pipeline_resolver/viz.rs +++ b/flowey/flowey_cli/src/pipeline_resolver/viz.rs @@ -75,6 +75,8 @@ fn viz_pipeline_generic( cond_param_idx: _, ref ado_pool, ado_variables: _, + gh_override_if: _, + gh_global_env: _, ref gh_pool, gh_permissions: _, ref external_read_vars, @@ -253,6 +255,8 @@ pub fn viz_pipeline_dot(pipeline: ResolvedPipeline, _backend: FlowBackend) -> an cond_param_idx: _, ado_pool, ado_variables: _, + gh_override_if: _, + gh_global_env: _, gh_pool, gh_permissions: _, external_read_vars: _, diff --git a/flowey/flowey_core/src/pipeline.rs b/flowey/flowey_core/src/pipeline.rs index 62e1ca426..fb7ba0ce1 100644 --- a/flowey/flowey_core/src/pipeline.rs +++ b/flowey/flowey_core/src/pipeline.rs @@ -531,6 +531,8 @@ impl Pipeline { cond_param_idx: None, ado_pool: None, ado_variables: BTreeMap::new(), + gh_override_if: None, + gh_global_env: BTreeMap::new(), gh_pool: None, gh_permissions: BTreeMap::new(), }); @@ -872,17 +874,18 @@ impl PipelineJob<'_> { /// (ADO only) Declare a job-level, named, read-only ADO variable. /// - /// `name` and `value` are both arbitrary strings. + /// `name` and `value` are both arbitrary strings, which may include ADO + /// template expressions. /// /// NOTE: Unless required by some particular third-party task, it's strongly /// recommended to _avoid_ using this method, and to simply use /// [`ReadVar::from_static`] to get a obtain a static variable. /// - /// DEVNOTE: In the future, this API may be updated to return a handle that will - /// allow resolving the resulting `AdoRuntimeVar`, but for implementation - /// expediency, this API does not currently do this. If you need to read the - /// value of this variable at runtime, you may need to invoke - /// [`AdoRuntimeVar::dangerous_from_global`] manually. + /// DEVNOTE: In the future, this API may be updated to return a handle that + /// will allow resolving the resulting `AdoRuntimeVar`, but for + /// implementation expediency, this API does not currently do this. If you + /// need to read the value of this variable at runtime, you may need to + /// invoke [`AdoRuntimeVar::dangerous_from_global`] manually. /// /// [`NodeCtx::get_ado_variable`]: crate::node::NodeCtx::get_ado_variable pub fn ado_new_named_variable(self, name: impl AsRef, value: impl AsRef) -> Self { @@ -929,6 +932,53 @@ impl PipelineJob<'_> { self } + /// (GitHub Actions only) Manually override the `if:` condition for this + /// particular job. + /// + /// **This is dangerous**, as an improperly set `if` condition may break + /// downstream flowey jobs which assume flowey is in control of the job's + /// scheduling logic. + /// + /// See + /// + /// for more info. + pub fn gh_dangerous_override_if(self, condition: impl AsRef) -> Self { + self.pipeline.jobs[self.job_idx].gh_override_if = Some(condition.as_ref().into()); + self + } + + /// (GitHub Actions only) Declare a global job-level environment variable, + /// visible to all downstream steps. + /// + /// `name` and `value` are both arbitrary strings, which may include GitHub + /// Actions template expressions. + /// + /// **This is dangerous**, as it is easy to misuse this API in order to + /// write a node which takes an implicit dependency on there being a global + /// variable set on its behalf by the top-level pipeline code, making it + /// difficult to "locally reason" about the behavior of a node simply by + /// reading its code. + /// + /// Whenever possible, nodes should "late bind" environment variables: + /// accepting a compile-time / runtime flowey parameter, and then setting it + /// prior to executing a child command that requires it. + /// + /// Only use this API in exceptional cases, such as obtaining an environment + /// variable whose value is determined by a job-level GitHub Actions + /// expression evaluation. + pub fn gh_dangerous_global_env_var( + self, + name: impl AsRef, + value: impl AsRef, + ) -> Self { + let name = name.as_ref(); + let value = value.as_ref(); + self.pipeline.jobs[self.job_idx] + .gh_global_env + .insert(name.into(), value.into()); + self + } + /// (GitHub Actions only) Grant permissions required by nodes in the job. /// /// For a given node handle, grant the specified permissions. @@ -1070,7 +1120,9 @@ pub mod internal { // backend specific pub ado_pool: Option, pub ado_variables: BTreeMap, + pub gh_override_if: Option, pub gh_pool: Option, + pub gh_global_env: BTreeMap, pub gh_permissions: BTreeMap>, } diff --git a/flowey/flowey_hvlite/src/pipelines/checkin_gates.rs b/flowey/flowey_hvlite/src/pipelines/checkin_gates.rs index 8a30532e1..48614d255 100644 --- a/flowey/flowey_hvlite/src/pipelines/checkin_gates.rs +++ b/flowey/flowey_hvlite/src/pipelines/checkin_gates.rs @@ -953,21 +953,37 @@ impl IntoPipeline for CheckinGatesCli { all_jobs.push(job); } - // Add a job that depends on all others as a workaround for https://github.com/orgs/community/discussions/12395. - // TODO: Add a way for this job to skip flowey setup and become a true no-op. - let all_good_job = pipeline - .new_job( - FlowPlatform::Windows, - FlowArch::X86_64, - "openvmm checkin gates", - ) - .gh_set_pool(crate::pipelines_shared::gh_pools::default_gh_hosted( - FlowPlatform::Windows, - )) - .finish(); + if matches!(config, PipelineConfig::Pr) { + // Add a job that depends on all others as a workaround for + // https://github.com/orgs/community/discussions/12395. + // + // This workaround then itself requires _another_ workaround, requiring + // the use of `gh_dangerous_override_if`, and some additional custom job + // logic, to deal with https://github.com/actions/runner/issues/2566. + // + // TODO: Add a way for this job to skip flowey setup and become a true + // no-op. + let all_good_job = pipeline + .new_job( + FlowPlatform::Linux, + FlowArch::X86_64, + "openvmm checkin gates", + ) + .gh_set_pool(crate::pipelines_shared::gh_pools::default_gh_hosted( + FlowPlatform::Linux, + )) + // always run this job, regardless whether or not any previous jobs failed + .gh_dangerous_override_if("${{ always() }}") + .gh_dangerous_global_env_var("ANY_JOBS_FAILED", "${{ contains(needs.*.result, 'cancelled') || contains(needs.*.result, 'failure') }}") + .dep_on(|ctx| flowey_lib_hvlite::_jobs::all_good_job::Params { + did_fail_env_var: "ANY_JOBS_FAILED".into(), + done: ctx.new_done_handle(), + }) + .finish(); - for job in all_jobs.iter() { - pipeline.non_artifact_dep(&all_good_job, job); + for job in all_jobs.iter() { + pipeline.non_artifact_dep(&all_good_job, job); + } } Ok(pipeline) diff --git a/flowey/flowey_lib_hvlite/src/_jobs/all_good_job.rs b/flowey/flowey_lib_hvlite/src/_jobs/all_good_job.rs new file mode 100644 index 000000000..ef13de31e --- /dev/null +++ b/flowey/flowey_lib_hvlite/src/_jobs/all_good_job.rs @@ -0,0 +1,52 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! (GitHub Actions only) Check that all dependent jobs prior to this job +//! completed successfully, and in turn, succeeding / failing itself. +//! +//! Workaround for . +//! +//! Workaround itself required _another_ workaround, in order to deal with +//! . + +use flowey::node::prelude::*; + +flowey_request! { + pub struct Params { + pub did_fail_env_var: String, + pub done: WriteVar, + } +} + +new_simple_flow_node!(struct Node); + +impl SimpleFlowNode for Node { + type Request = Params; + + fn imports(ctx: &mut ImportCtx<'_>) { + ctx.import::(); + ctx.import::(); + } + + fn process_request(request: Self::Request, ctx: &mut NodeCtx<'_>) -> anyhow::Result<()> { + let Params { + did_fail_env_var, + done, + } = request; + + ctx.emit_rust_step("Check if any jobs failed", |ctx| { + done.claim(ctx); + |_rt| { + let did_fail = std::env::var(did_fail_env_var)? + .to_lowercase() + .parse::()?; + if did_fail { + anyhow::bail!("Detected failures in one or more previous jobs!") + } + Ok(()) + } + }); + + Ok(()) + } +} diff --git a/flowey/flowey_lib_hvlite/src/_jobs/mod.rs b/flowey/flowey_lib_hvlite/src/_jobs/mod.rs index da59c2439..b0e088c16 100644 --- a/flowey/flowey_lib_hvlite/src/_jobs/mod.rs +++ b/flowey/flowey_lib_hvlite/src/_jobs/mod.rs @@ -4,6 +4,7 @@ //! Defines top-level "job nodes" which can be composed when defining a flowey //! pipeline using [`flowey::pipeline::prelude::PipelineJob::dep_on`]. +pub mod all_good_job; pub mod build_and_publish_guest_test_uefi; pub mod build_and_publish_guide; pub mod build_and_publish_igvmfilegen;