Build Cache

README.md

@@ -28,6 +28,7 @@ Once a DVF is published, any user can choose to trust the signer of that DVF and
    - [Update DVF](#update-dvf)
    - [Check Bytecode](#check-bytecode)
    - [Handle errors](#handle-errors)
+   - [Use Build Cache](#use-build-cache)
 
 6. [Advanced Usage](#advanced-usage)
 
@@ -190,6 +191,12 @@ If the Hardhat project does not store its compilation artifacts in the default d
 dv init --project <PROJECT_PATH> --address <ADDRESS> --contractname <NAME> --env hardhat --artifacts <ARTIFACTS> new.dvf.json
 ```
 
+If you have to use an external build-info (i.e., you don't want `dv` to build the project), you can specify the path to the build-info directory with `--buildcache`:
+
+```
+dv init --project <PROJECT_PATH> --address <ADDRESS> --contractname <NAME> --buildcache <PATH_TO_BUILD_INFO> new.dvf.json
+```
+
 In many cases, deployments are not completed in one block as parameters may be set in subsequent transactions. To receive the storage at a later block (and emitted events up to that block), you can pass the desired block number with `--initblock`:
 
 ```
@@ -339,7 +346,7 @@ dv bytecode-check --project <PROJECT_PATH> --address <ADDRESS> --contractname <N
 
 Please note that `<B>` must be equal to or larger than the deployment block of the contract.
 
-### Handle errors
+### Handle Errors
 
 If something goes wrong during a run of `dv`, you can add the `--verbose` option to get additional information:
 
@@ -357,6 +364,33 @@ Please refer to section [Common Problems](#common-problems) for help with unders
 
 If your problem cannot be solved by yourself or if you have found a bug (e.g., `dv` crashed), please refer to section [Getting Help](#getting-help).
 
+### Use Build Cache
+
+By default `dv` compiles the full project every time you run the `init` or `bytecode-check` commands.
+
+You can pass an external build-info path (containing the compiler output) to `dv` using the `--buildcache` flag. This can be used to:
+
+1. Circumvent the internal project compilation in case of issues.
+2. Skip subsequent compilations if you want to create DVFs for multiple contracts in a large project.
+
+For the second case, you can use the `generate-build-cache` command to generate a persisted build-info path that can be passed to `--buildcache`:
+
+```
+dv generate-build-cache --project <PROJECT_PATH>
+```
+
+You can also use the command for hardhat projects using `--env`:
+
+```
+dv generate-build-cache --project <PROJECT_PATH> --env hardhat
+```
+
+If the Hardhat project does not store its compilation artifacts in the default directory, you can pass the correct directory with `--artifacts`:
+
+```
+dv generate-build-cache --project <PROJECT_PATH> --env hardhat --artifacts <ARTIFACTS>
+```
+
 ## Advanced Usage
 
 Not all projects can be easily validated by validating single contracts. If the smart contracts in the project you are validating have dependencies to other contracts that are security relevant, please refer to this section.

lib/bytecode_verification/parse_json.rs

@@ -1,4 +1,5 @@
 use std::collections::HashMap;
+use std::fs;
 use std::path::PathBuf;
 
 use clap::ValueEnum;
@@ -1264,20 +1265,20 @@ impl ProjectInfo {
         false
     }
 
-    pub fn new(
-        contract_name: &String,
+    pub fn compile(
         project: &Path,
         env: Environment,
         artifacts_path: &Path,
-    ) -> Result<Self, ValidationError> {
+    ) -> Result<PathBuf, ValidationError> {
         let build_info_path: PathBuf;
         let build_info_dir: TempDir;
 
         match env {
             Environment::Foundry => {
                 assert!(Self::check_forge());
                 build_info_dir = Builder::new().prefix("dvf_bi").tempdir().unwrap();
-                build_info_path = build_info_dir.path().to_path_buf();
+                // Persist for now
+                build_info_path = build_info_dir.into_path();
                 Self::forge_build(project, &build_info_path)?;
             }
             Environment::Hardhat => {
@@ -1287,6 +1288,20 @@ impl ProjectInfo {
                 Self::hardhat_compile(project)?;
             }
         }
+        Ok(build_info_path)
+    }
+
+    pub fn new(
+        contract_name: &String,
+        project: &Path,
+        env: Environment,
+        artifacts_path: &Path,
+        build_cache: Option<&str>,
+    ) -> Result<Self, ValidationError> {
+        let build_info_path: PathBuf = match build_cache {
+            Some(s) => PathBuf::from(s),
+            None => Self::compile(project, env, artifacts_path)?,
+        };
 
         let command = match env {
             Environment::Foundry => "<forge clean>",
@@ -1456,6 +1471,11 @@ impl ProjectInfo {
         }
         let immutables = Self::extract_immutables(&deployed_bytecode, &id_to_ast);
 
+        // If we are not using build_cache then delete the tmp files
+        if build_cache.is_none() && env == Environment::Foundry {
+            fs::remove_dir_all(&build_info_path)?;
+        };
+
         let pi = ProjectInfo {
             compiled_bytecode: compiled_bytecode_str,
             init_code: init_code_str,

src/dvf.rs

@@ -448,6 +448,12 @@ fn main() {
                         .action(ArgAction::Set)
                         .default_value("artifacts")
                 )
+                .arg(
+                    Arg::with_name("buildcache")
+                        .long("buildcache")
+                        .help("Folder containing buildcache previously. Use with care")
+                        .action(ArgAction::Set)
+                )
                 .arg(
                     Arg::with_name("OUTPUT")
                         .help("Path of the generated DVF file")
@@ -533,6 +539,31 @@ fn main() {
         .subcommand(SubCommand::with_name("generate-config")
                 .about("interactively generate configuration file")
         )
+        .subcommand(SubCommand::with_name("generate-build-cache").about("generate the build cache")
+                .arg(
+                    Arg::with_name("project")
+                        .long("project")
+                        .help("Path to the root folder of source code project")
+                        .required(true)
+                        .validator(is_valid_path)
+                        .action(ArgAction::Set),
+                )
+                .arg(
+                    Arg::with_name("env")
+                        .long("env")
+                        .help("Project's development environment")
+                        .value_parser(clap::value_parser!(Environment))
+                        .default_value(Environment::Foundry.to_string().as_str())
+                        .action(ArgAction::Set)
+                )
+                .arg(
+                    Arg::with_name("artifacts")
+                        .long("artifacts")
+                        .help("Folder containing the artifacts (Hardhat only)")
+                        .default_value("artifacts")
+                        .action(ArgAction::Set)
+                )
+        )
         .subcommand(SubCommand::with_name("bytecode-check").about("perform just the bytecode check")
                 .arg(
                     Arg::with_name("initblock")
@@ -593,6 +624,12 @@ fn main() {
                         .default_value("artifacts")
                         .action(ArgAction::Set)
                 )
+                .arg(
+                    Arg::with_name("buildcache")
+                        .long("buildcache")
+                        .help("Folder containing buildcache previously. Use with care")
+                        .action(ArgAction::Set)
+                )
         )
         .get_matches();
 
@@ -641,6 +678,7 @@ enum ProgressMode {
     Update,
     Validation,
     BytecodeCheck,
+    GenerateBuildCache,
 }
 
 fn updated_filename(original_path: &Path) -> PathBuf {
@@ -669,6 +707,7 @@ fn print_progress(s: &str, i: &mut u64, pm: &ProgressMode) {
         ProgressMode::Update => 4,
         ProgressMode::Validation => 5,
         ProgressMode::BytecodeCheck => 3,
+        ProgressMode::GenerateBuildCache => 1,
     };
     println!("{} {}", style(format!("[{i:2}/{total:2}]")).bold().dim(), s);
     *i += 1;
@@ -696,6 +735,7 @@ fn process(matches: ArgMatches) -> Result<(), ValidationError> {
             let env = *sub_m.get_one::<Environment>("env").unwrap();
             let project = sub_m.value_of("project").unwrap();
             let artifacts = sub_m.value_of("artifacts").unwrap();
+            let build_cache = sub_m.value_of("buildcache");
             let (path, artifacts_path) = get_project_paths(project, artifacts);
 
             let mut imp_env = *sub_m.get_one::<Environment>("implementationenv").unwrap();
@@ -761,8 +801,13 @@ fn process(matches: ArgMatches) -> Result<(), ValidationError> {
 
             debug!("Fetching forge output");
             print_progress("Compiling local code.", &mut pc, &progress_mode);
-            let mut project_info =
-                ProjectInfo::new(&dumped.contract_name, &path, env, &artifacts_path)?;
+            let mut project_info = ProjectInfo::new(
+                &dumped.contract_name,
+                &path,
+                env,
+                &artifacts_path,
+                build_cache,
+            )?;
 
             print_progress("Comparing bytecode.", &mut pc, &progress_mode);
             let factory_mode = sub_m.get_flag("factory");
@@ -844,6 +889,7 @@ fn process(matches: ArgMatches) -> Result<(), ValidationError> {
                     &imp_path,
                     imp_env,
                     &imp_artifacts_path,
+                    None,
                 )?;
 
                 print_progress(
@@ -1401,6 +1447,25 @@ fn process(matches: ArgMatches) -> Result<(), ValidationError> {
 
             Ok(())
         }
+        Some(("generate-build-cache", sub_m)) => {
+            println!("Generating Build Cache.");
+
+            let env = *sub_m.get_one::<Environment>("env").unwrap();
+            let project = sub_m.value_of("project").unwrap();
+            let artifacts = sub_m.value_of("artifacts").unwrap();
+            let (path, artifacts_path) = get_project_paths(project, artifacts);
+
+            let mut pc = 1_u64;
+            let progress_mode: ProgressMode = ProgressMode::GenerateBuildCache;
+
+            // Bytecode and Immutable check
+            print_progress("Compiling local bytecode.", &mut pc, &progress_mode);
+
+            let build_cache_path = ProjectInfo::compile(&path, env, &artifacts_path)?;
+
+            println!("Build Cache: {}", build_cache_path.display());
+            exit(0);
+        }
         Some(("bytecode-check", sub_m)) => {
             println!("Starting bytecode check.");
 
@@ -1411,6 +1476,7 @@ fn process(matches: ArgMatches) -> Result<(), ValidationError> {
 
             let contract_name = sub_m.value_of("contractname").unwrap().to_string();
             let address = Address::from_str(sub_m.value_of("address").unwrap())?;
+            let build_cache = sub_m.value_of("buildcache");
             let chain_id = *sub_m.get_one("chainid").unwrap();
 
             config.set_chain_id(chain_id)?;
@@ -1433,7 +1499,8 @@ fn process(matches: ArgMatches) -> Result<(), ValidationError> {
             // Bytecode and Immutable check
             print_progress("Compiling local bytecode.", &mut pc, &progress_mode);
 
-            let mut project_info = ProjectInfo::new(&contract_name, &path, env, &artifacts_path)?;
+            let mut project_info =
+                ProjectInfo::new(&contract_name, &path, env, &artifacts_path, build_cache)?;
 
             print_progress("Comparing bytecode.", &mut pc, &progress_mode);
             let factory_mode = sub_m.get_flag("factory");

tests/ci_tests.sh

@@ -29,6 +29,8 @@ RUST_BACKTRACE=1 cargo test
 envsubst < tests/config.json > /tmp/eval_config.json
 cargo run --bin fetch-from-etherscan -- -c  /tmp/eval_config.json --address 0x5c69bee701ef814a2b6a3edd4b1652cb9cc5aa6f --project /tmp/uni-factory
 cargo run --bin dv --  --config  /tmp/eval_config.json init --address 0x5c69bee701ef814a2b6a3edd4b1652cb9cc5aa6f --project /tmp/uni-factory --chainid 1 --factory --contractname UniswapV2Factory UniswapV2Factory_0x5c69bee701ef814a2b6a3edd4b1652cb9cc5aa6f.dvf.json
+# TODO: Parse output
+cargo run --bin dv -- -c  /tmp/eval_config.json generate-build-cache --project /tmp/uni-factory
 cargo run --bin dv -- --config  /tmp/eval_config.json init --address 0x5e8422345238f34275888049021821e8e08caa1f --contractname frxETH --project examples/frxETH-public --initblock 15728402 examples/dvfs/frx_out.dvf.json
 cargo run --bin dv -- --config  /tmp/eval_config.json sign examples/dvfs/frxETH_filtered.dvf.json
 cargo run --bin dv -- --config  /tmp/eval_config.json validate --validationblock  15729502 examples/dvfs/frxETH_filtered.dvf.json