Fix required vars check & prefer recent config (#35)

* Update tests * Fix database vars check * Update boolean checks * Update test script * Bump version & clean up
LewisDavies · Apr 16, 2024 · 235dfc9 · 235dfc9
1 parent 0379c10
commit 235dfc9
Show file tree

Hide file tree

Showing 46 changed files with 574 additions and 99 deletions.
diff --git a/README.md b/README.md
@@ -272,7 +272,7 @@ Now that your dev models are using prod data, you DAG would look like this:
 ```
 
 ## Compatibility
-`upstream-prod` was initially designed on Snowflake and is now primarily tested on Databricks. Based on my experience and user reports, it is known to work on:
+`upstream-prod` is built and tested on Snowflake. Based on my experience and user reports, it is known to work on:
 - Snowflake
 - Databricks
 - BigQuery

diff --git a/dbt_project.yml b/dbt_project.yml
@@ -1,5 +1,5 @@
 name: 'upstream_prod'
-version: '0.8.0'
+version: '0.8.1'
 config-version: 2
 
 require-dbt-version: [">=1.5.0", "<2.0.0"]

diff --git a/integration_tests/_template/macros/ref.sql b/integration_tests/_template/macros/ref.sql
@@ -6,7 +6,8 @@
     fallback=var("upstream_prod_fallback", False),
     env_schemas=var("upstream_prod_env_schemas", False),
     version=None,
-    prefer_recent=var("upstream_prod_prefer_recent", False)
+    prefer_recent=var("upstream_prod_prefer_recent", False),
+    prod_database_replace=var("upstream_prod_database_replace", None)
 ) %}
 
     {% do return(upstream_prod.ref(parent_model, prod_database, prod_schema, enabled, fallback, env_schemas, version)) %}

diff --git a/integration_tests/_template/run_tests.sh b/integration_tests/_template/run_tests.sh
@@ -1,18 +1,24 @@
 # Setup
 set -e
-echo "\nSETTING UP ENVIRONMENT\n"
+echo ""
+echo "## SETTING UP ENVIRONMENT"
+echo ""
 dbt clean
 dbt deps
 dbt run-operation create_test_db --args '{db: upstream__prod_db}'
 dbt run-operation create_test_db --args '{db: upstream__dev_db}'
 
 # Create staging model in appropriate envs
-echo "\nBUILDING STAGING MODELS\n"
+echo ""
+echo "## BUILDING STAGING MODELS"
+echo ""
 dbt build -s stg__defer_prod stg__defer_vers stg__dev_newer --target prod
 dbt build -s stg__dev_fallback stg__dev_newer
 
 # Build & test downstream models
-echo "\nBUILDING DOWNSTREAM MODELS\n"
+echo ""
+echo "## BUILDING DOWNSTREAM MODELS"
+echo ""
 dbt build -s models/marts
 
 # Check dbt-codegen compatibility

diff --git a/integration_tests/create_symlinks.sh b/integration_tests/create_symlinks.sh
diff --git a/integration_tests/dev_db/macros/ref.sql b/integration_tests/dev_db/macros/ref.sql
@@ -6,7 +6,8 @@
     fallback=var("upstream_prod_fallback", False),
     env_schemas=var("upstream_prod_env_schemas", False),
     version=None,
-    prefer_recent=var("upstream_prod_prefer_recent", False)
+    prefer_recent=var("upstream_prod_prefer_recent", False),
+    prod_database_replace=var("upstream_prod_database_replace", None)
 ) %}
 
     {% do return(upstream_prod.ref(parent_model, prod_database, prod_schema, enabled, fallback, env_schemas, version)) %}

diff --git a/integration_tests/dev_db/run_tests.sh b/integration_tests/dev_db/run_tests.sh
@@ -1,18 +1,24 @@
 # Setup
 set -e
-echo "\nSETTING UP ENVIRONMENT\n"
+echo ""
+echo "## SETTING UP ENVIRONMENT"
+echo ""
 dbt clean
 dbt deps
 dbt run-operation create_test_db --args '{db: upstream__prod_db}'
 dbt run-operation create_test_db --args '{db: upstream__dev_db}'
 
 # Create staging model in appropriate envs
-echo "\nBUILDING STAGING MODELS\n"
+echo ""
+echo "## BUILDING STAGING MODELS"
+echo ""
 dbt build -s stg__defer_prod stg__defer_vers stg__dev_newer --target prod
 dbt build -s stg__dev_fallback stg__dev_newer
 
 # Build & test downstream models
-echo "\nBUILDING DOWNSTREAM MODELS\n"
+echo ""
+echo "## BUILDING DOWNSTREAM MODELS"
+echo ""
 dbt build -s models/marts
 
 # Check dbt-codegen compatibility

diff --git a/integration_tests/dev_db_dev_sch/macros/ref.sql b/integration_tests/dev_db_dev_sch/macros/ref.sql
@@ -6,7 +6,8 @@
     fallback=var("upstream_prod_fallback", False),
     env_schemas=var("upstream_prod_env_schemas", False),
     version=None,
-    prefer_recent=var("upstream_prod_prefer_recent", False)
+    prefer_recent=var("upstream_prod_prefer_recent", False),
+    prod_database_replace=var("upstream_prod_database_replace", None)
 ) %}
 
     {% do return(upstream_prod.ref(parent_model, prod_database, prod_schema, enabled, fallback, env_schemas, version)) %}

diff --git a/integration_tests/dev_db_dev_sch/run_tests.sh b/integration_tests/dev_db_dev_sch/run_tests.sh
@@ -1,18 +1,24 @@
 # Setup
 set -e
-echo "\nSETTING UP ENVIRONMENT\n"
+echo ""
+echo "## SETTING UP ENVIRONMENT"
+echo ""
 dbt clean
 dbt deps
 dbt run-operation create_test_db --args '{db: upstream__prod_db}'
 dbt run-operation create_test_db --args '{db: upstream__dev_db}'
 
 # Create staging model in appropriate envs
-echo "\nBUILDING STAGING MODELS\n"
+echo ""
+echo "## BUILDING STAGING MODELS"
+echo ""
 dbt build -s stg__defer_prod stg__defer_vers stg__dev_newer --target prod
 dbt build -s stg__dev_fallback stg__dev_newer
 
 # Build & test downstream models
-echo "\nBUILDING DOWNSTREAM MODELS\n"
+echo ""
+echo "## BUILDING DOWNSTREAM MODELS"
+echo ""
 dbt build -s models/marts
 
 # Check dbt-codegen compatibility

diff --git a/integration_tests/dev_db_env_sch/macros/ref.sql b/integration_tests/dev_db_env_sch/macros/ref.sql
@@ -6,7 +6,8 @@
     fallback=var("upstream_prod_fallback", False),
     env_schemas=var("upstream_prod_env_schemas", False),
     version=None,
-    prefer_recent=var("upstream_prod_prefer_recent", False)
+    prefer_recent=var("upstream_prod_prefer_recent", False),
+    prod_database_replace=var("upstream_prod_database_replace", None)
 ) %}
 
     {% do return(upstream_prod.ref(parent_model, prod_database, prod_schema, enabled, fallback, env_schemas, version)) %}

diff --git a/integration_tests/dev_db_env_sch/run_tests.sh b/integration_tests/dev_db_env_sch/run_tests.sh
@@ -1,18 +1,24 @@
 # Setup
 set -e
-echo "\nSETTING UP ENVIRONMENT\n"
+echo ""
+echo "## SETTING UP ENVIRONMENT"
+echo ""
 dbt clean
 dbt deps
 dbt run-operation create_test_db --args '{db: upstream__prod_db}'
 dbt run-operation create_test_db --args '{db: upstream__dev_db}'
 
 # Create staging model in appropriate envs
-echo "\nBUILDING STAGING MODELS\n"
+echo ""
+echo "## BUILDING STAGING MODELS"
+echo ""
 dbt build -s stg__defer_prod stg__defer_vers stg__dev_newer --target prod
 dbt build -s stg__dev_fallback stg__dev_newer
 
 # Build & test downstream models
-echo "\nBUILDING DOWNSTREAM MODELS\n"
+echo ""
+echo "## BUILDING DOWNSTREAM MODELS"
+echo ""
 dbt build -s models/marts
 
 # Check dbt-codegen compatibility

diff --git a/integration_tests/dev_db_replace/dbt_project.yml b/integration_tests/dev_db_replace/dbt_project.yml
@@ -0,0 +1,27 @@
+name: 'upstream_prod_integration_tests'
+version: '1.0.0'
+config-version: 2
+
+profile: 'upstream_prod_db'
+
+model-paths: ["models"]
+test-paths: ["tests"]
+macro-paths: ["macros"]
+
+target-path: "target"  # directory which will store compiled SQL files
+clean-targets:         # directories to be removed by `dbt clean`
+  - "target"
+  - "dbt_packages"
+
+
+models:
+  upstream_prod_integration_tests:
+    marts:
+      +schema: marts
+    staging:
+      +schema: stg
+
+vars:
+  upstream_prod_database_replace: [dev, prod]
+  upstream_prod_fallback: true
+  upstream_prod_prefer_recent: true
diff --git a/integration_tests/dev_db_replace/macros/create_test_db.sql b/integration_tests/dev_db_replace/macros/create_test_db.sql
@@ -0,0 +1,10 @@
+{% macro create_test_db(db) -%}
+
+    {% if target.type == "snowflake" %}
+        {% do run_query("create or replace database " ~ db) %}
+    {% elif target.type == "databricks" %}
+        {% do run_query("drop catalog if exists " ~ db ~ " cascade") %}
+        {% do run_query("create catalog " ~ db) %}
+    {% endif %}
+
+{%- endmacro %}
diff --git a/integration_tests/dev_db_replace/macros/ref.sql b/integration_tests/dev_db_replace/macros/ref.sql
@@ -0,0 +1,15 @@
+{% macro ref(
+    parent_model, 
+    prod_database=var("upstream_prod_database", None), 
+    prod_schema=var("upstream_prod_schema", None),
+    enabled=var("upstream_prod_enabled", True),
+    fallback=var("upstream_prod_fallback", False),
+    env_schemas=var("upstream_prod_env_schemas", False),
+    version=None,
+    prefer_recent=var("upstream_prod_prefer_recent", False),
+    prod_database_replace=var("upstream_prod_database_replace", None)
+) %}
+
+    {% do return(upstream_prod.ref(parent_model, prod_database, prod_schema, enabled, fallback, env_schemas, version)) %}
+
+{% endmacro %}