Merge pull request #35 from fivetran/MagicBot/dbt-utils-cross-db-migr…

…ation

Updates for dbt-utils to dbt-core cross-db macro migration

.buildkite/scripts/run_models.sh

@@ -18,4 +18,6 @@ cd integration_tests
 dbt deps
 dbt seed --target "$db" --full-refresh
 dbt run --target "$db" --full-refresh
-dbt test --target "$db"
+dbt test --target "$db"
+dbt run --vars '{ad_reporting__url_report__using_null_filter: false}' --target "$db" --full-refresh
+dbt test --vars '{ad_reporting__url_report__using_null_filter: false}' --target "$db"

CHANGELOG.md

@@ -1,3 +1,47 @@
+# dbt_google_ads v0.9.0
+
+## 🚨 Breaking Changes 🚨:
+[PR #35](https://github.com/fivetran/dbt_google_ads/pull/35) includes the following breaking changes:
+- Dispatch update for dbt-utils to dbt-core cross-db macros migration. Specifically `{{ dbt_utils.<macro> }}` have been updated to `{{ dbt.<macro> }}` for the below macros:
+    - `any_value`
+    - `bool_or`
+    - `cast_bool_to_text`
+    - `concat`
+    - `date_trunc`
+    - `dateadd`
+    - `datediff`
+    - `escape_single_quotes`
+    - `except`
+    - `hash`
+    - `intersect`
+    - `last_day`
+    - `length`
+    - `listagg`
+    - `position`
+    - `replace`
+    - `right`
+    - `safe_cast`
+    - `split_part`
+    - `string_literal`
+    - `type_bigint`
+    - `type_float`
+    - `type_int`
+    - `type_numeric`
+    - `type_string`
+    - `type_timestamp`
+    - `array_append`
+    - `array_concat`
+    - `array_construct`
+- For `current_timestamp` and `current_timestamp_in_utc` macros, the dispatch AND the macro names have been updated to the below, respectively:
+    - `dbt.current_timestamp_backcompat`
+    - `dbt.current_timestamp_in_utc_backcompat`
+- `packages.yml` has been updated to reflect new default `fivetran/fivetran_utils` version, previously `[">=0.3.0", "<0.4.0"]` now `[">=0.4.0", "<0.5.0"]`.
+
+## 🎉 Features 🎉
+- For use in the [dbt_ad_reporting package](https://github.com/fivetran/dbt_ad_reporting), users can now allow records having nulls in url fields to be included in the `ad_reporting__url_report` model. See the [dbt_ad_reporting README](https://github.com/fivetran/dbt_ad_reporting) for more details ([#39](https://github.com/fivetran/dbt_google_ads/pull/39)). 
+## 🚘 Under the Hood 🚘
+- Disabled the `not_null` test for `google_ads__url_report` when null urls are allowed ([#39](https://github.com/fivetran/dbt_google_ads/pull/39)).
+
 # dbt_google_ads v0.8.1
 
 ## Updates:

README.md

@@ -3,7 +3,7 @@
         href="https://github.com/fivetran/dbt_google_ads/blob/main/LICENSE">
         <img src="https://img.shields.io/badge/License-Apache%202.0-blue.svg" /></a>
     <a alt="dbt-core">
-        <img src="https://img.shields.io/badge/dbt_Core™_version->=1.0.0_,<2.0.0-orange.svg" /></a>
+        <img src="https://img.shields.io/badge/dbt_Core™_version->=1.3.0_,<2.0.0-orange.svg" /></a>
     <a alt="Maintained?">
         <img src="https://img.shields.io/badge/Maintained%3F-yes-green.svg" /></a>
     <a alt="PRs">
@@ -39,11 +39,14 @@ To use this dbt package, you must have the following:
 - A **BigQuery**, **Snowflake**, **Redshift**, **PostgreSQL**, or **Databricks** destination.
 
 ### Databricks Dispatch Configuration
-If you are using a Databricks destination with this package you will need to add the below (or a variation of the below) dispatch configuration within your `dbt_project.yml`. This is required in order for the package to accurately search for macros within the `dbt-labs/spark_utils` then the `dbt-labs/dbt_utils` packages respectively.
+If you are using a Databricks destination with this package you will need to add the below (or a variation of the below) dispatch configuration within your `dbt_project.yml`. This is required in order for the package to accurately search for macros within the `dbt-labs/spark_utils` then the `dbt-labs/dbt_utils` as well as the `calogica/dbt_expectations` then the `google_ads_source` packages respectively.
 ```yml
 dispatch:
   - macro_namespace: dbt_utils
     search_order: ['spark_utils', 'dbt_utils']
+
+  - macro_namespace: dbt_expectations
+    search_order: ['google_ads_source', 'dbt_expectations']
 ```
 
 ## Step 2: Install the package
@@ -52,7 +55,7 @@ Include the following google_ads package version in your `packages.yml` file:
 ```yaml
 packages:
   - package: fivetran/google_ads
-    version: [">=0.8.0", "<0.9.0"]
+    version: [">=0.9.0", "<0.10.0"]
 ```
 ## Step 3: Define database and schema variables
 By default, this package runs using your destination and the `google_ads` schema. If this is not where your Google Ads data is (for example, if your Google Ads schema is named `google_ads_fivetran`), add the following configuration to your root `dbt_project.yml` file:
@@ -131,13 +134,22 @@ This dbt package is dependent on the following dbt packages. Please be aware tha
 ```yml
 packages:
     - package: fivetran/google_ads_source
-      version: [">=0.8.0", "<0.9.0"]
+      version: [">=0.9.0", "<0.10.0"]
 
     - package: fivetran/fivetran_utils
-      version: [">=0.3.0", "<0.4.0"]
+      version: [">=0.4.0", "<0.5.0"]
 
     - package: dbt-labs/dbt_utils
+      version: [">=1.0.0", "<2.0.0"]
+
+    - package: dbt-labs/spark_utils
+      version: [">=0.3.0", "<0.4.0"]
+
+    - package: calogica/dbt_expectations
       version: [">=0.8.0", "<0.9.0"]
+
+    - package: calogica/dbt_date
+      version: [">=0.7.0", "<0.8.0"]
 ```
 # 🙌 How is this package maintained and can I contribute?
 ## Package Maintenance

dbt_project.yml

@@ -1,7 +1,8 @@
 name: 'google_ads'
-version: '0.8.1'
+version: '0.9.0'
+
 config-version: 2
-require-dbt-version: [">=1.0.0", "<2.0.0"]
+require-dbt-version: [">=1.3.0", "<2.0.0"]
 models:
   google_ads:
     +schema: google_ads
@@ -18,9 +19,7 @@ vars:
     campaign_stats: "{{ ref('stg_google_ads__campaign_stats') }}"
     keyword_stats: "{{ ref('stg_google_ads__keyword_stats') }}"
     account_stats: "{{ ref('stg_google_ads__account_stats') }}"
-
   google_auto_tagging_enabled: false
-
   google_ads__ad_group_stats_passthrough_metrics: []
   google_ads__ad_stats_passthrough_metrics: []
   google_ads__campaign_stats_passthrough_metrics: []

integration_tests/ci/sample.profiles.yml

@@ -1,5 +1,4 @@
-
-# HEY! This file is used in the dbt package integrations tests with CircleCI.
+# HEY! This file is used in the dbt package integrations tests with Buildkite.
 # You should __NEVER__ check credentials into version control. Thanks for reading :)
 
 config:

integration_tests/dbt_project.yml

@@ -1,5 +1,6 @@
 name: 'google_ads_integration_tests'
-version: '0.8.1'
+version: '0.9.0'
+
 profile: 'integration_tests'
 config-version: 2
 
@@ -32,3 +33,6 @@ seeds:
 dispatch:
   - macro_namespace: dbt_utils
     search_order: ['spark_utils', 'dbt_utils']
+
+  - macro_namespace: dbt_expectations
+    search_order: ['google_ads_source', 'dbt_expectations']

integration_tests/requirements.txt

@@ -1,7 +1,7 @@
-dbt-snowflake>=1.0.0,<1.3.0
-dbt-bigquery>=1.0.0,<1.3.0
-dbt-redshift>=1.0.0,<1.3.0
-dbt-postgres>=1.0.0,<1.3.0
-dbt-spark>=1.0.0,<1.3.0
-dbt-spark[PyHive]>=1.0.0,<1.3.0
-dbt-databricks>=1.0.0,<1.3.0
+dbt-snowflake>=1.3.0,<2.0.0
+dbt-bigquery>=1.3.0,<2.0.0
+dbt-redshift>=1.3.0,<2.0.0
+dbt-postgres>=1.3.0,<2.0.0
+dbt-spark>=1.3.0,<2.0.0
+dbt-spark[PyHive]>=1.3.0,<2.0.0
+dbt-databricks>=1.3.0,<2.0.0

models/google_ads.yml

@@ -9,6 +9,9 @@ models:
             - ad_id
             - ad_group_id
             - date_day
+          config:
+            # Below is so we don't run this test if the user has allowed urls to have null entries
+            enabled: "{{ var('ad_reporting__url_report__using_null_filter', true) }}"
     columns:
       - name: date_day
         description: "{{ doc('date') }}"
@@ -29,7 +32,10 @@ models:
       - name: base_url
         description: "{{ doc('base_url') }}"
         tests:
-          - not_null
+          - not_null:
+              config:
+                # Below is so we don't run this test if the user has allowed urls to have null entries
+                enabled: "{{ var('ad_reporting__url_report__using_null_filter', true) }}"
       - name: url_host
         description: "{{ doc('url_host') }}"
       - name: url_path

models/google_ads__url_report.sql

@@ -83,8 +83,10 @@ fields as (
     left join accounts
         on campaigns.account_id = accounts.account_id
 
-    -- We only want utm ads to populate this report. Therefore, we filter where url ads are populated.
-    where ads.source_final_urls is not null
+    {% if var('ad_reporting__url_report__using_null_filter', True) %}
+        where ads.source_final_urls is not null
+    {% endif %}
+
     {{ dbt_utils.group_by(16) }}
 )
 

packages.yml

@@ -1,3 +1,3 @@
 packages:
-  - package: fivetran/google_ads_source
-    version: [">=0.8.0", "<0.9.0"]
+- package: fivetran/google_ads_source
+  version: [">=0.9.0", "<0.10.0"]