From 4b8541b560cd79f3c212a995165e2af48b9d03e7 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Thu, 12 Mar 2026 15:45:04 -0500 Subject: [PATCH 01/20] test/custom-naming --- integration_tests/dbt_project.yml | 9 ++++++- .../seeds/ad_group_history_data.csv | 2 +- .../seeds/ad_group_level_report_data.csv | 2 +- .../staging/get_ad_group_history_columns.sql | 4 +++- .../get_ad_group_level_report_columns.sql | 4 +++- .../get_advertised_product_report_columns.sql | 4 +++- .../staging/get_campaign_history_columns.sql | 4 +++- .../get_campaign_level_report_columns.sql | 4 +++- .../staging/get_keyword_history_columns.sql | 4 +++- .../staging/get_portfolio_history_columns.sql | 4 +++- .../get_product_ad_history_columns.sql | 4 +++- macros/staging/get_profile_columns.sql | 4 +++- ..._search_term_ad_keyword_report_columns.sql | 4 +++- .../get_targeting_keyword_report_columns.sql | 4 +++- macros/staging/resolve_column_names.sql | 24 +++++++++++++++++++ 15 files changed, 67 insertions(+), 14 deletions(-) create mode 100644 macros/staging/resolve_column_names.sql diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 4b069ae..d41878f 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -7,7 +7,7 @@ profile: 'integration_tests' vars: amazon_ads: - amazon_ads_schema: amazon_ads_integration_tests_03 + amazon_ads_schema: zz_dbt_catherine # amazon_ads_integration_tests_03 amazon_ads_ad_group_history_identifier: "ad_group_history_data" amazon_ads_ad_group_level_report_identifier: "ad_group_level_report_data" amazon_ads_advertised_product_report_identifier: "advertised_product_report_data" @@ -19,6 +19,13 @@ vars: amazon_ads_keyword_history_identifier: "keyword_history_data" amazon_ads_targeting_keyword_report_identifier: "targeting_keyword_report_data" amazon_ads_search_term_ad_keyword_report_identifier: "search_term_ad_keyword_report_data" + using_custom_names: true + custom_column_names: + ad_group_history: + creation_date: creationDate + default_bid: defaultBid + ad_group_level_report: + campaign_bidding_strategy: campaignBiddingStrategy amazon_ads__campaign_passthrough_metrics: - name: sales_7_d diff --git a/integration_tests/seeds/ad_group_history_data.csv b/integration_tests/seeds/ad_group_history_data.csv index 7c241b1..2fa59be 100644 --- a/integration_tests/seeds/ad_group_history_data.csv +++ b/integration_tests/seeds/ad_group_history_data.csv @@ -1,3 +1,3 @@ -id,last_updated_date,_fivetran_synced,campaign_id,creation_date,default_bid,name,serving_status,state +id,last_updated_date,_fivetran_synced,campaign_id,creationDate,defaultBid,name,serving_status,state 421,2022-07-11 22:38:16.551000,2022-12-13 17:10:16.297000,2187,2022-07-11 22:38:16.551000,1.75,Red 7,CAMPAIGN_PAUSED,enabled 501,2022-11-09 14:37:05.332000,2022-12-13 17:11:08.594000,5555,2022-11-09 14:37:05.332000,1.45,All,AD_GROUP_STATUS_ENABLED,enabled \ No newline at end of file diff --git a/integration_tests/seeds/ad_group_level_report_data.csv b/integration_tests/seeds/ad_group_level_report_data.csv index 9b0c5bc..869819a 100644 --- a/integration_tests/seeds/ad_group_level_report_data.csv +++ b/integration_tests/seeds/ad_group_level_report_data.csv @@ -1,4 +1,4 @@ -ad_group_id,date,_fivetran_synced,campaign_bidding_strategy,clicks,cost,impressions +ad_group_id,date,_fivetran_synced,campaignBiddingStrategy,clicks,cost,impressions 501,2022-12-03,2022-12-13 11:35:56.905000,legacy,0,0.0,88 501,2022-12-04,2022-12-13 11:35:56.908000,legacy,0,0.0,111 501,2022-12-01,2022-12-13 11:35:56.899000,legacy,0,0.0,252 diff --git a/macros/staging/get_ad_group_history_columns.sql b/macros/staging/get_ad_group_history_columns.sql index 8d28004..4765c6f 100644 --- a/macros/staging/get_ad_group_history_columns.sql +++ b/macros/staging/get_ad_group_history_columns.sql @@ -1,6 +1,6 @@ {% macro get_ad_group_history_columns() %} -{% set columns = [ +{% set _fivetran_columns = [ {"name": "campaign_id", "datatype": dbt.type_int()}, {"name": "creation_date", "datatype": dbt.type_timestamp()}, {"name": "default_bid", "datatype": dbt.type_float()}, @@ -11,6 +11,8 @@ {"name": "state", "datatype": dbt.type_string()} ] %} +{% set columns = amazon_ads.resolve_column_names('ad_group_history', _fivetran_columns) %} + {{ return(columns) }} {% endmacro %} diff --git a/macros/staging/get_ad_group_level_report_columns.sql b/macros/staging/get_ad_group_level_report_columns.sql index 9c076a8..002de5a 100644 --- a/macros/staging/get_ad_group_level_report_columns.sql +++ b/macros/staging/get_ad_group_level_report_columns.sql @@ -1,6 +1,6 @@ {% macro get_ad_group_level_report_columns() %} -{% set columns = [ +{% set _fivetran_columns = [ {"name": "ad_group_id", "datatype": dbt.type_int()}, {"name": "campaign_bidding_strategy", "datatype": dbt.type_string()}, {"name": "clicks", "datatype": dbt.type_int()}, @@ -11,6 +11,8 @@ {"name": "sales_30_d", "datatype": dbt.type_float()} ] %} +{% set columns = amazon_ads.resolve_column_names('ad_group_level_report', _fivetran_columns) %} + {# Add backwards compatibility if conversion metrics were added via passthrough columns prior to them being brought in by default #} {{ amazon_ads_add_pass_through_columns(base_columns=columns, pass_through_fields=var('amazon_ads__ad_group_passthrough_metrics'), except_fields=['purchases_30_d', 'sales_30_d']) }} diff --git a/macros/staging/get_advertised_product_report_columns.sql b/macros/staging/get_advertised_product_report_columns.sql index 7447056..5d10266 100644 --- a/macros/staging/get_advertised_product_report_columns.sql +++ b/macros/staging/get_advertised_product_report_columns.sql @@ -1,6 +1,6 @@ {% macro get_advertised_product_report_columns() %} -{% set columns = [ +{% set _fivetran_columns = [ {"name": "ad_group_id", "datatype": dbt.type_int()}, {"name": "ad_id", "datatype": dbt.type_int()}, {"name": "advertised_asin", "datatype": dbt.type_string()}, @@ -17,6 +17,8 @@ {"name": "sales_30_d", "datatype": dbt.type_float()} ] %} +{% set columns = amazon_ads.resolve_column_names('advertised_product_report', _fivetran_columns) %} + {# Add backwards compatibility if conversion metrics were added via passthrough columns prior to them being brought in by default #} {{ amazon_ads_add_pass_through_columns(base_columns=columns, pass_through_fields=var('amazon_ads__advertised_product_passthrough_metrics'), except_fields=['purchases_30_d', 'sales_30_d']) }} diff --git a/macros/staging/get_campaign_history_columns.sql b/macros/staging/get_campaign_history_columns.sql index 52215ab..bf52757 100644 --- a/macros/staging/get_campaign_history_columns.sql +++ b/macros/staging/get_campaign_history_columns.sql @@ -1,6 +1,6 @@ {% macro get_campaign_history_columns() %} -{% set columns = [ +{% set _fivetran_columns = [ {"name": "bidding_strategy", "datatype": dbt.type_string()}, {"name": "creation_date", "datatype": dbt.type_timestamp()}, {"name": "end_date", "datatype": "date"}, @@ -18,6 +18,8 @@ {"name": "effective_budget", "datatype": dbt.type_float()} ] %} +{% set columns = amazon_ads.resolve_column_names('campaign_history', _fivetran_columns) %} + {{ return(columns) }} {% endmacro %} diff --git a/macros/staging/get_campaign_level_report_columns.sql b/macros/staging/get_campaign_level_report_columns.sql index 977e9d3..fd7d478 100644 --- a/macros/staging/get_campaign_level_report_columns.sql +++ b/macros/staging/get_campaign_level_report_columns.sql @@ -1,6 +1,6 @@ {% macro get_campaign_level_report_columns() %} -{% set columns = [ +{% set _fivetran_columns = [ {"name": "campaign_applicable_budget_rule_id", "datatype": dbt.type_string()}, {"name": "campaign_applicable_budget_rule_name", "datatype": dbt.type_string()}, {"name": "campaign_bidding_strategy", "datatype": dbt.type_string()}, @@ -17,6 +17,8 @@ {"name": "sales_30_d", "datatype": dbt.type_float()} ] %} +{% set columns = amazon_ads.resolve_column_names('campaign_level_report', _fivetran_columns) %} + {# Add backwards compatibility if conversion metrics were added via passthrough columns prior to them being brought in by default #} {{ amazon_ads_add_pass_through_columns(base_columns=columns, pass_through_fields=var('amazon_ads__campaign_passthrough_metrics'), except_fields=['purchases_30_d', 'sales_30_d']) }} diff --git a/macros/staging/get_keyword_history_columns.sql b/macros/staging/get_keyword_history_columns.sql index dc32c8f..5a0f179 100644 --- a/macros/staging/get_keyword_history_columns.sql +++ b/macros/staging/get_keyword_history_columns.sql @@ -1,6 +1,6 @@ {% macro get_keyword_history_columns() %} -{% set columns = [ +{% set _fivetran_columns = [ {"name": "ad_group_id", "datatype": dbt.type_int()}, {"name": "bid", "datatype": dbt.type_float()}, {"name": "campaign_id", "datatype": dbt.type_int()}, @@ -15,6 +15,8 @@ {"name": "native_language_locale", "datatype": dbt.type_string()} ] %} +{% set columns = amazon_ads.resolve_column_names('keyword_history', _fivetran_columns) %} + {{ return(columns) }} {% endmacro %} diff --git a/macros/staging/get_portfolio_history_columns.sql b/macros/staging/get_portfolio_history_columns.sql index bac28e5..046ef71 100644 --- a/macros/staging/get_portfolio_history_columns.sql +++ b/macros/staging/get_portfolio_history_columns.sql @@ -1,6 +1,6 @@ {% macro get_portfolio_history_columns() %} -{% set columns = [ +{% set _fivetran_columns = [ {"name": "budget_amount", "datatype": dbt.type_float()}, {"name": "budget_currency_code", "datatype": dbt.type_string()}, {"name": "budget_end_date", "datatype": "date"}, @@ -16,6 +16,8 @@ {"name": "state", "datatype": dbt.type_string()} ] %} +{% set columns = amazon_ads.resolve_column_names('portfolio_history', _fivetran_columns) %} + {{ return(columns) }} {% endmacro %} diff --git a/macros/staging/get_product_ad_history_columns.sql b/macros/staging/get_product_ad_history_columns.sql index 36b5c99..ba735e2 100644 --- a/macros/staging/get_product_ad_history_columns.sql +++ b/macros/staging/get_product_ad_history_columns.sql @@ -1,6 +1,6 @@ {% macro get_product_ad_history_columns() %} -{% set columns = [ +{% set _fivetran_columns = [ {"name": "ad_group_id", "datatype": dbt.type_int()}, {"name": "asin", "datatype": dbt.type_string()}, {"name": "campaign_id", "datatype": dbt.type_int()}, @@ -12,6 +12,8 @@ {"name": "state", "datatype": dbt.type_string()} ] %} +{% set columns = amazon_ads.resolve_column_names('product_ad_history', _fivetran_columns) %} + {{ return(columns) }} {% endmacro %} diff --git a/macros/staging/get_profile_columns.sql b/macros/staging/get_profile_columns.sql index 6670012..364c6ab 100644 --- a/macros/staging/get_profile_columns.sql +++ b/macros/staging/get_profile_columns.sql @@ -1,6 +1,6 @@ {% macro get_profile_columns() %} -{% set columns = [ +{% set _fivetran_columns = [ {"name": "id", "datatype": dbt.type_int()}, {"name": "account_id", "datatype": dbt.type_string()}, {"name": "account_marketplace_string_id", "datatype": dbt.type_string()}, @@ -15,6 +15,8 @@ {"name": "_fivetran_deleted", "datatype": dbt.type_boolean()} ] %} +{% set columns = amazon_ads.resolve_column_names('profile', _fivetran_columns) %} + {{ return(columns) }} {% endmacro %} \ No newline at end of file diff --git a/macros/staging/get_search_term_ad_keyword_report_columns.sql b/macros/staging/get_search_term_ad_keyword_report_columns.sql index 35900c4..f96458c 100644 --- a/macros/staging/get_search_term_ad_keyword_report_columns.sql +++ b/macros/staging/get_search_term_ad_keyword_report_columns.sql @@ -1,6 +1,6 @@ {% macro get_search_term_ad_keyword_report_columns() %} -{% set columns = [ +{% set _fivetran_columns = [ {"name": "ad_group_id", "datatype": dbt.type_int()}, {"name": "ad_keyword_status", "datatype": dbt.type_string()}, {"name": "campaign_budget_amount", "datatype": dbt.type_float()}, @@ -19,6 +19,8 @@ {"name": "sales_30_d", "datatype": dbt.type_float()} ] %} +{% set columns = amazon_ads.resolve_column_names('search_term_ad_keyword_report', _fivetran_columns) %} + {# Add backwards compatibility if conversion metrics were added via passthrough columns prior to them being brought in by default #} {{ amazon_ads_add_pass_through_columns(base_columns=columns, pass_through_fields=var('amazon_ads__search_term_ad_keyword_passthrough_metrics'), except_fields=['purchases_30_d', 'sales_30_d']) }} diff --git a/macros/staging/get_targeting_keyword_report_columns.sql b/macros/staging/get_targeting_keyword_report_columns.sql index 24b557a..31ac3ca 100644 --- a/macros/staging/get_targeting_keyword_report_columns.sql +++ b/macros/staging/get_targeting_keyword_report_columns.sql @@ -1,6 +1,6 @@ {% macro get_targeting_keyword_report_columns() %} -{% set columns = [ +{% set _fivetran_columns = [ {"name": "ad_group_id", "datatype": dbt.type_int()}, {"name": "ad_keyword_status", "datatype": dbt.type_string()}, {"name": "campaign_budget_amount", "datatype": dbt.type_float()}, @@ -20,6 +20,8 @@ {"name": "sales_30_d", "datatype": dbt.type_float()} ] %} +{% set columns = amazon_ads.resolve_column_names('targeting_keyword_report', _fivetran_columns) %} + {# Add backwards compatibility if conversion metrics were added via passthrough columns prior to them being brought in by default #} {{ amazon_ads_add_pass_through_columns(base_columns=columns, pass_through_fields=var('amazon_ads__targeting_keyword_passthrough_metrics'), except_fields=['purchases_30_d', 'sales_30_d']) }} diff --git a/macros/staging/resolve_column_names.sql b/macros/staging/resolve_column_names.sql new file mode 100644 index 0000000..132abfc --- /dev/null +++ b/macros/staging/resolve_column_names.sql @@ -0,0 +1,24 @@ +{% macro resolve_column_names(_fivetran_table_name, _fivetran_columns) %} + {{ return(adapter.dispatch('resolve_column_names', 'amazon_ads')(_fivetran_table_name, _fivetran_columns)) }} +{% endmacro %} + +{% macro default__resolve_column_names(_fivetran_table_name, _fivetran_columns) %} + +{% set resolved_column_names = _fivetran_columns %} + +{% if var('using_custom_names', false) %} + {% set resolved_column_names = [] %} + {% set custom_column_names = var('custom_column_names', {}) %} + + {% for column in _fivetran_columns %} + {% set column_name = column.name %} + {# Use the custom name if it exists, otherwise use the original column name #} + {% set resolved_name = custom_column_names.get(_fivetran_table_name, {}).get(column_name, column_name) %} + {% set updated_column = dict(column, name=resolved_name, alias=column_name) %} + {% do resolved_column_names.append(updated_column) %} + {% endfor %} +{% endif %} + +{{ return(resolved_column_names) }} + +{% endmacro %} \ No newline at end of file From 5ec6afdf62f6663b02b93c9f273047e014c1d639 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Thu, 12 Mar 2026 15:46:33 -0500 Subject: [PATCH 02/20] yml --- integration_tests/dbt_project.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index d41878f..817424d 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -7,7 +7,7 @@ profile: 'integration_tests' vars: amazon_ads: - amazon_ads_schema: zz_dbt_catherine # amazon_ads_integration_tests_03 + amazon_ads_schema: amazon_ads_integration_tests_03 amazon_ads_ad_group_history_identifier: "ad_group_history_data" amazon_ads_ad_group_level_report_identifier: "ad_group_level_report_data" amazon_ads_advertised_product_report_identifier: "advertised_product_report_data" From 884a78530d53d055a24b093d06dc5bef04c8f24f Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Thu, 12 Mar 2026 15:47:40 -0500 Subject: [PATCH 03/20] pre-exit --- .buildkite/hooks/pre-exit | 1 + 1 file changed, 1 insertion(+) create mode 100644 .buildkite/hooks/pre-exit diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit new file mode 100644 index 0000000..bd49d0f --- /dev/null +++ b/.buildkite/hooks/pre-exit @@ -0,0 +1 @@ +dbt run-operation fivetran_utils.drop_schemas_automation --target "$db" \ No newline at end of file From bda1cff16530fa882e967ca6228d56e8b9117924 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Thu, 12 Mar 2026 15:51:14 -0500 Subject: [PATCH 04/20] pre-exit --- .buildkite/hooks/pre-exit | 15 ++++++++++++++- .buildkite/scripts/run_models.sh | 4 +--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit index bd49d0f..23fe2a9 100644 --- a/.buildkite/hooks/pre-exit +++ b/.buildkite/hooks/pre-exit @@ -1 +1,14 @@ -dbt run-operation fivetran_utils.drop_schemas_automation --target "$db" \ No newline at end of file +#!/bin/bash + +# Pre-exit hook - runs when Buildkite job is about to exit (success, failure, or cancellation) + +set -euo pipefail + +echo "Running schema cleanup..." + +# Drop test schemas to avoid leaving behind temporary databases +if [ -n "${db:-}" ]; then + dbt run-operation fivetran_utils.drop_schemas_automation --target "$db" || echo "Schema cleanup failed, continuing..." +fi + +echo "Schema cleanup completed" \ No newline at end of file diff --git a/.buildkite/scripts/run_models.sh b/.buildkite/scripts/run_models.sh index fb75f9b..2b9b793 100644 --- a/.buildkite/scripts/run_models.sh +++ b/.buildkite/scripts/run_models.sh @@ -20,6 +20,4 @@ dbt seed --target "$db" --full-refresh dbt run --target "$db" --full-refresh dbt test --target "$db" dbt run --vars '{amazon_ads__portfolio_history_enabled: false}' --target "$db" --full-refresh -dbt test --vars '{amazon_ads__portfolio_history_enabled: false}' --target "$db" - -dbt run-operation fivetran_utils.drop_schemas_automation --target "$db" +dbt test --vars '{amazon_ads__portfolio_history_enabled: false}' --target "$db" \ No newline at end of file From a87200d94ddd70f3115c69a1b543969a956d0471 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Thu, 12 Mar 2026 16:01:53 -0500 Subject: [PATCH 05/20] undo pre-exit --- .buildkite/hooks/pre-exit | 14 -------------- .buildkite/scripts/run_models.sh | 4 +++- 2 files changed, 3 insertions(+), 15 deletions(-) delete mode 100644 .buildkite/hooks/pre-exit diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit deleted file mode 100644 index 23fe2a9..0000000 --- a/.buildkite/hooks/pre-exit +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -# Pre-exit hook - runs when Buildkite job is about to exit (success, failure, or cancellation) - -set -euo pipefail - -echo "Running schema cleanup..." - -# Drop test schemas to avoid leaving behind temporary databases -if [ -n "${db:-}" ]; then - dbt run-operation fivetran_utils.drop_schemas_automation --target "$db" || echo "Schema cleanup failed, continuing..." -fi - -echo "Schema cleanup completed" \ No newline at end of file diff --git a/.buildkite/scripts/run_models.sh b/.buildkite/scripts/run_models.sh index 2b9b793..61992d7 100644 --- a/.buildkite/scripts/run_models.sh +++ b/.buildkite/scripts/run_models.sh @@ -20,4 +20,6 @@ dbt seed --target "$db" --full-refresh dbt run --target "$db" --full-refresh dbt test --target "$db" dbt run --vars '{amazon_ads__portfolio_history_enabled: false}' --target "$db" --full-refresh -dbt test --vars '{amazon_ads__portfolio_history_enabled: false}' --target "$db" \ No newline at end of file +dbt test --vars '{amazon_ads__portfolio_history_enabled: false}' --target "$db" + +dbt run-operation fivetran_utils.drop_schemas_automation --target "$db" \ No newline at end of file From 2c0819c91d95e11ef958683063b8a50a726afba1 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Thu, 12 Mar 2026 17:05:07 -0500 Subject: [PATCH 06/20] revise var names --- integration_tests/dbt_project.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 817424d..735b949 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -19,9 +19,9 @@ vars: amazon_ads_keyword_history_identifier: "keyword_history_data" amazon_ads_targeting_keyword_report_identifier: "targeting_keyword_report_data" amazon_ads_search_term_ad_keyword_report_identifier: "search_term_ad_keyword_report_data" - using_custom_names: true - custom_column_names: - ad_group_history: + amazon_ads_using_custom_names: true + amazon_ads_custom_column_names: + amazon_ads_custom_column_names: creation_date: creationDate default_bid: defaultBid ad_group_level_report: From 4b7aa1997fe6637639a9331903377dd309321c25 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Thu, 12 Mar 2026 17:06:22 -0500 Subject: [PATCH 07/20] revise var names --- integration_tests/dbt_project.yml | 2 +- macros/staging/resolve_column_names.sql | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 735b949..797f52c 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -21,7 +21,7 @@ vars: amazon_ads_search_term_ad_keyword_report_identifier: "search_term_ad_keyword_report_data" amazon_ads_using_custom_names: true amazon_ads_custom_column_names: - amazon_ads_custom_column_names: + ad_group_history: creation_date: creationDate default_bid: defaultBid ad_group_level_report: diff --git a/macros/staging/resolve_column_names.sql b/macros/staging/resolve_column_names.sql index 132abfc..0a627f5 100644 --- a/macros/staging/resolve_column_names.sql +++ b/macros/staging/resolve_column_names.sql @@ -6,9 +6,9 @@ {% set resolved_column_names = _fivetran_columns %} -{% if var('using_custom_names', false) %} +{% if var('amazon_ads_using_custom_names', false) %} {% set resolved_column_names = [] %} - {% set custom_column_names = var('custom_column_names', {}) %} + {% set custom_column_names = var('amazon_ads_custom_column_names', {}) %} {% for column in _fivetran_columns %} {% set column_name = column.name %} From 9fe483f2afa09feced68e040c8e42d7608f22b88 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Thu, 12 Mar 2026 17:10:09 -0500 Subject: [PATCH 08/20] revise var names --- macros/staging/resolve_column_names.sql | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/macros/staging/resolve_column_names.sql b/macros/staging/resolve_column_names.sql index 0a627f5..6b84722 100644 --- a/macros/staging/resolve_column_names.sql +++ b/macros/staging/resolve_column_names.sql @@ -4,11 +4,12 @@ {% macro default__resolve_column_names(_fivetran_table_name, _fivetran_columns) %} +{% set package_name = 'amazon_ads' %} {% set resolved_column_names = _fivetran_columns %} -{% if var('amazon_ads_using_custom_names', false) %} +{% if var(package_name ~ '_using_custom_names', false) %} {% set resolved_column_names = [] %} - {% set custom_column_names = var('amazon_ads_custom_column_names', {}) %} + {% set custom_column_names = var(package_name ~ '_custom_column_names', {}) %} {% for column in _fivetran_columns %} {% set column_name = column.name %} From fd0b474bec1951786d67809b58594a8e7c8ab38a Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Thu, 12 Mar 2026 17:36:25 -0500 Subject: [PATCH 09/20] revise macro --- macros/staging/resolve_column_names.sql | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/macros/staging/resolve_column_names.sql b/macros/staging/resolve_column_names.sql index 6b84722..e48083b 100644 --- a/macros/staging/resolve_column_names.sql +++ b/macros/staging/resolve_column_names.sql @@ -1,11 +1,8 @@ -{% macro resolve_column_names(_fivetran_table_name, _fivetran_columns) %} - {{ return(adapter.dispatch('resolve_column_names', 'amazon_ads')(_fivetran_table_name, _fivetran_columns)) }} +{% macro resolve_column_names(_fivetran_table_name, _fivetran_columns, package_name='amazon_ads') %} + {{ return(adapter.dispatch('resolve_column_names', 'amazon_ads')(_fivetran_table_name, _fivetran_columns, package_name)) }} {% endmacro %} -{% macro default__resolve_column_names(_fivetran_table_name, _fivetran_columns) %} - -{% set package_name = 'amazon_ads' %} -{% set resolved_column_names = _fivetran_columns %} +{% macro default__resolve_column_names(_fivetran_table_name, _fivetran_columns, package_name) %} {% if var(package_name ~ '_using_custom_names', false) %} {% set resolved_column_names = [] %} From 012f58ef0c2c81813e8a8ed74790dac0b9001d9b Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Thu, 12 Mar 2026 17:36:55 -0500 Subject: [PATCH 10/20] revise macro --- integration_tests/dbt_project.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 797f52c..74cd061 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -20,6 +20,7 @@ vars: amazon_ads_targeting_keyword_report_identifier: "targeting_keyword_report_data" amazon_ads_search_term_ad_keyword_report_identifier: "search_term_ad_keyword_report_data" amazon_ads_using_custom_names: true + amazon_ads_custom_column_names: ad_group_history: creation_date: creationDate From ae3c6e1f495c38aa809ba581d9417ead0a8e1acb Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Thu, 19 Mar 2026 17:39:20 -0500 Subject: [PATCH 11/20] update stg --- integration_tests/dbt_project.yml | 41 +++++++------ .../staging/get_ad_group_history_columns.sql | 4 +- .../get_ad_group_level_report_columns.sql | 4 +- .../get_advertised_product_report_columns.sql | 4 +- .../staging/get_campaign_history_columns.sql | 4 +- .../get_campaign_level_report_columns.sql | 4 +- .../staging/get_keyword_history_columns.sql | 4 +- .../staging/get_portfolio_history_columns.sql | 4 +- .../get_product_ad_history_columns.sql | 4 +- macros/staging/get_profile_columns.sql | 4 +- ..._search_term_ad_keyword_report_columns.sql | 4 +- .../get_targeting_keyword_report_columns.sql | 4 +- macros/staging/resolve_column_names.sql | 57 +++++++++++++++---- .../stg_amazon_ads__ad_group_history.sql | 30 +++++----- .../stg_amazon_ads__ad_group_level_report.sql | 33 +++++------ ..._amazon_ads__advertised_product_report.sql | 32 +++++------ .../stg_amazon_ads__campaign_history.sql | 30 +++++----- .../stg_amazon_ads__campaign_level_report.sql | 30 +++++----- .../stg_amazon_ads__keyword_history.sql | 30 +++++----- .../stg_amazon_ads__portfolio_history.sql | 30 +++++----- .../stg_amazon_ads__product_ad_history.sql | 30 +++++----- models/staging/stg_amazon_ads__profile.sql | 30 +++++----- ...zon_ads__search_term_ad_keyword_report.sql | 30 +++++----- ...g_amazon_ads__targeting_keyword_report.sql | 30 +++++----- 24 files changed, 248 insertions(+), 229 deletions(-) diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 74cd061..cbaa034 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -7,26 +7,33 @@ profile: 'integration_tests' vars: amazon_ads: - amazon_ads_schema: amazon_ads_integration_tests_03 - amazon_ads_ad_group_history_identifier: "ad_group_history_data" - amazon_ads_ad_group_level_report_identifier: "ad_group_level_report_data" - amazon_ads_advertised_product_report_identifier: "advertised_product_report_data" - amazon_ads_campaign_history_identifier: "campaign_history_data" - amazon_ads_campaign_level_report_identifier: "campaign_level_report_data" - amazon_ads_portfolio_history_identifier: "portfolio_history_data" - amazon_ads_product_ad_history_identifier: "product_ad_history_data" - amazon_ads_profile_identifier: "profile_data" - amazon_ads_keyword_history_identifier: "keyword_history_data" - amazon_ads_targeting_keyword_report_identifier: "targeting_keyword_report_data" - amazon_ads_search_term_ad_keyword_report_identifier: "search_term_ad_keyword_report_data" + amazon_ads_union_schemas: ['zz_dbt_catherine_3', 'zz_dbt_catherine_4'] + # amazon_ads_sources: + # - database: postgres + # schema: zz_dbt_catherine_3 # Required + # - database: postgres + # schema: zz_dbt_catherine_4 + # amazon_ads_ad_group_history_identifier: "ad_group_history_data" + # amazon_ads_ad_group_level_report_identifier: "ad_group_level_report_data" + # amazon_ads_advertised_product_report_identifier: "advertised_product_report_data" + # amazon_ads_campaign_history_identifier: "campaign_history_data" + # amazon_ads_campaign_level_report_identifier: "campaign_level_report_data" + # amazon_ads_portfolio_history_identifier: "portfolio_history_data" + # amazon_ads_product_ad_history_identifier: "product_ad_history_data" + # amazon_ads_profile_identifier: "profile_data" + # amazon_ads_keyword_history_identifier: "keyword_history_data" + # amazon_ads_targeting_keyword_report_identifier: "targeting_keyword_report_data" + # amazon_ads_search_term_ad_keyword_report_identifier: "search_term_ad_keyword_report_data" amazon_ads_using_custom_names: true amazon_ads_custom_column_names: - ad_group_history: - creation_date: creationDate - default_bid: defaultBid - ad_group_level_report: - campaign_bidding_strategy: campaignBiddingStrategy + zz_dbt_catherine_3: + ad_group_history: + creation_date: creationDate + default_bid: defaultBid + zz_dbt_catherine_4: + ad_group_level_report: + campaign_bidding_strategy: campaignBiddingStrategy amazon_ads__campaign_passthrough_metrics: - name: sales_7_d diff --git a/macros/staging/get_ad_group_history_columns.sql b/macros/staging/get_ad_group_history_columns.sql index 4765c6f..8d28004 100644 --- a/macros/staging/get_ad_group_history_columns.sql +++ b/macros/staging/get_ad_group_history_columns.sql @@ -1,6 +1,6 @@ {% macro get_ad_group_history_columns() %} -{% set _fivetran_columns = [ +{% set columns = [ {"name": "campaign_id", "datatype": dbt.type_int()}, {"name": "creation_date", "datatype": dbt.type_timestamp()}, {"name": "default_bid", "datatype": dbt.type_float()}, @@ -11,8 +11,6 @@ {"name": "state", "datatype": dbt.type_string()} ] %} -{% set columns = amazon_ads.resolve_column_names('ad_group_history', _fivetran_columns) %} - {{ return(columns) }} {% endmacro %} diff --git a/macros/staging/get_ad_group_level_report_columns.sql b/macros/staging/get_ad_group_level_report_columns.sql index 002de5a..9c076a8 100644 --- a/macros/staging/get_ad_group_level_report_columns.sql +++ b/macros/staging/get_ad_group_level_report_columns.sql @@ -1,6 +1,6 @@ {% macro get_ad_group_level_report_columns() %} -{% set _fivetran_columns = [ +{% set columns = [ {"name": "ad_group_id", "datatype": dbt.type_int()}, {"name": "campaign_bidding_strategy", "datatype": dbt.type_string()}, {"name": "clicks", "datatype": dbt.type_int()}, @@ -11,8 +11,6 @@ {"name": "sales_30_d", "datatype": dbt.type_float()} ] %} -{% set columns = amazon_ads.resolve_column_names('ad_group_level_report', _fivetran_columns) %} - {# Add backwards compatibility if conversion metrics were added via passthrough columns prior to them being brought in by default #} {{ amazon_ads_add_pass_through_columns(base_columns=columns, pass_through_fields=var('amazon_ads__ad_group_passthrough_metrics'), except_fields=['purchases_30_d', 'sales_30_d']) }} diff --git a/macros/staging/get_advertised_product_report_columns.sql b/macros/staging/get_advertised_product_report_columns.sql index 5d10266..7447056 100644 --- a/macros/staging/get_advertised_product_report_columns.sql +++ b/macros/staging/get_advertised_product_report_columns.sql @@ -1,6 +1,6 @@ {% macro get_advertised_product_report_columns() %} -{% set _fivetran_columns = [ +{% set columns = [ {"name": "ad_group_id", "datatype": dbt.type_int()}, {"name": "ad_id", "datatype": dbt.type_int()}, {"name": "advertised_asin", "datatype": dbt.type_string()}, @@ -17,8 +17,6 @@ {"name": "sales_30_d", "datatype": dbt.type_float()} ] %} -{% set columns = amazon_ads.resolve_column_names('advertised_product_report', _fivetran_columns) %} - {# Add backwards compatibility if conversion metrics were added via passthrough columns prior to them being brought in by default #} {{ amazon_ads_add_pass_through_columns(base_columns=columns, pass_through_fields=var('amazon_ads__advertised_product_passthrough_metrics'), except_fields=['purchases_30_d', 'sales_30_d']) }} diff --git a/macros/staging/get_campaign_history_columns.sql b/macros/staging/get_campaign_history_columns.sql index bf52757..52215ab 100644 --- a/macros/staging/get_campaign_history_columns.sql +++ b/macros/staging/get_campaign_history_columns.sql @@ -1,6 +1,6 @@ {% macro get_campaign_history_columns() %} -{% set _fivetran_columns = [ +{% set columns = [ {"name": "bidding_strategy", "datatype": dbt.type_string()}, {"name": "creation_date", "datatype": dbt.type_timestamp()}, {"name": "end_date", "datatype": "date"}, @@ -18,8 +18,6 @@ {"name": "effective_budget", "datatype": dbt.type_float()} ] %} -{% set columns = amazon_ads.resolve_column_names('campaign_history', _fivetran_columns) %} - {{ return(columns) }} {% endmacro %} diff --git a/macros/staging/get_campaign_level_report_columns.sql b/macros/staging/get_campaign_level_report_columns.sql index fd7d478..977e9d3 100644 --- a/macros/staging/get_campaign_level_report_columns.sql +++ b/macros/staging/get_campaign_level_report_columns.sql @@ -1,6 +1,6 @@ {% macro get_campaign_level_report_columns() %} -{% set _fivetran_columns = [ +{% set columns = [ {"name": "campaign_applicable_budget_rule_id", "datatype": dbt.type_string()}, {"name": "campaign_applicable_budget_rule_name", "datatype": dbt.type_string()}, {"name": "campaign_bidding_strategy", "datatype": dbt.type_string()}, @@ -17,8 +17,6 @@ {"name": "sales_30_d", "datatype": dbt.type_float()} ] %} -{% set columns = amazon_ads.resolve_column_names('campaign_level_report', _fivetran_columns) %} - {# Add backwards compatibility if conversion metrics were added via passthrough columns prior to them being brought in by default #} {{ amazon_ads_add_pass_through_columns(base_columns=columns, pass_through_fields=var('amazon_ads__campaign_passthrough_metrics'), except_fields=['purchases_30_d', 'sales_30_d']) }} diff --git a/macros/staging/get_keyword_history_columns.sql b/macros/staging/get_keyword_history_columns.sql index 5a0f179..dc32c8f 100644 --- a/macros/staging/get_keyword_history_columns.sql +++ b/macros/staging/get_keyword_history_columns.sql @@ -1,6 +1,6 @@ {% macro get_keyword_history_columns() %} -{% set _fivetran_columns = [ +{% set columns = [ {"name": "ad_group_id", "datatype": dbt.type_int()}, {"name": "bid", "datatype": dbt.type_float()}, {"name": "campaign_id", "datatype": dbt.type_int()}, @@ -15,8 +15,6 @@ {"name": "native_language_locale", "datatype": dbt.type_string()} ] %} -{% set columns = amazon_ads.resolve_column_names('keyword_history', _fivetran_columns) %} - {{ return(columns) }} {% endmacro %} diff --git a/macros/staging/get_portfolio_history_columns.sql b/macros/staging/get_portfolio_history_columns.sql index 046ef71..bac28e5 100644 --- a/macros/staging/get_portfolio_history_columns.sql +++ b/macros/staging/get_portfolio_history_columns.sql @@ -1,6 +1,6 @@ {% macro get_portfolio_history_columns() %} -{% set _fivetran_columns = [ +{% set columns = [ {"name": "budget_amount", "datatype": dbt.type_float()}, {"name": "budget_currency_code", "datatype": dbt.type_string()}, {"name": "budget_end_date", "datatype": "date"}, @@ -16,8 +16,6 @@ {"name": "state", "datatype": dbt.type_string()} ] %} -{% set columns = amazon_ads.resolve_column_names('portfolio_history', _fivetran_columns) %} - {{ return(columns) }} {% endmacro %} diff --git a/macros/staging/get_product_ad_history_columns.sql b/macros/staging/get_product_ad_history_columns.sql index ba735e2..36b5c99 100644 --- a/macros/staging/get_product_ad_history_columns.sql +++ b/macros/staging/get_product_ad_history_columns.sql @@ -1,6 +1,6 @@ {% macro get_product_ad_history_columns() %} -{% set _fivetran_columns = [ +{% set columns = [ {"name": "ad_group_id", "datatype": dbt.type_int()}, {"name": "asin", "datatype": dbt.type_string()}, {"name": "campaign_id", "datatype": dbt.type_int()}, @@ -12,8 +12,6 @@ {"name": "state", "datatype": dbt.type_string()} ] %} -{% set columns = amazon_ads.resolve_column_names('product_ad_history', _fivetran_columns) %} - {{ return(columns) }} {% endmacro %} diff --git a/macros/staging/get_profile_columns.sql b/macros/staging/get_profile_columns.sql index 364c6ab..6670012 100644 --- a/macros/staging/get_profile_columns.sql +++ b/macros/staging/get_profile_columns.sql @@ -1,6 +1,6 @@ {% macro get_profile_columns() %} -{% set _fivetran_columns = [ +{% set columns = [ {"name": "id", "datatype": dbt.type_int()}, {"name": "account_id", "datatype": dbt.type_string()}, {"name": "account_marketplace_string_id", "datatype": dbt.type_string()}, @@ -15,8 +15,6 @@ {"name": "_fivetran_deleted", "datatype": dbt.type_boolean()} ] %} -{% set columns = amazon_ads.resolve_column_names('profile', _fivetran_columns) %} - {{ return(columns) }} {% endmacro %} \ No newline at end of file diff --git a/macros/staging/get_search_term_ad_keyword_report_columns.sql b/macros/staging/get_search_term_ad_keyword_report_columns.sql index f96458c..35900c4 100644 --- a/macros/staging/get_search_term_ad_keyword_report_columns.sql +++ b/macros/staging/get_search_term_ad_keyword_report_columns.sql @@ -1,6 +1,6 @@ {% macro get_search_term_ad_keyword_report_columns() %} -{% set _fivetran_columns = [ +{% set columns = [ {"name": "ad_group_id", "datatype": dbt.type_int()}, {"name": "ad_keyword_status", "datatype": dbt.type_string()}, {"name": "campaign_budget_amount", "datatype": dbt.type_float()}, @@ -19,8 +19,6 @@ {"name": "sales_30_d", "datatype": dbt.type_float()} ] %} -{% set columns = amazon_ads.resolve_column_names('search_term_ad_keyword_report', _fivetran_columns) %} - {# Add backwards compatibility if conversion metrics were added via passthrough columns prior to them being brought in by default #} {{ amazon_ads_add_pass_through_columns(base_columns=columns, pass_through_fields=var('amazon_ads__search_term_ad_keyword_passthrough_metrics'), except_fields=['purchases_30_d', 'sales_30_d']) }} diff --git a/macros/staging/get_targeting_keyword_report_columns.sql b/macros/staging/get_targeting_keyword_report_columns.sql index 31ac3ca..24b557a 100644 --- a/macros/staging/get_targeting_keyword_report_columns.sql +++ b/macros/staging/get_targeting_keyword_report_columns.sql @@ -1,6 +1,6 @@ {% macro get_targeting_keyword_report_columns() %} -{% set _fivetran_columns = [ +{% set columns = [ {"name": "ad_group_id", "datatype": dbt.type_int()}, {"name": "ad_keyword_status", "datatype": dbt.type_string()}, {"name": "campaign_budget_amount", "datatype": dbt.type_float()}, @@ -20,8 +20,6 @@ {"name": "sales_30_d", "datatype": dbt.type_float()} ] %} -{% set columns = amazon_ads.resolve_column_names('targeting_keyword_report', _fivetran_columns) %} - {# Add backwards compatibility if conversion metrics were added via passthrough columns prior to them being brought in by default #} {{ amazon_ads_add_pass_through_columns(base_columns=columns, pass_through_fields=var('amazon_ads__targeting_keyword_passthrough_metrics'), except_fields=['purchases_30_d', 'sales_30_d']) }} diff --git a/macros/staging/resolve_column_names.sql b/macros/staging/resolve_column_names.sql index e48083b..a187c86 100644 --- a/macros/staging/resolve_column_names.sql +++ b/macros/staging/resolve_column_names.sql @@ -1,22 +1,55 @@ -{% macro resolve_column_names(_fivetran_table_name, _fivetran_columns, package_name='amazon_ads') %} - {{ return(adapter.dispatch('resolve_column_names', 'amazon_ads')(_fivetran_table_name, _fivetran_columns, package_name)) }} +{% macro resolve_column_names(_fivetran_table_name, columns, package_name='amazon_ads') %} + {#- + Resolves custom column name mappings for a given table across all configured schemas. + + This macro creates a complete mapping of standard column names to their possible variants + (both custom names and standard names) for use in column standardization. + + Args: + _fivetran_table_name: The base table name (e.g., 'ad_group_level_report') + columns: List of column objects with 'name' and 'datatype' attributes + package_name: Package name for variable lookup (default: 'amazon_ads') + + Returns: + Dict mapping standard names to lists of possible column names. + + Example output for campaign_bidding_strategy with custom mapping: + { + "campaign_bidding_strategy": ["campaignBiddingStrategy", "campaign_bidding_strategy"], + "ad_group_id": ["ad_group_id"], + "clicks": ["clicks"] + } + -#} + {{ return(adapter.dispatch('resolve_column_names', 'amazon_ads')(_fivetran_table_name, columns, package_name)) }} {% endmacro %} -{% macro default__resolve_column_names(_fivetran_table_name, _fivetran_columns, package_name) %} +{% macro default__resolve_column_names(_fivetran_table_name, columns, package_name) %} {% if var(package_name ~ '_using_custom_names', false) %} - {% set resolved_column_names = [] %} + {# Step 1: Get the custom column name configuration from variables #} {% set custom_column_names = var(package_name ~ '_custom_column_names', {}) %} + {% set column_name_mappings = {} %} - {% for column in _fivetran_columns %} - {% set column_name = column.name %} - {# Use the custom name if it exists, otherwise use the original column name #} - {% set resolved_name = custom_column_names.get(_fivetran_table_name, {}).get(column_name, column_name) %} - {% set updated_column = dict(column, name=resolved_name, alias=column_name) %} - {% do resolved_column_names.append(updated_column) %} + {# Step 2: Process each expected column to build complete name mappings #} + {% for column in columns %} + {% set standard_name = column.name %} + {% set custom_names = [] %} + + {# Step 3: Look through all schemas for custom names for this standard field #} + {% for schema_name, custom_cols_for_schema in custom_column_names.items() %} + {% set custom_name = custom_cols_for_schema.get(_fivetran_table_name, {}).get(standard_name, false) %} + {% do custom_names.append(custom_name) if custom_name and custom_name not in custom_names %} + {% endfor %} + + {# Step 4: Create complete name list (custom names first, then standard as fallback) #} + {% set all_names = custom_names + [standard_name] if custom_names else [standard_name] %} + {% do column_name_mappings.update({standard_name: all_names}) %} {% endfor %} -{% endif %} -{{ return(resolved_column_names) }} + {{ return(column_name_mappings) }} +{% else %} + {# Custom names not enabled - return empty dict #} + {{ return({}) }} +{% endif %} {% endmacro %} \ No newline at end of file diff --git a/models/staging/stg_amazon_ads__ad_group_history.sql b/models/staging/stg_amazon_ads__ad_group_history.sql index aa9da60..fb341b1 100644 --- a/models/staging/stg_amazon_ads__ad_group_history.sql +++ b/models/staging/stg_amazon_ads__ad_group_history.sql @@ -1,33 +1,33 @@ {{ config(enabled=var('ad_reporting__amazon_ads_enabled', True)) }} +{% set stg_tmp_relation = ref('stg_amazon_ads__ad_group_history_tmp') %} + with base as ( - select * - from {{ ref('stg_amazon_ads__ad_group_history_tmp') }} + select * + from {{ stg_tmp_relation }} ), fields as ( - select - {{ - fivetran_utils.fill_staging_columns( - source_columns=adapter.get_columns_in_relation(ref('stg_amazon_ads__ad_group_history_tmp')), - staging_columns=get_ad_group_history_columns() - ) - }} - - {{ fivetran_utils.source_relation( - union_schema_variable='amazon_ads_union_schemas', - union_database_variable='amazon_ads_union_databases') +select + {{ amazon_ads.standardize_column_names( + tmp_relation=stg_tmp_relation, + table_name='ad_group_history', + base_columns=get_ad_group_history_columns()) }} - from base + {{ fivetran_utils.source_relation( + union_schema_variable='amazon_ads_union_schemas', + union_database_variable='amazon_ads_union_databases') + }} +from base ), final as ( select - source_relation, + source_relation, cast(id as {{ dbt.type_string() }}) as ad_group_id, cast(campaign_id as {{ dbt.type_string() }}) as campaign_id, creation_date, diff --git a/models/staging/stg_amazon_ads__ad_group_level_report.sql b/models/staging/stg_amazon_ads__ad_group_level_report.sql index e4ce5b2..a3ec067 100644 --- a/models/staging/stg_amazon_ads__ad_group_level_report.sql +++ b/models/staging/stg_amazon_ads__ad_group_level_report.sql @@ -1,32 +1,33 @@ {{ config(enabled=var('ad_reporting__amazon_ads_enabled', True)) }} + +{% set stg_tmp_relation = ref('stg_amazon_ads__ad_group_level_report_tmp') %} + with base as ( select * - from {{ ref('stg_amazon_ads__ad_group_level_report_tmp') }} + from {{ stg_tmp_relation }} ), fields as ( - select - {{ - fivetran_utils.fill_staging_columns( - source_columns=adapter.get_columns_in_relation(ref('stg_amazon_ads__ad_group_level_report_tmp')), - staging_columns=get_ad_group_level_report_columns() - ) - }} - - {{ fivetran_utils.source_relation( - union_schema_variable='amazon_ads_union_schemas', - union_database_variable='amazon_ads_union_databases') +select + {{ amazon_ads.standardize_column_names( + tmp_relation=stg_tmp_relation, + table_name='ad_group_level_report', + base_columns=get_ad_group_level_report_columns()) }} - from base + {{ fivetran_utils.source_relation( + union_schema_variable='amazon_ads_union_schemas', + union_database_variable='amazon_ads_union_databases') + }} +from base ), final as ( select - source_relation, + source_relation, cast(ad_group_id as {{ dbt.type_string() }}) as ad_group_id, campaign_bidding_strategy, clicks, @@ -35,8 +36,8 @@ final as ( impressions, purchases_30_d, sales_30_d - - {{ amazon_ads_fill_pass_through_columns(pass_through_fields=var('amazon_ads__ad_group_passthrough_metrics'), except=['purchases_30_d', 'sales_30_d']) }} + + {{ amazon_ads_fill_pass_through_columns(pass_through_fields=var('amazon_ads__ad_group_passthrough_metrics'), except=['purchases_30_d', 'sales_30_d']) -}} from fields ) diff --git a/models/staging/stg_amazon_ads__advertised_product_report.sql b/models/staging/stg_amazon_ads__advertised_product_report.sql index 3bc8ba2..c64a7e7 100644 --- a/models/staging/stg_amazon_ads__advertised_product_report.sql +++ b/models/staging/stg_amazon_ads__advertised_product_report.sql @@ -1,33 +1,33 @@ {{ config(enabled=var('ad_reporting__amazon_ads_enabled', True)) }} +{% set stg_tmp_relation = ref('stg_amazon_ads__advertised_product_report_tmp') %} + with base as ( - select * - from {{ ref('stg_amazon_ads__advertised_product_report_tmp') }} + select * + from {{ stg_tmp_relation }} ), fields as ( - select - {{ - fivetran_utils.fill_staging_columns( - source_columns=adapter.get_columns_in_relation(ref('stg_amazon_ads__advertised_product_report_tmp')), - staging_columns=get_advertised_product_report_columns() - ) - }} - - {{ fivetran_utils.source_relation( - union_schema_variable='amazon_ads_union_schemas', - union_database_variable='amazon_ads_union_databases') +select + {{ amazon_ads.standardize_column_names( + tmp_relation=stg_tmp_relation, + table_name='advertised_product_report', + base_columns=get_advertised_product_report_columns()) }} - from base + {{ fivetran_utils.source_relation( + union_schema_variable='amazon_ads_union_schemas', + union_database_variable='amazon_ads_union_databases') + }} +from base ), final as ( select - source_relation, + source_relation, cast(ad_id as {{ dbt.type_string() }}) as ad_id, cast(ad_group_id as {{ dbt.type_string() }}) as ad_group_id, advertised_asin, @@ -42,7 +42,7 @@ final as ( impressions, purchases_30_d, sales_30_d - + {{ amazon_ads_fill_pass_through_columns(pass_through_fields=var('amazon_ads__advertised_product_passthrough_metrics'), except=['purchases_30_d', 'sales_30_d']) }} from fields diff --git a/models/staging/stg_amazon_ads__campaign_history.sql b/models/staging/stg_amazon_ads__campaign_history.sql index 97a0e1c..1a96157 100644 --- a/models/staging/stg_amazon_ads__campaign_history.sql +++ b/models/staging/stg_amazon_ads__campaign_history.sql @@ -1,33 +1,33 @@ {{ config(enabled=var('ad_reporting__amazon_ads_enabled', True)) }} +{% set stg_tmp_relation = ref('stg_amazon_ads__campaign_history_tmp') %} + with base as ( - select * - from {{ ref('stg_amazon_ads__campaign_history_tmp') }} + select * + from {{ stg_tmp_relation }} ), fields as ( - select - {{ - fivetran_utils.fill_staging_columns( - source_columns=adapter.get_columns_in_relation(ref('stg_amazon_ads__campaign_history_tmp')), - staging_columns=get_campaign_history_columns() - ) - }} - - {{ fivetran_utils.source_relation( - union_schema_variable='amazon_ads_union_schemas', - union_database_variable='amazon_ads_union_databases') +select + {{ amazon_ads.standardize_column_names( + tmp_relation=stg_tmp_relation, + table_name='campaign_history', + base_columns=get_campaign_history_columns()) }} - from base + {{ fivetran_utils.source_relation( + union_schema_variable='amazon_ads_union_schemas', + union_database_variable='amazon_ads_union_databases') + }} +from base ), final as ( select - source_relation, + source_relation, cast(id as {{ dbt.type_string() }}) as campaign_id, last_updated_date, bidding_strategy, diff --git a/models/staging/stg_amazon_ads__campaign_level_report.sql b/models/staging/stg_amazon_ads__campaign_level_report.sql index cdab605..f96c3e0 100644 --- a/models/staging/stg_amazon_ads__campaign_level_report.sql +++ b/models/staging/stg_amazon_ads__campaign_level_report.sql @@ -1,33 +1,33 @@ {{ config(enabled=var('ad_reporting__amazon_ads_enabled', True)) }} +{% set stg_tmp_relation = ref('stg_amazon_ads__campaign_level_report_tmp') %} + with base as ( - select * - from {{ ref('stg_amazon_ads__campaign_level_report_tmp') }} + select * + from {{ stg_tmp_relation }} ), fields as ( - select - {{ - fivetran_utils.fill_staging_columns( - source_columns=adapter.get_columns_in_relation(ref('stg_amazon_ads__campaign_level_report_tmp')), - staging_columns=get_campaign_level_report_columns() - ) - }} - - {{ fivetran_utils.source_relation( - union_schema_variable='amazon_ads_union_schemas', - union_database_variable='amazon_ads_union_databases') +select + {{ amazon_ads.standardize_column_names( + tmp_relation=stg_tmp_relation, + table_name='campaign_level_report', + base_columns=get_campaign_level_report_columns()) }} - from base + {{ fivetran_utils.source_relation( + union_schema_variable='amazon_ads_union_schemas', + union_database_variable='amazon_ads_union_databases') + }} +from base ), final as ( select - source_relation, + source_relation, campaign_applicable_budget_rule_id, campaign_applicable_budget_rule_name, campaign_bidding_strategy, diff --git a/models/staging/stg_amazon_ads__keyword_history.sql b/models/staging/stg_amazon_ads__keyword_history.sql index 6287e7f..2dac744 100644 --- a/models/staging/stg_amazon_ads__keyword_history.sql +++ b/models/staging/stg_amazon_ads__keyword_history.sql @@ -1,33 +1,33 @@ {{ config(enabled=var('ad_reporting__amazon_ads_enabled', True)) }} +{% set stg_tmp_relation = ref('stg_amazon_ads__keyword_history_tmp') %} + with base as ( - select * - from {{ ref('stg_amazon_ads__keyword_history_tmp') }} + select * + from {{ stg_tmp_relation }} ), fields as ( - select - {{ - fivetran_utils.fill_staging_columns( - source_columns=adapter.get_columns_in_relation(ref('stg_amazon_ads__keyword_history_tmp')), - staging_columns=get_keyword_history_columns() - ) - }} - - {{ fivetran_utils.source_relation( - union_schema_variable='amazon_ads_union_schemas', - union_database_variable='amazon_ads_union_databases') +select + {{ amazon_ads.standardize_column_names( + tmp_relation=stg_tmp_relation, + table_name='keyword_history', + base_columns=get_keyword_history_columns()) }} - from base + {{ fivetran_utils.source_relation( + union_schema_variable='amazon_ads_union_schemas', + union_database_variable='amazon_ads_union_databases') + }} +from base ), final as ( select - source_relation, + source_relation, cast(id as {{ dbt.type_string() }}) as keyword_id, cast(ad_group_id as {{ dbt.type_string() }}) as ad_group_id, bid, diff --git a/models/staging/stg_amazon_ads__portfolio_history.sql b/models/staging/stg_amazon_ads__portfolio_history.sql index 6f242ac..ebdccc5 100644 --- a/models/staging/stg_amazon_ads__portfolio_history.sql +++ b/models/staging/stg_amazon_ads__portfolio_history.sql @@ -1,33 +1,33 @@ {{ config(enabled=fivetran_utils.enabled_vars(['ad_reporting__amazon_ads_enabled','amazon_ads__portfolio_history_enabled'])) }} +{% set stg_tmp_relation = ref('stg_amazon_ads__portfolio_history_tmp') %} + with base as ( - select * - from {{ ref('stg_amazon_ads__portfolio_history_tmp') }} + select * + from {{ stg_tmp_relation }} ), fields as ( - select - {{ - fivetran_utils.fill_staging_columns( - source_columns=adapter.get_columns_in_relation(ref('stg_amazon_ads__portfolio_history_tmp')), - staging_columns=get_portfolio_history_columns() - ) - }} - - {{ fivetran_utils.source_relation( - union_schema_variable='amazon_ads_union_schemas', - union_database_variable='amazon_ads_union_databases') +select + {{ amazon_ads.standardize_column_names( + tmp_relation=stg_tmp_relation, + table_name='portfolio_history', + base_columns=get_portfolio_history_columns()) }} - from base + {{ fivetran_utils.source_relation( + union_schema_variable='amazon_ads_union_schemas', + union_database_variable='amazon_ads_union_databases') + }} +from base ), final as ( select - source_relation, + source_relation, cast(id as {{ dbt.type_string() }}) as portfolio_id, budget_amount, budget_currency_code, diff --git a/models/staging/stg_amazon_ads__product_ad_history.sql b/models/staging/stg_amazon_ads__product_ad_history.sql index 25568dc..d108d49 100644 --- a/models/staging/stg_amazon_ads__product_ad_history.sql +++ b/models/staging/stg_amazon_ads__product_ad_history.sql @@ -1,33 +1,33 @@ {{ config(enabled=var('ad_reporting__amazon_ads_enabled', True)) }} +{% set stg_tmp_relation = ref('stg_amazon_ads__product_ad_history_tmp') %} + with base as ( - select * - from {{ ref('stg_amazon_ads__product_ad_history_tmp') }} + select * + from {{ stg_tmp_relation }} ), fields as ( - select - {{ - fivetran_utils.fill_staging_columns( - source_columns=adapter.get_columns_in_relation(ref('stg_amazon_ads__product_ad_history_tmp')), - staging_columns=get_product_ad_history_columns() - ) - }} - - {{ fivetran_utils.source_relation( - union_schema_variable='amazon_ads_union_schemas', - union_database_variable='amazon_ads_union_databases') +select + {{ amazon_ads.standardize_column_names( + tmp_relation=stg_tmp_relation, + table_name='product_ad_history', + base_columns=get_product_ad_history_columns()) }} - from base + {{ fivetran_utils.source_relation( + union_schema_variable='amazon_ads_union_schemas', + union_database_variable='amazon_ads_union_databases') + }} +from base ), final as ( select - source_relation, + source_relation, cast(id as {{ dbt.type_string() }}) as ad_id, cast(ad_group_id as {{ dbt.type_string() }}) as ad_group_id, asin, diff --git a/models/staging/stg_amazon_ads__profile.sql b/models/staging/stg_amazon_ads__profile.sql index 99d0cd5..00db9b8 100644 --- a/models/staging/stg_amazon_ads__profile.sql +++ b/models/staging/stg_amazon_ads__profile.sql @@ -1,33 +1,33 @@ {{ config(enabled=var('ad_reporting__amazon_ads_enabled', True)) }} +{% set stg_tmp_relation = ref('stg_amazon_ads__profile_tmp') %} + with base as ( - select * - from {{ ref('stg_amazon_ads__profile_tmp') }} + select * + from {{ stg_tmp_relation }} ), fields as ( - select - {{ - fivetran_utils.fill_staging_columns( - source_columns=adapter.get_columns_in_relation(ref('stg_amazon_ads__profile_tmp')), - staging_columns=get_profile_columns() - ) - }} - - {{ fivetran_utils.source_relation( - union_schema_variable='amazon_ads_union_schemas', - union_database_variable='amazon_ads_union_databases') +select + {{ amazon_ads.standardize_column_names( + tmp_relation=stg_tmp_relation, + table_name='profile', + base_columns=get_profile_columns()) }} - from base + {{ fivetran_utils.source_relation( + union_schema_variable='amazon_ads_union_schemas', + union_database_variable='amazon_ads_union_databases') + }} +from base ), final as ( select - source_relation, + source_relation, cast(id as {{ dbt.type_string() }}) as profile_id, cast(account_id as {{ dbt.type_string() }}) as account_id, account_marketplace_string_id, diff --git a/models/staging/stg_amazon_ads__search_term_ad_keyword_report.sql b/models/staging/stg_amazon_ads__search_term_ad_keyword_report.sql index 692cc68..9a15167 100644 --- a/models/staging/stg_amazon_ads__search_term_ad_keyword_report.sql +++ b/models/staging/stg_amazon_ads__search_term_ad_keyword_report.sql @@ -1,33 +1,33 @@ {{ config(enabled=var('ad_reporting__amazon_ads_enabled', True)) }} +{% set stg_tmp_relation = ref('stg_amazon_ads__search_term_ad_keyword_report_tmp') %} + with base as ( - select * - from {{ ref('stg_amazon_ads__search_term_ad_keyword_report_tmp') }} + select * + from {{ stg_tmp_relation }} ), fields as ( - select - {{ - fivetran_utils.fill_staging_columns( - source_columns=adapter.get_columns_in_relation(ref('stg_amazon_ads__search_term_ad_keyword_report_tmp')), - staging_columns=get_search_term_ad_keyword_report_columns() - ) - }} - - {{ fivetran_utils.source_relation( - union_schema_variable='amazon_ads_union_schemas', - union_database_variable='amazon_ads_union_databases') +select + {{ amazon_ads.standardize_column_names( + tmp_relation=stg_tmp_relation, + table_name='search_term_ad_keyword_report', + base_columns=get_search_term_ad_keyword_report_columns()) }} - from base + {{ fivetran_utils.source_relation( + union_schema_variable='amazon_ads_union_schemas', + union_database_variable='amazon_ads_union_databases') + }} +from base ), final as ( select - source_relation, + source_relation, cast(ad_group_id as {{ dbt.type_string() }}) as ad_group_id, ad_keyword_status, campaign_budget_amount, diff --git a/models/staging/stg_amazon_ads__targeting_keyword_report.sql b/models/staging/stg_amazon_ads__targeting_keyword_report.sql index 89c2adc..8dd8392 100644 --- a/models/staging/stg_amazon_ads__targeting_keyword_report.sql +++ b/models/staging/stg_amazon_ads__targeting_keyword_report.sql @@ -1,33 +1,33 @@ {{ config(enabled=var('ad_reporting__amazon_ads_enabled', True)) }} +{% set stg_tmp_relation = ref('stg_amazon_ads__targeting_keyword_report_tmp') %} + with base as ( - select * - from {{ ref('stg_amazon_ads__targeting_keyword_report_tmp') }} + select * + from {{ stg_tmp_relation }} ), fields as ( - select - {{ - fivetran_utils.fill_staging_columns( - source_columns=adapter.get_columns_in_relation(ref('stg_amazon_ads__targeting_keyword_report_tmp')), - staging_columns=get_targeting_keyword_report_columns() - ) - }} - - {{ fivetran_utils.source_relation( - union_schema_variable='amazon_ads_union_schemas', - union_database_variable='amazon_ads_union_databases') +select + {{ amazon_ads.standardize_column_names( + tmp_relation=stg_tmp_relation, + table_name='targeting_keyword_report', + base_columns=get_targeting_keyword_report_columns()) }} - from base + {{ fivetran_utils.source_relation( + union_schema_variable='amazon_ads_union_schemas', + union_database_variable='amazon_ads_union_databases') + }} +from base ), final as ( select - source_relation, + source_relation, cast(ad_group_id as {{ dbt.type_string() }}) as ad_group_id, ad_keyword_status, campaign_budget_amount, From be36b357efed5456456fcd148a871fb49ae0845c Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Thu, 19 Mar 2026 17:39:28 -0500 Subject: [PATCH 12/20] add macro --- macros/staging/standardize_column_names.sql | 69 +++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 macros/staging/standardize_column_names.sql diff --git a/macros/staging/standardize_column_names.sql b/macros/staging/standardize_column_names.sql new file mode 100644 index 0000000..6bf03b6 --- /dev/null +++ b/macros/staging/standardize_column_names.sql @@ -0,0 +1,69 @@ +{% macro standardize_column_names(tmp_relation, table_name, base_columns, package_name='amazon_ads') %} + {#- + Standardizes column names from source tables with mixed naming conventions. + + This macro handles both column name standardization (coalescing custom names to + standard names) and missing column filling (creating NULLs for expected columns + that don't exist in the source). + + Args: + tmp_relation: The source relation (typically from tmp staging model) + table_name: The base table name for custom name lookups + base_columns: List of expected column objects with 'name' and 'datatype' + package_name: Package name for variable lookup (default: 'amazon_ads') + + Returns: + SQL select statement with standardized column names. + + Example output: + select + coalesce("campaignBiddingStrategy", "campaign_bidding_strategy") as campaign_bidding_strategy, + "ad_group_id" as ad_group_id, + cast(null as integer) as missing_column + from tmp_relation + -#} + {{ return(adapter.dispatch('standardize_column_names', 'amazon_ads')(tmp_relation, table_name, base_columns, package_name)) }} +{% endmacro %} + +{% macro default__standardize_column_names(tmp_relation, table_name, base_columns, package_name) %} + +{# Step 1: Inspect the source relation to see what columns actually exist #} +{%- set actual_columns = adapter.get_columns_in_relation(tmp_relation) %} +{%- set actual_column_names = actual_columns | map(attribute='name') | list %} + +{# Step 2: Get custom name mappings for this table if custom names are enabled #} +{%- set custom_mappings = amazon_ads.resolve_column_names(table_name, base_columns, package_name) if var(package_name ~ '_using_custom_names', false) else {} %} + +select +{# Step 3: Process each expected column to generate standardized select statements #} +{%- for column in base_columns %} + {%- set standard_name = column.name %} + {%- set column_datatype = column.datatype %} + + {%- if standard_name in custom_mappings -%} + {# Step 3a: Column has custom mappings - check which variants actually exist #} + {%- set available_variants = [] -%} + {%- for variant in custom_mappings[standard_name] if variant in actual_column_names -%} + {%- do available_variants.append(variant) -%} + {%- endfor -%} + {%- if available_variants|length > 1 -%} + coalesce({% for variant in available_variants %}{{ adapter.quote(variant) }}{{ ', ' if not loop.last }}{% endfor %}) + {%- elif available_variants|length == 1 -%} + {{ adapter.quote(available_variants[0]) }} + {%- else -%} + cast(null as {{ column_datatype }}) + {%- endif %} + {%- else -%} + {# Step 3b: Column has no custom mappings - check if standard name exists #} + {%- if standard_name in actual_column_names -%} + {{ adapter.quote(standard_name) }} + {%- else -%} + cast(null as {{ column_datatype }}) + {%- endif %} + {% endif %} + as {{ standard_name }}{{ ',' if not loop.last }} + +{% endfor %} +from {{ tmp_relation }} + +{% endmacro %} \ No newline at end of file From dddf8387ca3c89840bee9f555cd1354f7aa4c10c Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Thu, 19 Mar 2026 17:41:30 -0500 Subject: [PATCH 13/20] add fix yml --- integration_tests/dbt_project.yml | 45 +++++++++---------------------- 1 file changed, 12 insertions(+), 33 deletions(-) diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index cbaa034..53806ee 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -7,39 +7,18 @@ profile: 'integration_tests' vars: amazon_ads: - amazon_ads_union_schemas: ['zz_dbt_catherine_3', 'zz_dbt_catherine_4'] - # amazon_ads_sources: - # - database: postgres - # schema: zz_dbt_catherine_3 # Required - # - database: postgres - # schema: zz_dbt_catherine_4 - # amazon_ads_ad_group_history_identifier: "ad_group_history_data" - # amazon_ads_ad_group_level_report_identifier: "ad_group_level_report_data" - # amazon_ads_advertised_product_report_identifier: "advertised_product_report_data" - # amazon_ads_campaign_history_identifier: "campaign_history_data" - # amazon_ads_campaign_level_report_identifier: "campaign_level_report_data" - # amazon_ads_portfolio_history_identifier: "portfolio_history_data" - # amazon_ads_product_ad_history_identifier: "product_ad_history_data" - # amazon_ads_profile_identifier: "profile_data" - # amazon_ads_keyword_history_identifier: "keyword_history_data" - # amazon_ads_targeting_keyword_report_identifier: "targeting_keyword_report_data" - # amazon_ads_search_term_ad_keyword_report_identifier: "search_term_ad_keyword_report_data" - amazon_ads_using_custom_names: true - - amazon_ads_custom_column_names: - zz_dbt_catherine_3: - ad_group_history: - creation_date: creationDate - default_bid: defaultBid - zz_dbt_catherine_4: - ad_group_level_report: - campaign_bidding_strategy: campaignBiddingStrategy - - amazon_ads__campaign_passthrough_metrics: - - name: sales_7_d - - name: purchases_30_d - alias: purchases_alias - - name: purchases_14_d + amazon_ads_schema: amazon_ads_integration_tests_03 + amazon_ads_ad_group_history_identifier: "ad_group_history_data" + amazon_ads_ad_group_level_report_identifier: "ad_group_level_report_data" + amazon_ads_advertised_product_report_identifier: "advertised_product_report_data" + amazon_ads_campaign_history_identifier: "campaign_history_data" + amazon_ads_campaign_level_report_identifier: "campaign_level_report_data" + amazon_ads_portfolio_history_identifier: "portfolio_history_data" + amazon_ads_product_ad_history_identifier: "product_ad_history_data" + amazon_ads_profile_identifier: "profile_data" + amazon_ads_keyword_history_identifier: "keyword_history_data" + amazon_ads_targeting_keyword_report_identifier: "targeting_keyword_report_data" + amazon_ads_search_term_ad_keyword_report_identifier: "search_term_ad_keyword_report_data" models: +persist_docs: From 270883f2cf59d389d38c16dbdbf5d52fc5c1df2e Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Thu, 19 Mar 2026 17:47:17 -0500 Subject: [PATCH 14/20] fix yml --- integration_tests/dbt_project.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 53806ee..56e139f 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -19,6 +19,21 @@ vars: amazon_ads_keyword_history_identifier: "keyword_history_data" amazon_ads_targeting_keyword_report_identifier: "targeting_keyword_report_data" amazon_ads_search_term_ad_keyword_report_identifier: "search_term_ad_keyword_report_data" + amazon_ads__using_custom_names: true + + amazon_ads_custom_column_names: + amazon_ads_integration_tests_03: + ad_group_history: + creation_date: creationDate + default_bid: defaultBid + ad_group_level_report: + campaign_bidding_strategy: campaignBiddingStrategy + + amazon_ads__campaign_passthrough_metrics: + - name: sales_7_d + - name: purchases_30_d + alias: purchases_alias + - name: purchases_14_d models: +persist_docs: From 2d5ca1f552ece02c9dc5fa87b5ff2c0752b1ec92 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Thu, 19 Mar 2026 17:47:40 -0500 Subject: [PATCH 15/20] fix yml --- integration_tests/dbt_project.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 56e139f..2c69087 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -19,8 +19,8 @@ vars: amazon_ads_keyword_history_identifier: "keyword_history_data" amazon_ads_targeting_keyword_report_identifier: "targeting_keyword_report_data" amazon_ads_search_term_ad_keyword_report_identifier: "search_term_ad_keyword_report_data" - amazon_ads__using_custom_names: true + amazon_ads__using_custom_names: true amazon_ads_custom_column_names: amazon_ads_integration_tests_03: ad_group_history: From db927b7b8c8f48d2849c203d1708033f5d704937 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Thu, 19 Mar 2026 18:34:08 -0500 Subject: [PATCH 16/20] adjust select placement --- macros/staging/standardize_column_names.sql | 2 -- 1 file changed, 2 deletions(-) diff --git a/macros/staging/standardize_column_names.sql b/macros/staging/standardize_column_names.sql index 6bf03b6..06fe13d 100644 --- a/macros/staging/standardize_column_names.sql +++ b/macros/staging/standardize_column_names.sql @@ -34,7 +34,6 @@ {# Step 2: Get custom name mappings for this table if custom names are enabled #} {%- set custom_mappings = amazon_ads.resolve_column_names(table_name, base_columns, package_name) if var(package_name ~ '_using_custom_names', false) else {} %} -select {# Step 3: Process each expected column to generate standardized select statements #} {%- for column in base_columns %} {%- set standard_name = column.name %} @@ -64,6 +63,5 @@ select as {{ standard_name }}{{ ',' if not loop.last }} {% endfor %} -from {{ tmp_relation }} {% endmacro %} \ No newline at end of file From 6e256ed8531f966ba5d7e264b3e0dbf831627ec6 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Thu, 19 Mar 2026 18:39:38 -0500 Subject: [PATCH 17/20] considering aliases --- macros/staging/standardize_column_names.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/macros/staging/standardize_column_names.sql b/macros/staging/standardize_column_names.sql index 06fe13d..a448dcc 100644 --- a/macros/staging/standardize_column_names.sql +++ b/macros/staging/standardize_column_names.sql @@ -38,6 +38,7 @@ {%- for column in base_columns %} {%- set standard_name = column.name %} {%- set column_datatype = column.datatype %} + {%- set column_alias = column.alias if column.alias else standard_name %} {%- if standard_name in custom_mappings -%} {# Step 3a: Column has custom mappings - check which variants actually exist #} @@ -60,7 +61,7 @@ cast(null as {{ column_datatype }}) {%- endif %} {% endif %} - as {{ standard_name }}{{ ',' if not loop.last }} + as {{ column_alias }}{{ ',' if not loop.last }} {% endfor %} From 244ced20b32fc571d381918dd4b91ed0e8766e48 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Thu, 19 Mar 2026 19:33:27 -0500 Subject: [PATCH 18/20] merge --- integration_tests/dbt_project.yml | 2 +- macros/staging/standardize_column_names.sql | 15 +++++++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 2c69087..2d8c166 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -20,7 +20,7 @@ vars: amazon_ads_targeting_keyword_report_identifier: "targeting_keyword_report_data" amazon_ads_search_term_ad_keyword_report_identifier: "search_term_ad_keyword_report_data" - amazon_ads__using_custom_names: true + amazon_ads_using_custom_names: true amazon_ads_custom_column_names: amazon_ads_integration_tests_03: ad_group_history: diff --git a/macros/staging/standardize_column_names.sql b/macros/staging/standardize_column_names.sql index a448dcc..d998f3c 100644 --- a/macros/staging/standardize_column_names.sql +++ b/macros/staging/standardize_column_names.sql @@ -43,24 +43,27 @@ {%- if standard_name in custom_mappings -%} {# Step 3a: Column has custom mappings - check which variants actually exist #} {%- set available_variants = [] -%} - {%- for variant in custom_mappings[standard_name] if variant in actual_column_names -%} + {%- for variant in custom_mappings[standard_name] if variant|lower in actual_column_names|map('lower') -%} {%- do available_variants.append(variant) -%} {%- endfor -%} + {%- if available_variants|length > 1 -%} - coalesce({% for variant in available_variants %}{{ adapter.quote(variant) }}{{ ', ' if not loop.last }}{% endfor %}) + coalesce({{ available_variants | join(', ') }}) {%- elif available_variants|length == 1 -%} - {{ adapter.quote(available_variants[0]) }} + {{ available_variants[0] }} {%- else -%} - cast(null as {{ column_datatype }}) + cast(null as {{ column_datatype }}) {%- endif %} + {%- else -%} {# Step 3b: Column has no custom mappings - check if standard name exists #} {%- if standard_name in actual_column_names -%} - {{ adapter.quote(standard_name) }} + {{ standard_name }} {%- else -%} - cast(null as {{ column_datatype }}) + cast(null as {{ column_datatype }}) {%- endif %} {% endif %} + as {{ column_alias }}{{ ',' if not loop.last }} {% endfor %} From badae608c18b15ea393a26961142d6b60d546617 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Mon, 4 May 2026 16:03:02 -0500 Subject: [PATCH 19/20] testing --- integration_tests/dbt_project.yml | 10 ++++++++-- ...{campaign_history_data.csv => campaign_history.csv} | 2 +- macros/staging/get_campaign_history_columns.sql | 2 +- models/staging/stg_amazon_ads__campaign_history.sql | 2 +- 4 files changed, 11 insertions(+), 5 deletions(-) rename integration_tests/seeds/{campaign_history_data.csv => campaign_history.csv} (88%) diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 2d8c166..c54fc27 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -7,11 +7,11 @@ profile: 'integration_tests' vars: amazon_ads: - amazon_ads_schema: amazon_ads_integration_tests_03 + # amazon_ads_schema: zz_dbt_catherine #amazon_ads_integration_tests_03 amazon_ads_ad_group_history_identifier: "ad_group_history_data" amazon_ads_ad_group_level_report_identifier: "ad_group_level_report_data" amazon_ads_advertised_product_report_identifier: "advertised_product_report_data" - amazon_ads_campaign_history_identifier: "campaign_history_data" + amazon_ads_campaign_history_identifier: "campaign_history" amazon_ads_campaign_level_report_identifier: "campaign_level_report_data" amazon_ads_portfolio_history_identifier: "portfolio_history_data" amazon_ads_product_ad_history_identifier: "product_ad_history_data" @@ -19,6 +19,7 @@ vars: amazon_ads_keyword_history_identifier: "keyword_history_data" amazon_ads_targeting_keyword_report_identifier: "targeting_keyword_report_data" amazon_ads_search_term_ad_keyword_report_identifier: "search_term_ad_keyword_report_data" + amazon_ads_union_schemas: ['zz_dbt_catherine', 'zz_dbt_catherine_02'] amazon_ads_using_custom_names: true amazon_ads_custom_column_names: @@ -28,6 +29,11 @@ vars: default_bid: defaultBid ad_group_level_report: campaign_bidding_strategy: campaignBiddingStrategy + campaign_history: + name: campaignName + amazon_ads_integration_tests_04: + campaign_history: + name: campaignName02 amazon_ads__campaign_passthrough_metrics: - name: sales_7_d diff --git a/integration_tests/seeds/campaign_history_data.csv b/integration_tests/seeds/campaign_history.csv similarity index 88% rename from integration_tests/seeds/campaign_history_data.csv rename to integration_tests/seeds/campaign_history.csv index b80ae66..e2ebddd 100644 --- a/integration_tests/seeds/campaign_history_data.csv +++ b/integration_tests/seeds/campaign_history.csv @@ -1,4 +1,4 @@ -id,last_updated_date,_fivetran_synced,bidding_strategy,creation_date,budget,end_date,name,portfolio_id,profile_id,serving_status,start_date,state,targeting_type,budget_type,effective_budget +id,last_updated_date,_fivetran_synced,bidding_strategy,creation_date,budget,end_date,campaignName02,portfolio_id,profile_id,serving_status,start_date,state,targeting_type,budget_type,effective_budget 2187,2022-08-25 15:36:51.687000,2022-12-13 17:09:00.337000,autoForSales,2022-07-11 22:38:16.327000,2000,,Gold Leader,1138,66,CAMPAIGN_PAUSED,2022-07-11,paused,manual,daily, 2187,2022-07-14 11:58:58.857000,2022-08-09 11:09:43.874000,autoForSales,2022-07-11 22:38:16.327000,2000,,Gold Leader,1138,66,CAMPAIGN_PAUSED,2022-07-11,paused,manual,daily, 2187,2022-07-11 22:38:16.327000,2022-07-12 11:07:55.188000,autoForSales,2022-07-11 22:38:16.327000,300,,Red Leader,1138,66,CAMPAIGN_STATUS_ENABLED,2022-07-11,enabled,manual,daily, diff --git a/macros/staging/get_campaign_history_columns.sql b/macros/staging/get_campaign_history_columns.sql index 52215ab..4639139 100644 --- a/macros/staging/get_campaign_history_columns.sql +++ b/macros/staging/get_campaign_history_columns.sql @@ -6,7 +6,7 @@ {"name": "end_date", "datatype": "date"}, {"name": "id", "datatype": dbt.type_string()}, {"name": "last_updated_date", "datatype": dbt.type_timestamp()}, - {"name": "name", "datatype": dbt.type_string()}, + {"name": "name", "datatype": dbt.type_string(), "alias": "campaign_name"}, {"name": "portfolio_id", "datatype": dbt.type_int()}, {"name": "profile_id", "datatype": dbt.type_int()}, {"name": "serving_status", "datatype": dbt.type_string()}, diff --git a/models/staging/stg_amazon_ads__campaign_history.sql b/models/staging/stg_amazon_ads__campaign_history.sql index 1a96157..40e7da7 100644 --- a/models/staging/stg_amazon_ads__campaign_history.sql +++ b/models/staging/stg_amazon_ads__campaign_history.sql @@ -33,7 +33,7 @@ final as ( bidding_strategy, creation_date, end_date, - name as campaign_name, + campaign_name, -- used alias in get_campaign_history_columns macro to test the alias feature cast(portfolio_id as {{ dbt.type_string() }}) as portfolio_id, cast(profile_id as {{ dbt.type_string() }}) as profile_id, serving_status, From c1db544b592f568743685a995c71df795709a20a Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Mon, 4 May 2026 16:03:26 -0500 Subject: [PATCH 20/20] testing --- integration_tests/dbt_project.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index c54fc27..b243632 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -7,7 +7,7 @@ profile: 'integration_tests' vars: amazon_ads: - # amazon_ads_schema: zz_dbt_catherine #amazon_ads_integration_tests_03 + amazon_ads_schema: amazon_ads_integration_tests_03 amazon_ads_ad_group_history_identifier: "ad_group_history_data" amazon_ads_ad_group_level_report_identifier: "ad_group_level_report_data" amazon_ads_advertised_product_report_identifier: "advertised_product_report_data" @@ -19,7 +19,7 @@ vars: amazon_ads_keyword_history_identifier: "keyword_history_data" amazon_ads_targeting_keyword_report_identifier: "targeting_keyword_report_data" amazon_ads_search_term_ad_keyword_report_identifier: "search_term_ad_keyword_report_data" - amazon_ads_union_schemas: ['zz_dbt_catherine', 'zz_dbt_catherine_02'] + # amazon_ads_union_schemas: ['zz_dbt_catherine', 'zz_dbt_catherine_02'] amazon_ads_using_custom_names: true amazon_ads_custom_column_names: