Buildkite improvements FY27 Q2

.buildkite/hooks/pre-command

@@ -1,25 +1,3 @@
 #!/bin/bash
-
-set -e
-
-# Export secrets for Docker containers.
-# Restrict exposing secrets only to the steps that need them
-export GCLOUD_SERVICE_KEY=$(gcloud secrets versions access latest --secret="GCLOUD_SERVICE_KEY" --project="dbt-package-testing-363917")
-export CI_POSTGRES_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_HOST" --project="dbt-package-testing-363917")
-export CI_POSTGRES_DBT_USER=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_USER" --project="dbt-package-testing-363917")
-export CI_POSTGRES_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_PASS" --project="dbt-package-testing-363917")
-export CI_POSTGRES_DBT_DBNAME=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_DBNAME" --project="dbt-package-testing-363917")
-export CI_REDSHIFT_DBT_DBNAME=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_DBNAME" --project="dbt-package-testing-363917")
-export CI_REDSHIFT_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_HOST" --project="dbt-package-testing-363917")
-export CI_REDSHIFT_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_PASS" --project="dbt-package-testing-363917")
-export CI_REDSHIFT_DBT_USER=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_USER" --project="dbt-package-testing-363917")
-export CI_SNOWFLAKE_DBT_ACCOUNT=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_ACCOUNT" --project="dbt-package-testing-363917")
-export CI_SNOWFLAKE_DBT_DATABASE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_DATABASE" --project="dbt-package-testing-363917")
-export CI_SNOWFLAKE_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_PASS" --project="dbt-package-testing-363917")
-export CI_SNOWFLAKE_DBT_ROLE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_ROLE" --project="dbt-package-testing-363917")
-export CI_SNOWFLAKE_DBT_USER=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_USER" --project="dbt-package-testing-363917")
-export CI_SNOWFLAKE_DBT_WAREHOUSE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_WAREHOUSE" --project="dbt-package-testing-363917")
-export CI_DATABRICKS_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HOST" --project="dbt-package-testing-363917")
-export CI_DATABRICKS_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HTTP_PATH" --project="dbt-package-testing-363917")
-export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917")
-export CI_DATABRICKS_DBT_CATALOG=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_CATALOG" --project="dbt-package-testing-363917")
+curl -s -o /tmp/build_setup.sh "https://raw.githubusercontent.com/fivetran/dbt_package_automations/refs/heads/feature/buildkite-scripts/.buildkite/scripts/build_setup.sh"
+source /tmp/build_setup.sh

.buildkite/pipeline.yml

@@ -1,74 +1,104 @@
+env:
+  RUN_MODELS_URL: "https://raw.githubusercontent.com/fivetran/dbt_package_automations/refs/heads/feature/buildkite-scripts/.buildkite/scripts/run_models.sh"
+
 steps:
   - label: ":postgres: Run Tests - Postgres"
-    key: "run-dbt-postgres"
+    key: "run_dbt_postgres"
+    retry:
+      automatic:
+        - exit_status: -1  # Timeout
+          limit: 1
     plugins:
       - docker#v3.13.0:
-          image: "python:3.10.13"
+          image: "python:3.13"
           shell: [ "/bin/bash", "-e", "-c" ]
           environment:
             - "BASH_ENV=/tmp/.bashrc"
+            - "BUILD_SCHEMA"
             - "CI_POSTGRES_DBT_DBNAME"
             - "CI_POSTGRES_DBT_HOST"
             - "CI_POSTGRES_DBT_PASS"
             - "CI_POSTGRES_DBT_USER"
     commands: |
-      bash .buildkite/scripts/run_models.sh postgres 
+      curl -s "${RUN_MODELS_URL}" | bash -s postgres 
 
   - label: ":snowflake-db: Run Tests - Snowflake"
     key: "run_dbt_snowflake"
+    retry:
+      automatic:
+        - exit_status: -1  # Timeout
+          limit: 1
     plugins:
       - docker#v3.13.0:
-          image: "python:3.10.13"
+          image: "python:3.13"
           shell: [ "/bin/bash", "-e", "-c" ]
           environment:
             - "BASH_ENV=/tmp/.bashrc"
+            - "BUILD_SCHEMA"
             - "CI_SNOWFLAKE_DBT_ACCOUNT"
             - "CI_SNOWFLAKE_DBT_DATABASE"
             - "CI_SNOWFLAKE_DBT_PASS"
             - "CI_SNOWFLAKE_DBT_ROLE"
             - "CI_SNOWFLAKE_DBT_USER"
             - "CI_SNOWFLAKE_DBT_WAREHOUSE"
     commands: |
-      bash .buildkite/scripts/run_models.sh snowflake
+      curl -s "${RUN_MODELS_URL}" | bash -s snowflake
 
   - label: ":gcloud: Run Tests - BigQuery"
     key: "run_dbt_bigquery"
+    retry:
+      automatic:
+        - exit_status: -1  # Timeout
+          limit: 1
     plugins:
       - docker#v3.13.0:
-          image: "python:3.10.13"
+          image: "python:3.13"
           shell: [ "/bin/bash", "-e", "-c" ]
           environment:
             - "BASH_ENV=/tmp/.bashrc"
+            - "BUILD_SCHEMA"
             - "GCLOUD_SERVICE_KEY"
     commands: |
-      bash .buildkite/scripts/run_models.sh bigquery
+      curl -s "${RUN_MODELS_URL}" | bash -s bigquery
 
   - label: ":amazon-redshift: Run Tests - Redshift"
     key: "run_dbt_redshift"
+    concurrency: 3
+    concurrency_group: "warehouse/redshift"
+    retry:
+      automatic:
+        - exit_status: -1  # Timeout
+          limit: 1
     plugins:
       - docker#v3.13.0:
-          image: "python:3.10.13"
+          image: "python:3.13"
           shell: [ "/bin/bash", "-e", "-c" ]
           environment:
             - "BASH_ENV=/tmp/.bashrc"
+            - "BUILD_SCHEMA"
             - "CI_REDSHIFT_DBT_DBNAME"
             - "CI_REDSHIFT_DBT_HOST"
             - "CI_REDSHIFT_DBT_PASS"
             - "CI_REDSHIFT_DBT_USER"
     commands: |
-      bash .buildkite/scripts/run_models.sh redshift
+      curl -s "${RUN_MODELS_URL}" | bash -s redshift
 
   - label: ":databricks: Run Tests - Databricks"
     key: "run_dbt_databricks"
+    retry:
+      automatic:
+        - exit_status: -1  # Timeout
+          limit: 1
     plugins:
       - docker#v3.13.0:
-          image: "python:3.10.13"
+          image: "python:3.13"
           shell: [ "/bin/bash", "-e", "-c" ]
           environment:
             - "BASH_ENV=/tmp/.bashrc"
+            - "BUILD_SCHEMA"
             - "CI_DATABRICKS_DBT_HOST"
             - "CI_DATABRICKS_DBT_HTTP_PATH"
             - "CI_DATABRICKS_DBT_TOKEN"
             - "CI_DATABRICKS_DBT_CATALOG"
     commands: |
-      bash .buildkite/scripts/run_models.sh databricks
+      curl -s "${RUN_MODELS_URL}" | bash -s databricks

integration_tests/ci/test_scenarios.yml

@@ -0,0 +1,8 @@
+schema_variable_name: "amazon_ads_schema"
+default_include_incremental: false
+
+test_scenarios:
+  - name: "portfolio history disabled"
+    vars:
+      amazon_ads__portfolio_history_enabled: false
+    include_incremental: false

integration_tests/dbt_project.yml

@@ -1,54 +1,74 @@
 config-version: 2
-
-name: 'amazon_ads_integration_tests'
-version: '1.2.0'
-
-profile: 'integration_tests' 
-
+name: amazon_ads_integration_tests
+version: 1.2.0
+profile: integration_tests
 vars:
   amazon_ads:
-    amazon_ads_schema: amazon_ads_integration_tests_03
-    amazon_ads_ad_group_history_identifier: "ad_group_history_data"
-    amazon_ads_ad_group_level_report_identifier: "ad_group_level_report_data"
-    amazon_ads_advertised_product_report_identifier: "advertised_product_report_data"
-    amazon_ads_campaign_history_identifier: "campaign_history_data"
-    amazon_ads_campaign_level_report_identifier: "campaign_level_report_data"
-    amazon_ads_portfolio_history_identifier: "portfolio_history_data"
-    amazon_ads_product_ad_history_identifier: "product_ad_history_data"
-    amazon_ads_profile_identifier: "profile_data"
-    amazon_ads_keyword_history_identifier: "keyword_history_data"
-    amazon_ads_targeting_keyword_report_identifier: "targeting_keyword_report_data"
-    amazon_ads_search_term_ad_keyword_report_identifier: "search_term_ad_keyword_report_data"
-
+    amazon_ads_schema: amazon_ads_integration_tests
+    amazon_ads_ad_group_history_identifier: ad_group_history_data
+    amazon_ads_ad_group_level_report_identifier: ad_group_level_report_data
+    amazon_ads_advertised_product_report_identifier: advertised_product_report_data
+    amazon_ads_campaign_history_identifier: campaign_history_data
+    amazon_ads_campaign_level_report_identifier: campaign_level_report_data
+    amazon_ads_portfolio_history_identifier: portfolio_history_data
+    amazon_ads_product_ad_history_identifier: product_ad_history_data
+    amazon_ads_profile_identifier: profile_data
+    amazon_ads_keyword_history_identifier: keyword_history_data
+    amazon_ads_targeting_keyword_report_identifier: targeting_keyword_report_data
+    amazon_ads_search_term_ad_keyword_report_identifier: search_term_ad_keyword_report_data
   amazon_ads__campaign_passthrough_metrics:
-      - name: sales_7_d
-      - name: purchases_30_d 
-        alias: purchases_alias
-      - name: purchases_14_d
-
+  - name: sales_7_d
+  - name: purchases_30_d
+    alias: purchases_alias
+  - name: purchases_14_d
 models:
   +persist_docs:
     relation: true
-    columns: "{{ false if target.type in ('databricks') else true }}"
-  +schema: "amazon_ads_{{ var('directed_schema','dev') }}"
-
+    columns: '{{ false if target.type in (''databricks'') else true }}'
+  +schema: amazon_ads_{{ var('directed_schema','dev') }}
 dispatch:
-  - macro_namespace: dbt_utils
-    search_order: ['spark_utils', 'dbt_utils']
-
+- macro_namespace: dbt_utils
+  search_order:
+  - spark_utils
+  - dbt_utils
 seeds:
   +docs:
-    show: False
-  +quote_columns: "{{ true if target.type == 'redshift' else false }}"
+    show: false
+  +quote_columns: '{{ true if target.type == ''redshift'' else false }}'
+  +column_types:
+    _fivetran_synced: timestamp
   amazon_ads_integration_tests:
-    +column_types:
-      _fivetran_synced: "timestamp"
-      campaign_budget_amount: "float"
-      click_through_rate: "float"
-      keyword_bid: "float"
-      clicks: "float"
-      impressions: "float"
-      cost: "float"
+    advertised_product_report_data:
+      +column_types:
+        campaign_budget_amount: float
+        clicks: float
+        impressions: float
+        cost: float
+    campaign_level_report_data:
+      +column_types:
+        campaign_budget_amount: float
+        clicks: float
+        impressions: float
+        cost: float
+    ad_group_level_report_data:
+      +column_types:
+        clicks: float
+        cost: float
+        impressions: float
+    search_term_ad_keyword_report_data:
+      +column_types:
+        campaign_budget_amount: float
+        clicks: float
+        cost: float
+        impressions: float
+        keyword_bid: float
+    targeting_keyword_report_data:
+      +column_types:
+        campaign_budget_amount: float
+        clicks: float
+        cost: float
+        impressions: float
+        keyword_bid: float
 flags:
-  send_anonymous_usage_stats: False
-  use_colors: True
+  send_anonymous_usage_stats: false
+  use_colors: true