From f37945b491624a0ce1fb2d9b0ccb6f13b555a78a Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Fri, 20 Mar 2026 11:15:02 -0500 Subject: [PATCH 01/41] test/buildkite-improvements --- .buildkite/hooks/pre-command | 12 ++++++++++++ .buildkite/pipeline.yml | 10 ++++++++++ integration_tests/ci/sample.profiles.yml | 10 +++++----- integration_tests/dbt_project.yml | 4 +++- 4 files changed, 30 insertions(+), 6 deletions(-) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 81b7b00..8dabc52 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -2,6 +2,18 @@ set -e +# Generate unique schema name for this build +# Format: bk___ +# Get package name from environment variable set in pipeline.yml, default to "unknown" if not set +PACKAGE_NAME="${DBT_PACKAGE_NAME:-unknown}" +BUILD_NUMBER="${BUILDKITE_BUILD_NUMBER:-0}" + +# Create unique schema name using commit SHA and build number +COMMIT_SHA=$(echo "${BUILDKITE_COMMIT:-unknown}" | cut -c1-7) +BUILD_SCHEMA=$(echo "bk_${PACKAGE_NAME}_${COMMIT_SHA}_${BUILD_NUMBER}" | cut -c1-63 | sed 's/_$//')\ +export BUILD_SCHEMA +echo "Build schema: ${BUILD_SCHEMA}" + # Export secrets for Docker containers. # Restrict exposing secrets only to the steps that need them export GCLOUD_SERVICE_KEY=$(gcloud secrets versions access latest --secret="GCLOUD_SERVICE_KEY" --project="dbt-package-testing-363917") diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 2603602..e5fda17 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,3 +1,6 @@ +env: + DBT_PACKAGE_NAME: "amazon_ads" + steps: - label: ":postgres: Run Tests - Postgres" key: "run-dbt-postgres" @@ -7,6 +10,7 @@ steps: shell: [ "/bin/bash", "-e", "-c" ] environment: - "BASH_ENV=/tmp/.bashrc" + - "BUILD_SCHEMA" - "CI_POSTGRES_DBT_DBNAME" - "CI_POSTGRES_DBT_HOST" - "CI_POSTGRES_DBT_PASS" @@ -22,6 +26,7 @@ steps: shell: [ "/bin/bash", "-e", "-c" ] environment: - "BASH_ENV=/tmp/.bashrc" + - "BUILD_SCHEMA" - "CI_SNOWFLAKE_DBT_ACCOUNT" - "CI_SNOWFLAKE_DBT_DATABASE" - "CI_SNOWFLAKE_DBT_PASS" @@ -39,18 +44,22 @@ steps: shell: [ "/bin/bash", "-e", "-c" ] environment: - "BASH_ENV=/tmp/.bashrc" + - "BUILD_SCHEMA" - "GCLOUD_SERVICE_KEY" commands: | bash .buildkite/scripts/run_models.sh bigquery - label: ":amazon-redshift: Run Tests - Redshift" key: "run_dbt_redshift" + concurrency: 3 + concurrency_group: "warehouse/redshift" plugins: - docker#v3.13.0: image: "python:3.10.13" shell: [ "/bin/bash", "-e", "-c" ] environment: - "BASH_ENV=/tmp/.bashrc" + - "BUILD_SCHEMA" - "CI_REDSHIFT_DBT_DBNAME" - "CI_REDSHIFT_DBT_HOST" - "CI_REDSHIFT_DBT_PASS" @@ -66,6 +75,7 @@ steps: shell: [ "/bin/bash", "-e", "-c" ] environment: - "BASH_ENV=/tmp/.bashrc" + - "BUILD_SCHEMA" - "CI_DATABRICKS_DBT_HOST" - "CI_DATABRICKS_DBT_HTTP_PATH" - "CI_DATABRICKS_DBT_TOKEN" diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml index e7c86b2..db201ec 100644 --- a/integration_tests/ci/sample.profiles.yml +++ b/integration_tests/ci/sample.profiles.yml @@ -12,13 +12,13 @@ integration_tests: pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}" dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}" port: 5439 - schema: amazon_ads_integration_tests_03 + schema: "{{ env_var('BUILD_SCHEMA', 'amazon_ads_integration_tests') }}" threads: 8 bigquery: type: bigquery method: service-account-json project: 'dbt-package-testing' - schema: amazon_ads_integration_tests_03 + schema: "{{ env_var('BUILD_SCHEMA', 'amazon_ads_integration_tests') }}" threads: 8 keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}" snowflake: @@ -29,7 +29,7 @@ integration_tests: role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}" database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}" warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}" - schema: amazon_ads_integration_tests_03 + schema: "{{ env_var('BUILD_SCHEMA', 'amazon_ads_integration_tests') }}" threads: 8 postgres: type: postgres @@ -38,13 +38,13 @@ integration_tests: pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}" dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}" port: 5432 - schema: amazon_ads_integration_tests_03 + schema: "{{ env_var('BUILD_SCHEMA', 'amazon_ads_integration_tests') }}" threads: 8 databricks: catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}" host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}" http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}" - schema: amazon_ads_integration_tests_03 + schema: "{{ env_var('BUILD_SCHEMA', 'amazon_ads_integration_tests') }}" threads: 8 token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}" type: databricks diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 4b069ae..23db4fc 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -7,7 +7,9 @@ profile: 'integration_tests' vars: amazon_ads: - amazon_ads_schema: amazon_ads_integration_tests_03 + # Buildkite sets env_var('BUILD_SCHEMA' to a unique value for each run. + # When running locally, it will default to 'amazon_ads_integration_tests' + amazon_ads_schema: "{{ env_var('BUILD_SCHEMA', 'amazon_ads_integration_tests') }}" amazon_ads_ad_group_history_identifier: "ad_group_history_data" amazon_ads_ad_group_level_report_identifier: "ad_group_level_report_data" amazon_ads_advertised_product_report_identifier: "advertised_product_report_data" From 3a42fa50f1e87de4722a6bb88fe984c49361e1a8 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Fri, 20 Mar 2026 14:51:45 -0500 Subject: [PATCH 02/41] test/buildkite-improvements --- .buildkite/hooks/pre-command | 5 ++-- .buildkite/hooks/pre-exit | 21 +++++++++++++ .buildkite/pipeline.yml | 2 ++ .buildkite/scripts/run_models.sh | 2 -- .../macros/buildkite_cleanup.sql | 30 +++++++++++++++++++ 5 files changed, 55 insertions(+), 5 deletions(-) create mode 100644 .buildkite/hooks/pre-exit create mode 100644 integration_tests/macros/buildkite_cleanup.sql diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 8dabc52..8eb1a2c 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -3,14 +3,13 @@ set -e # Generate unique schema name for this build -# Format: bk___ +# Format: bk__ # Get package name from environment variable set in pipeline.yml, default to "unknown" if not set -PACKAGE_NAME="${DBT_PACKAGE_NAME:-unknown}" BUILD_NUMBER="${BUILDKITE_BUILD_NUMBER:-0}" # Create unique schema name using commit SHA and build number COMMIT_SHA=$(echo "${BUILDKITE_COMMIT:-unknown}" | cut -c1-7) -BUILD_SCHEMA=$(echo "bk_${PACKAGE_NAME}_${COMMIT_SHA}_${BUILD_NUMBER}" | cut -c1-63 | sed 's/_$//')\ +BUILD_SCHEMA=$(echo "bk_${COMMIT_SHA}_${BUILD_NUMBER}" | cut -c1-63 | sed 's/_$//')\ export BUILD_SCHEMA echo "Build schema: ${BUILD_SCHEMA}" diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit new file mode 100644 index 0000000..b78c96a --- /dev/null +++ b/.buildkite/hooks/pre-exit @@ -0,0 +1,21 @@ +#!/bin/bash + +# Cleanup hook - drops the build schema when the agent exits +# Runs regardless of build success, failure, cancellation, or interruption + +set +e # Don't exit on errors during cleanup + +echo "Cleaning up build schema: ${BUILD_SCHEMA:-not_set}" + +# Only proceed if BUILD_SCHEMA is set +if [[ -z "${BUILD_SCHEMA}" ]]; then + echo "BUILD_SCHEMA not set, skipping cleanup" + exit 0 +fi + +echo "Running schema cleanup..." +cd integration_tests +dbt run-operation amazon_ads_integration_tests.buildkite_cleanup +cd .. + +echo "Schema cleanup completed" \ No newline at end of file diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index e5fda17..97b6389 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -11,6 +11,7 @@ steps: environment: - "BASH_ENV=/tmp/.bashrc" - "BUILD_SCHEMA" + - "WAREHOUSE_TYPE=postgres" - "CI_POSTGRES_DBT_DBNAME" - "CI_POSTGRES_DBT_HOST" - "CI_POSTGRES_DBT_PASS" @@ -27,6 +28,7 @@ steps: environment: - "BASH_ENV=/tmp/.bashrc" - "BUILD_SCHEMA" + - "WAREHOUSE_TYPE=snowflake" - "CI_SNOWFLAKE_DBT_ACCOUNT" - "CI_SNOWFLAKE_DBT_DATABASE" - "CI_SNOWFLAKE_DBT_PASS" diff --git a/.buildkite/scripts/run_models.sh b/.buildkite/scripts/run_models.sh index fb75f9b..fadbdec 100644 --- a/.buildkite/scripts/run_models.sh +++ b/.buildkite/scripts/run_models.sh @@ -21,5 +21,3 @@ dbt run --target "$db" --full-refresh dbt test --target "$db" dbt run --vars '{amazon_ads__portfolio_history_enabled: false}' --target "$db" --full-refresh dbt test --vars '{amazon_ads__portfolio_history_enabled: false}' --target "$db" - -dbt run-operation fivetran_utils.drop_schemas_automation --target "$db" diff --git a/integration_tests/macros/buildkite_cleanup.sql b/integration_tests/macros/buildkite_cleanup.sql new file mode 100644 index 0000000..670c1ff --- /dev/null +++ b/integration_tests/macros/buildkite_cleanup.sql @@ -0,0 +1,30 @@ +{% macro buildkite_cleanup() %} + {{ return(adapter.dispatch('buildkite_cleanup', 'amazon_ads_integration_tests')()) }} +{%- endmacro %} + +{% macro default__buildkite_cleanup() %} + +{% set build_schema = env_var('BUILD_SCHEMA', '') %} +{% set package_name = env_var('DBT_PACKAGE_NAME', 'amazon_ads') %} + +{% if build_schema == '' %} + {{ print('No BUILD_SCHEMA specified for cleanup, skipping.') }} + {{ return('') }} +{% endif %} + +{{ print('Cleaning up schemas for ' ~ target.type ~ ' target...') }} + +{# Clean up both the source schema and the model output schema #} +{% set schemas_to_clean = [ + build_schema, + package_name ~ '_dev_' ~ build_schema +] %} + +{% for schema_name in schemas_to_clean %} + {% do adapter.drop_schema(api.Relation.create(database=target.database, schema=schema_name)) %} + {{ print('Schema ' ~ schema_name ~ ' dropped successfully.') }} +{% endfor %} + +{{ print('Cleanup completed for ' ~ target.type ~ '.') }} + +{% endmacro %} \ No newline at end of file From bbeda27fec053304b29bbaf324c342e78ecfac66 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Fri, 20 Mar 2026 14:53:11 -0500 Subject: [PATCH 03/41] fix --- .buildkite/hooks/pre-command | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 8eb1a2c..cf0de9c 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -9,7 +9,7 @@ BUILD_NUMBER="${BUILDKITE_BUILD_NUMBER:-0}" # Create unique schema name using commit SHA and build number COMMIT_SHA=$(echo "${BUILDKITE_COMMIT:-unknown}" | cut -c1-7) -BUILD_SCHEMA=$(echo "bk_${COMMIT_SHA}_${BUILD_NUMBER}" | cut -c1-63 | sed 's/_$//')\ +BUILD_SCHEMA=$(echo "bk_${COMMIT_SHA}_${BUILD_NUMBER}" | cut -c1-63 | sed 's/_$//') export BUILD_SCHEMA echo "Build schema: ${BUILD_SCHEMA}" From 44682e34d9829e4402ca34feaaced6a4a0fe42cf Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Fri, 20 Mar 2026 16:26:45 -0500 Subject: [PATCH 04/41] test again --- .buildkite/hooks/pre-exit | 60 ++++++++++++++++++- .buildkite/pipeline.yml | 12 ++-- integration_tests/ci/sample.profiles.yml | 10 ++-- integration_tests/dbt_project.yml | 4 +- .../macros/buildkite_cleanup.sql | 30 ---------- 5 files changed, 68 insertions(+), 48 deletions(-) delete mode 100644 integration_tests/macros/buildkite_cleanup.sql diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit index b78c96a..4063cf3 100644 --- a/.buildkite/hooks/pre-exit +++ b/.buildkite/hooks/pre-exit @@ -14,8 +14,62 @@ if [[ -z "${BUILD_SCHEMA}" ]]; then fi echo "Running schema cleanup..." -cd integration_tests -dbt run-operation amazon_ads_integration_tests.buildkite_cleanup -cd .. + +# Validate schema name (prevent injection) +if [[ ! "${BUILD_SCHEMA}" =~ ^[a-zA-Z0-9_]+$ ]]; then + echo "Invalid schema name format" + exit 0 +fi + +# Schemas to clean +PACKAGE_NAME="${DBT_PACKAGE_NAME:-amazon_ads}" +SCHEMA1="${BUILD_SCHEMA}" +SCHEMA2="${PACKAGE_NAME}_dev_${BUILD_SCHEMA}" + +echo "Cleaning schemas: ${SCHEMA1}, ${SCHEMA2}" + +# BigQuery cleanup +if [[ -n "${GCLOUD_SERVICE_KEY:-}" ]] && command -v bq >/dev/null 2>&1; then + echo "${GCLOUD_SERVICE_KEY}" | gcloud auth activate-service-account --key-file=- --quiet 2>/dev/null + bq rm -r -f -d "dbt-package-testing:${SCHEMA1}" --quiet 2>/dev/null || true + bq rm -r -f -d "dbt-package-testing:${SCHEMA2}" --quiet 2>/dev/null || true + echo "BigQuery cleanup completed" +fi + +# PostgreSQL cleanup +if [[ -n "${CI_POSTGRES_DBT_HOST:-}" ]] && command -v psql >/dev/null 2>&1; then + PGPASSWORD="${CI_POSTGRES_DBT_PASS}" psql -h "${CI_POSTGRES_DBT_HOST}" -U "${CI_POSTGRES_DBT_USER}" -d "${CI_POSTGRES_DBT_DBNAME}" -c "DROP SCHEMA IF EXISTS \"${SCHEMA1}\" CASCADE;" -q 2>/dev/null || true + PGPASSWORD="${CI_POSTGRES_DBT_PASS}" psql -h "${CI_POSTGRES_DBT_HOST}" -U "${CI_POSTGRES_DBT_USER}" -d "${CI_POSTGRES_DBT_DBNAME}" -c "DROP SCHEMA IF EXISTS \"${SCHEMA2}\" CASCADE;" -q 2>/dev/null || true + echo "PostgreSQL cleanup completed" +fi + +# Redshift cleanup +if [[ -n "${CI_REDSHIFT_DBT_HOST:-}" ]] && command -v psql >/dev/null 2>&1; then + PGPASSWORD="${CI_REDSHIFT_DBT_PASS}" psql -h "${CI_REDSHIFT_DBT_HOST}" -U "${CI_REDSHIFT_DBT_USER}" -d "${CI_REDSHIFT_DBT_DBNAME}" -c "DROP SCHEMA IF EXISTS \"${SCHEMA1}\" CASCADE;" -q 2>/dev/null || true + PGPASSWORD="${CI_REDSHIFT_DBT_PASS}" psql -h "${CI_REDSHIFT_DBT_HOST}" -U "${CI_REDSHIFT_DBT_USER}" -d "${CI_REDSHIFT_DBT_DBNAME}" -c "DROP SCHEMA IF EXISTS \"${SCHEMA2}\" CASCADE;" -q 2>/dev/null || true + echo "Redshift cleanup completed" +fi + +# Snowflake cleanup +if [[ -n "${CI_SNOWFLAKE_DBT_ACCOUNT:-}" ]] && command -v snowsql >/dev/null 2>&1; then + snowsql -a "${CI_SNOWFLAKE_DBT_ACCOUNT}" -u "${CI_SNOWFLAKE_DBT_USER}" -p "${CI_SNOWFLAKE_DBT_PASS}" \ + -r "${CI_SNOWFLAKE_DBT_ROLE}" -w "${CI_SNOWFLAKE_DBT_WAREHOUSE}" -d "${CI_SNOWFLAKE_DBT_DATABASE}" \ + -q "DROP SCHEMA IF EXISTS \"${SCHEMA1}\" CASCADE;" --quiet 2>/dev/null || true + snowsql -a "${CI_SNOWFLAKE_DBT_ACCOUNT}" -u "${CI_SNOWFLAKE_DBT_USER}" -p "${CI_SNOWFLAKE_DBT_PASS}" \ + -r "${CI_SNOWFLAKE_DBT_ROLE}" -w "${CI_SNOWFLAKE_DBT_WAREHOUSE}" -d "${CI_SNOWFLAKE_DBT_DATABASE}" \ + -q "DROP SCHEMA IF EXISTS \"${SCHEMA2}\" CASCADE;" --quiet 2>/dev/null || true + echo "Snowflake cleanup completed" +fi + +# Databricks cleanup +if [[ -n "${CI_DATABRICKS_DBT_HOST:-}" ]] && command -v curl >/dev/null 2>&1; then + curl -s -X POST -H "Authorization: Bearer ${CI_DATABRICKS_DBT_TOKEN}" -H "Content-Type: application/json" \ + "${CI_DATABRICKS_DBT_HOST}/api/2.0/sql/statements" \ + -d "{\"statement\": \"DROP SCHEMA IF EXISTS \\\"${CI_DATABRICKS_DBT_CATALOG}\\\".\\\"${SCHEMA1}\\\" CASCADE\"}" >/dev/null 2>&1 || true + curl -s -X POST -H "Authorization: Bearer ${CI_DATABRICKS_DBT_TOKEN}" -H "Content-Type: application/json" \ + "${CI_DATABRICKS_DBT_HOST}/api/2.0/sql/statements" \ + -d "{\"statement\": \"DROP SCHEMA IF EXISTS \\\"${CI_DATABRICKS_DBT_CATALOG}\\\".\\\"${SCHEMA2}\\\" CASCADE\"}" >/dev/null 2>&1 || true + echo "Databricks cleanup completed" +fi echo "Schema cleanup completed" \ No newline at end of file diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 97b6389..f90c0d3 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -6,12 +6,11 @@ steps: key: "run-dbt-postgres" plugins: - docker#v3.13.0: - image: "python:3.10.13" + image: "python:3.13" shell: [ "/bin/bash", "-e", "-c" ] environment: - "BASH_ENV=/tmp/.bashrc" - "BUILD_SCHEMA" - - "WAREHOUSE_TYPE=postgres" - "CI_POSTGRES_DBT_DBNAME" - "CI_POSTGRES_DBT_HOST" - "CI_POSTGRES_DBT_PASS" @@ -23,12 +22,11 @@ steps: key: "run_dbt_snowflake" plugins: - docker#v3.13.0: - image: "python:3.10.13" + image: "python:3.13" shell: [ "/bin/bash", "-e", "-c" ] environment: - "BASH_ENV=/tmp/.bashrc" - "BUILD_SCHEMA" - - "WAREHOUSE_TYPE=snowflake" - "CI_SNOWFLAKE_DBT_ACCOUNT" - "CI_SNOWFLAKE_DBT_DATABASE" - "CI_SNOWFLAKE_DBT_PASS" @@ -42,7 +40,7 @@ steps: key: "run_dbt_bigquery" plugins: - docker#v3.13.0: - image: "python:3.10.13" + image: "python:3.13" shell: [ "/bin/bash", "-e", "-c" ] environment: - "BASH_ENV=/tmp/.bashrc" @@ -57,7 +55,7 @@ steps: concurrency_group: "warehouse/redshift" plugins: - docker#v3.13.0: - image: "python:3.10.13" + image: "python:3.13" shell: [ "/bin/bash", "-e", "-c" ] environment: - "BASH_ENV=/tmp/.bashrc" @@ -73,7 +71,7 @@ steps: key: "run_dbt_databricks" plugins: - docker#v3.13.0: - image: "python:3.10.13" + image: "python:3.13" shell: [ "/bin/bash", "-e", "-c" ] environment: - "BASH_ENV=/tmp/.bashrc" diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml index db201ec..9a4e866 100644 --- a/integration_tests/ci/sample.profiles.yml +++ b/integration_tests/ci/sample.profiles.yml @@ -12,13 +12,13 @@ integration_tests: pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}" dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}" port: 5439 - schema: "{{ env_var('BUILD_SCHEMA', 'amazon_ads_integration_tests') }}" + schema: "{{ env_var('BUILD_SCHEMA', 'amazon_ads_integration_tests') | as_text }}" threads: 8 bigquery: type: bigquery method: service-account-json project: 'dbt-package-testing' - schema: "{{ env_var('BUILD_SCHEMA', 'amazon_ads_integration_tests') }}" + schema: "{{ env_var('BUILD_SCHEMA', 'amazon_ads_integration_tests') | as_text }}" threads: 8 keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}" snowflake: @@ -29,7 +29,7 @@ integration_tests: role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}" database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}" warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}" - schema: "{{ env_var('BUILD_SCHEMA', 'amazon_ads_integration_tests') }}" + schema: "{{ env_var('BUILD_SCHEMA', 'amazon_ads_integration_tests') | as_text }}" threads: 8 postgres: type: postgres @@ -38,13 +38,13 @@ integration_tests: pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}" dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}" port: 5432 - schema: "{{ env_var('BUILD_SCHEMA', 'amazon_ads_integration_tests') }}" + schema: "{{ env_var('BUILD_SCHEMA', 'amazon_ads_integration_tests') | as_text }}" threads: 8 databricks: catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}" host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}" http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}" - schema: "{{ env_var('BUILD_SCHEMA', 'amazon_ads_integration_tests') }}" + schema: "{{ env_var('BUILD_SCHEMA', 'amazon_ads_integration_tests') | as_text }}" threads: 8 token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}" type: databricks diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 23db4fc..93e245d 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -7,9 +7,7 @@ profile: 'integration_tests' vars: amazon_ads: - # Buildkite sets env_var('BUILD_SCHEMA' to a unique value for each run. - # When running locally, it will default to 'amazon_ads_integration_tests' - amazon_ads_schema: "{{ env_var('BUILD_SCHEMA', 'amazon_ads_integration_tests') }}" + amazon_ads_schema: "{{ env_var('BUILD_SCHEMA', 'amazon_ads_integration_tests') | as_text }}" amazon_ads_ad_group_history_identifier: "ad_group_history_data" amazon_ads_ad_group_level_report_identifier: "ad_group_level_report_data" amazon_ads_advertised_product_report_identifier: "advertised_product_report_data" diff --git a/integration_tests/macros/buildkite_cleanup.sql b/integration_tests/macros/buildkite_cleanup.sql deleted file mode 100644 index 670c1ff..0000000 --- a/integration_tests/macros/buildkite_cleanup.sql +++ /dev/null @@ -1,30 +0,0 @@ -{% macro buildkite_cleanup() %} - {{ return(adapter.dispatch('buildkite_cleanup', 'amazon_ads_integration_tests')()) }} -{%- endmacro %} - -{% macro default__buildkite_cleanup() %} - -{% set build_schema = env_var('BUILD_SCHEMA', '') %} -{% set package_name = env_var('DBT_PACKAGE_NAME', 'amazon_ads') %} - -{% if build_schema == '' %} - {{ print('No BUILD_SCHEMA specified for cleanup, skipping.') }} - {{ return('') }} -{% endif %} - -{{ print('Cleaning up schemas for ' ~ target.type ~ ' target...') }} - -{# Clean up both the source schema and the model output schema #} -{% set schemas_to_clean = [ - build_schema, - package_name ~ '_dev_' ~ build_schema -] %} - -{% for schema_name in schemas_to_clean %} - {% do adapter.drop_schema(api.Relation.create(database=target.database, schema=schema_name)) %} - {{ print('Schema ' ~ schema_name ~ ' dropped successfully.') }} -{% endfor %} - -{{ print('Cleanup completed for ' ~ target.type ~ '.') }} - -{% endmacro %} \ No newline at end of file From 5e99d168d511972262b580b104582de0eea6c9ab Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Fri, 20 Mar 2026 16:41:08 -0500 Subject: [PATCH 05/41] test again --- .buildkite/hooks/pre-exit | 96 +++++++++++++----------- integration_tests/ci/sample.profiles.yml | 10 +-- integration_tests/dbt_project.yml | 2 +- 3 files changed, 58 insertions(+), 50 deletions(-) diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit index 4063cf3..896b65a 100644 --- a/.buildkite/hooks/pre-exit +++ b/.buildkite/hooks/pre-exit @@ -24,52 +24,60 @@ fi # Schemas to clean PACKAGE_NAME="${DBT_PACKAGE_NAME:-amazon_ads}" SCHEMA1="${BUILD_SCHEMA}" -SCHEMA2="${PACKAGE_NAME}_dev_${BUILD_SCHEMA}" +SCHEMA2="${BUILD_SCHEMA}_${PACKAGE_NAME}_dev" echo "Cleaning schemas: ${SCHEMA1}, ${SCHEMA2}" +echo "Step key: ${BUILDKITE_STEP_KEY:-unknown}" -# BigQuery cleanup -if [[ -n "${GCLOUD_SERVICE_KEY:-}" ]] && command -v bq >/dev/null 2>&1; then - echo "${GCLOUD_SERVICE_KEY}" | gcloud auth activate-service-account --key-file=- --quiet 2>/dev/null - bq rm -r -f -d "dbt-package-testing:${SCHEMA1}" --quiet 2>/dev/null || true - bq rm -r -f -d "dbt-package-testing:${SCHEMA2}" --quiet 2>/dev/null || true - echo "BigQuery cleanup completed" -fi - -# PostgreSQL cleanup -if [[ -n "${CI_POSTGRES_DBT_HOST:-}" ]] && command -v psql >/dev/null 2>&1; then - PGPASSWORD="${CI_POSTGRES_DBT_PASS}" psql -h "${CI_POSTGRES_DBT_HOST}" -U "${CI_POSTGRES_DBT_USER}" -d "${CI_POSTGRES_DBT_DBNAME}" -c "DROP SCHEMA IF EXISTS \"${SCHEMA1}\" CASCADE;" -q 2>/dev/null || true - PGPASSWORD="${CI_POSTGRES_DBT_PASS}" psql -h "${CI_POSTGRES_DBT_HOST}" -U "${CI_POSTGRES_DBT_USER}" -d "${CI_POSTGRES_DBT_DBNAME}" -c "DROP SCHEMA IF EXISTS \"${SCHEMA2}\" CASCADE;" -q 2>/dev/null || true - echo "PostgreSQL cleanup completed" -fi - -# Redshift cleanup -if [[ -n "${CI_REDSHIFT_DBT_HOST:-}" ]] && command -v psql >/dev/null 2>&1; then - PGPASSWORD="${CI_REDSHIFT_DBT_PASS}" psql -h "${CI_REDSHIFT_DBT_HOST}" -U "${CI_REDSHIFT_DBT_USER}" -d "${CI_REDSHIFT_DBT_DBNAME}" -c "DROP SCHEMA IF EXISTS \"${SCHEMA1}\" CASCADE;" -q 2>/dev/null || true - PGPASSWORD="${CI_REDSHIFT_DBT_PASS}" psql -h "${CI_REDSHIFT_DBT_HOST}" -U "${CI_REDSHIFT_DBT_USER}" -d "${CI_REDSHIFT_DBT_DBNAME}" -c "DROP SCHEMA IF EXISTS \"${SCHEMA2}\" CASCADE;" -q 2>/dev/null || true - echo "Redshift cleanup completed" -fi - -# Snowflake cleanup -if [[ -n "${CI_SNOWFLAKE_DBT_ACCOUNT:-}" ]] && command -v snowsql >/dev/null 2>&1; then - snowsql -a "${CI_SNOWFLAKE_DBT_ACCOUNT}" -u "${CI_SNOWFLAKE_DBT_USER}" -p "${CI_SNOWFLAKE_DBT_PASS}" \ - -r "${CI_SNOWFLAKE_DBT_ROLE}" -w "${CI_SNOWFLAKE_DBT_WAREHOUSE}" -d "${CI_SNOWFLAKE_DBT_DATABASE}" \ - -q "DROP SCHEMA IF EXISTS \"${SCHEMA1}\" CASCADE;" --quiet 2>/dev/null || true - snowsql -a "${CI_SNOWFLAKE_DBT_ACCOUNT}" -u "${CI_SNOWFLAKE_DBT_USER}" -p "${CI_SNOWFLAKE_DBT_PASS}" \ - -r "${CI_SNOWFLAKE_DBT_ROLE}" -w "${CI_SNOWFLAKE_DBT_WAREHOUSE}" -d "${CI_SNOWFLAKE_DBT_DATABASE}" \ - -q "DROP SCHEMA IF EXISTS \"${SCHEMA2}\" CASCADE;" --quiet 2>/dev/null || true - echo "Snowflake cleanup completed" -fi - -# Databricks cleanup -if [[ -n "${CI_DATABRICKS_DBT_HOST:-}" ]] && command -v curl >/dev/null 2>&1; then - curl -s -X POST -H "Authorization: Bearer ${CI_DATABRICKS_DBT_TOKEN}" -H "Content-Type: application/json" \ - "${CI_DATABRICKS_DBT_HOST}/api/2.0/sql/statements" \ - -d "{\"statement\": \"DROP SCHEMA IF EXISTS \\\"${CI_DATABRICKS_DBT_CATALOG}\\\".\\\"${SCHEMA1}\\\" CASCADE\"}" >/dev/null 2>&1 || true - curl -s -X POST -H "Authorization: Bearer ${CI_DATABRICKS_DBT_TOKEN}" -H "Content-Type: application/json" \ - "${CI_DATABRICKS_DBT_HOST}/api/2.0/sql/statements" \ - -d "{\"statement\": \"DROP SCHEMA IF EXISTS \\\"${CI_DATABRICKS_DBT_CATALOG}\\\".\\\"${SCHEMA2}\\\" CASCADE\"}" >/dev/null 2>&1 || true - echo "Databricks cleanup completed" -fi +# Clean up only the warehouse for this specific step +case "${BUILDKITE_STEP_KEY:-}" in + "run_dbt_bigquery") + if [[ -n "${GCLOUD_SERVICE_KEY:-}" ]] && command -v bq >/dev/null 2>&1; then + echo "${GCLOUD_SERVICE_KEY}" | gcloud auth activate-service-account --key-file=- --quiet 2>/dev/null + bq --quiet rm -r -f -d "dbt-package-testing:${SCHEMA1}" 2>/dev/null || true + bq --quiet rm -r -f -d "dbt-package-testing:${SCHEMA2}" 2>/dev/null || true + echo "BigQuery cleanup completed" + fi + ;; + "run-dbt-postgres") + if [[ -n "${CI_POSTGRES_DBT_HOST:-}" ]] && command -v psql >/dev/null 2>&1; then + PGPASSWORD="${CI_POSTGRES_DBT_PASS}" psql -h "${CI_POSTGRES_DBT_HOST}" -U "${CI_POSTGRES_DBT_USER}" -d "${CI_POSTGRES_DBT_DBNAME}" -c "DROP SCHEMA IF EXISTS \"${SCHEMA1}\" CASCADE;" -q 2>/dev/null || true + PGPASSWORD="${CI_POSTGRES_DBT_PASS}" psql -h "${CI_POSTGRES_DBT_HOST}" -U "${CI_POSTGRES_DBT_USER}" -d "${CI_POSTGRES_DBT_DBNAME}" -c "DROP SCHEMA IF EXISTS \"${SCHEMA2}\" CASCADE;" -q 2>/dev/null || true + echo "PostgreSQL cleanup completed" + fi + ;; + "run_dbt_redshift") + if [[ -n "${CI_REDSHIFT_DBT_HOST:-}" ]] && command -v psql >/dev/null 2>&1; then + PGPASSWORD="${CI_REDSHIFT_DBT_PASS}" psql -h "${CI_REDSHIFT_DBT_HOST}" -U "${CI_REDSHIFT_DBT_USER}" -d "${CI_REDSHIFT_DBT_DBNAME}" -c "DROP SCHEMA IF EXISTS \"${SCHEMA1}\" CASCADE;" -q 2>/dev/null || true + PGPASSWORD="${CI_REDSHIFT_DBT_PASS}" psql -h "${CI_REDSHIFT_DBT_HOST}" -U "${CI_REDSHIFT_DBT_USER}" -d "${CI_REDSHIFT_DBT_DBNAME}" -c "DROP SCHEMA IF EXISTS \"${SCHEMA2}\" CASCADE;" -q 2>/dev/null || true + echo "Redshift cleanup completed" + fi + ;; + "run_dbt_snowflake") + if [[ -n "${CI_SNOWFLAKE_DBT_ACCOUNT:-}" ]] && command -v snowsql >/dev/null 2>&1; then + snowsql -a "${CI_SNOWFLAKE_DBT_ACCOUNT}" -u "${CI_SNOWFLAKE_DBT_USER}" -p "${CI_SNOWFLAKE_DBT_PASS}" \ + -r "${CI_SNOWFLAKE_DBT_ROLE}" -w "${CI_SNOWFLAKE_DBT_WAREHOUSE}" -d "${CI_SNOWFLAKE_DBT_DATABASE}" \ + -q "DROP SCHEMA IF EXISTS \"${SCHEMA1}\" CASCADE;" --quiet 2>/dev/null || true + snowsql -a "${CI_SNOWFLAKE_DBT_ACCOUNT}" -u "${CI_SNOWFLAKE_DBT_USER}" -p "${CI_SNOWFLAKE_DBT_PASS}" \ + -r "${CI_SNOWFLAKE_DBT_ROLE}" -w "${CI_SNOWFLAKE_DBT_WAREHOUSE}" -d "${CI_SNOWFLAKE_DBT_DATABASE}" \ + -q "DROP SCHEMA IF EXISTS \"${SCHEMA2}\" CASCADE;" --quiet 2>/dev/null || true + echo "Snowflake cleanup completed" + fi + ;; + "run_dbt_databricks") + if [[ -n "${CI_DATABRICKS_DBT_HOST:-}" ]] && command -v curl >/dev/null 2>&1; then + curl -s -X POST -H "Authorization: Bearer ${CI_DATABRICKS_DBT_TOKEN}" -H "Content-Type: application/json" \ + "${CI_DATABRICKS_DBT_HOST}/api/2.0/sql/statements" \ + -d "{\"statement\": \"DROP SCHEMA IF EXISTS \\\"${CI_DATABRICKS_DBT_CATALOG}\\\".\\\"${SCHEMA1}\\\" CASCADE\"}" >/dev/null 2>&1 || true + curl -s -X POST -H "Authorization: Bearer ${CI_DATABRICKS_DBT_TOKEN}" -H "Content-Type: application/json" \ + "${CI_DATABRICKS_DBT_HOST}/api/2.0/sql/statements" \ + -d "{\"statement\": \"DROP SCHEMA IF EXISTS \\\"${CI_DATABRICKS_DBT_CATALOG}\\\".\\\"${SCHEMA2}\\\" CASCADE\"}" >/dev/null 2>&1 || true + echo "Databricks cleanup completed" + fi + ;; + *) + echo "Unknown or no step key, skipping cleanup" + ;; +esac echo "Schema cleanup completed" \ No newline at end of file diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml index 9a4e866..db201ec 100644 --- a/integration_tests/ci/sample.profiles.yml +++ b/integration_tests/ci/sample.profiles.yml @@ -12,13 +12,13 @@ integration_tests: pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}" dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}" port: 5439 - schema: "{{ env_var('BUILD_SCHEMA', 'amazon_ads_integration_tests') | as_text }}" + schema: "{{ env_var('BUILD_SCHEMA', 'amazon_ads_integration_tests') }}" threads: 8 bigquery: type: bigquery method: service-account-json project: 'dbt-package-testing' - schema: "{{ env_var('BUILD_SCHEMA', 'amazon_ads_integration_tests') | as_text }}" + schema: "{{ env_var('BUILD_SCHEMA', 'amazon_ads_integration_tests') }}" threads: 8 keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}" snowflake: @@ -29,7 +29,7 @@ integration_tests: role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}" database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}" warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}" - schema: "{{ env_var('BUILD_SCHEMA', 'amazon_ads_integration_tests') | as_text }}" + schema: "{{ env_var('BUILD_SCHEMA', 'amazon_ads_integration_tests') }}" threads: 8 postgres: type: postgres @@ -38,13 +38,13 @@ integration_tests: pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}" dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}" port: 5432 - schema: "{{ env_var('BUILD_SCHEMA', 'amazon_ads_integration_tests') | as_text }}" + schema: "{{ env_var('BUILD_SCHEMA', 'amazon_ads_integration_tests') }}" threads: 8 databricks: catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}" host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}" http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}" - schema: "{{ env_var('BUILD_SCHEMA', 'amazon_ads_integration_tests') | as_text }}" + schema: "{{ env_var('BUILD_SCHEMA', 'amazon_ads_integration_tests') }}" threads: 8 token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}" type: databricks diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 93e245d..6c4c35a 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -7,7 +7,7 @@ profile: 'integration_tests' vars: amazon_ads: - amazon_ads_schema: "{{ env_var('BUILD_SCHEMA', 'amazon_ads_integration_tests') | as_text }}" + amazon_ads_schema: "{{ env_var('BUILD_SCHEMA', 'amazon_ads_integration_tests') }}" amazon_ads_ad_group_history_identifier: "ad_group_history_data" amazon_ads_ad_group_level_report_identifier: "ad_group_level_report_data" amazon_ads_advertised_product_report_identifier: "advertised_product_report_data" From d93c8aed3b7987a1929a0896ccfd1d5f8d438a56 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Fri, 20 Mar 2026 16:59:21 -0500 Subject: [PATCH 06/41] test again --- .buildkite/hooks/pre-command | 65 +++++++++++++++++++++---------- .buildkite/scripts/run_models.sh | 10 ++++- integration_tests/dbt_project.yml | 2 +- 3 files changed, 54 insertions(+), 23 deletions(-) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index cf0de9c..1cf0a29 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -13,24 +13,47 @@ BUILD_SCHEMA=$(echo "bk_${COMMIT_SHA}_${BUILD_NUMBER}" | cut -c1-63 | sed 's/_$/ export BUILD_SCHEMA echo "Build schema: ${BUILD_SCHEMA}" -# Export secrets for Docker containers. -# Restrict exposing secrets only to the steps that need them -export GCLOUD_SERVICE_KEY=$(gcloud secrets versions access latest --secret="GCLOUD_SERVICE_KEY" --project="dbt-package-testing-363917") -export CI_POSTGRES_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_HOST" --project="dbt-package-testing-363917") -export CI_POSTGRES_DBT_USER=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_USER" --project="dbt-package-testing-363917") -export CI_POSTGRES_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_PASS" --project="dbt-package-testing-363917") -export CI_POSTGRES_DBT_DBNAME=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_DBNAME" --project="dbt-package-testing-363917") -export CI_REDSHIFT_DBT_DBNAME=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_DBNAME" --project="dbt-package-testing-363917") -export CI_REDSHIFT_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_HOST" --project="dbt-package-testing-363917") -export CI_REDSHIFT_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_PASS" --project="dbt-package-testing-363917") -export CI_REDSHIFT_DBT_USER=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_USER" --project="dbt-package-testing-363917") -export CI_SNOWFLAKE_DBT_ACCOUNT=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_ACCOUNT" --project="dbt-package-testing-363917") -export CI_SNOWFLAKE_DBT_DATABASE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_DATABASE" --project="dbt-package-testing-363917") -export CI_SNOWFLAKE_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_PASS" --project="dbt-package-testing-363917") -export CI_SNOWFLAKE_DBT_ROLE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_ROLE" --project="dbt-package-testing-363917") -export CI_SNOWFLAKE_DBT_USER=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_USER" --project="dbt-package-testing-363917") -export CI_SNOWFLAKE_DBT_WAREHOUSE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_WAREHOUSE" --project="dbt-package-testing-363917") -export CI_DATABRICKS_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HOST" --project="dbt-package-testing-363917") -export CI_DATABRICKS_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HTTP_PATH" --project="dbt-package-testing-363917") -export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917") -export CI_DATABRICKS_DBT_CATALOG=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_CATALOG" --project="dbt-package-testing-363917") \ No newline at end of file +# Export secrets only for the warehouse needed by this step +echo "Step key: ${BUILDKITE_STEP_KEY:-unknown}" + +case "${BUILDKITE_STEP_KEY:-}" in + "run_dbt_bigquery") + echo "Setting up BigQuery credentials and dependencies" + export GCLOUD_SERVICE_KEY=$(gcloud secrets versions access latest --secret="GCLOUD_SERVICE_KEY" --project="dbt-package-testing-363917") + DBT_ADAPTER="dbt-bigquery" + ;; + "run-dbt-postgres") + echo "Setting up PostgreSQL credentials and dependencies" + export CI_POSTGRES_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_HOST" --project="dbt-package-testing-363917") + export CI_POSTGRES_DBT_USER=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_USER" --project="dbt-package-testing-363917") + export CI_POSTGRES_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_PASS" --project="dbt-package-testing-363917") + export CI_POSTGRES_DBT_DBNAME=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_DBNAME" --project="dbt-package-testing-363917") + DBT_ADAPTER="dbt-postgres" + ;; + "run_dbt_redshift") + echo "Setting up Redshift credentials and dependencies" + export CI_REDSHIFT_DBT_DBNAME=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_DBNAME" --project="dbt-package-testing-363917") + export CI_REDSHIFT_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_HOST" --project="dbt-package-testing-363917") + export CI_REDSHIFT_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_PASS" --project="dbt-package-testing-363917") + export CI_REDSHIFT_DBT_USER=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_USER" --project="dbt-package-testing-363917") + DBT_ADAPTER="dbt-redshift" + ;; + "run_dbt_snowflake") + echo "Setting up Snowflake credentials and dependencies" + export CI_SNOWFLAKE_DBT_ACCOUNT=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_ACCOUNT" --project="dbt-package-testing-363917") + export CI_SNOWFLAKE_DBT_DATABASE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_DATABASE" --project="dbt-package-testing-363917") + export CI_SNOWFLAKE_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_PASS" --project="dbt-package-testing-363917") + export CI_SNOWFLAKE_DBT_ROLE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_ROLE" --project="dbt-package-testing-363917") + export CI_SNOWFLAKE_DBT_USER=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_USER" --project="dbt-package-testing-363917") + export CI_SNOWFLAKE_DBT_WAREHOUSE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_WAREHOUSE" --project="dbt-package-testing-363917") + DBT_ADAPTER="dbt-snowflake" + ;; + "run_dbt_databricks") + echo "Setting up Databricks credentials and dependencies" + export CI_DATABRICKS_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HOST" --project="dbt-package-testing-363917") + export CI_DATABRICKS_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HTTP_PATH" --project="dbt-package-testing-363917") + export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917") + export CI_DATABRICKS_DBT_CATALOG=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_CATALOG" --project="dbt-package-testing-363917") + DBT_ADAPTER="dbt-databricks" + ;; +esac \ No newline at end of file diff --git a/.buildkite/scripts/run_models.sh b/.buildkite/scripts/run_models.sh index fadbdec..0285b28 100644 --- a/.buildkite/scripts/run_models.sh +++ b/.buildkite/scripts/run_models.sh @@ -8,7 +8,15 @@ apt-get install libsasl2-dev python3 -m venv venv . venv/bin/activate pip install --upgrade pip setuptools -pip install -r integration_tests/requirements.txt + +# Install specific adapter for this warehouse or fall back to full requirements +if [[ -n "${DBT_ADAPTER:-}" ]]; then + echo "Installing specific dbt adapter: ${DBT_ADAPTER}" + pip install "${DBT_ADAPTER}>=1.3.0,<2.0.0" +else + echo "Installing all adapters from requirements.txt" + pip install -r integration_tests/requirements.txt +fi mkdir -p ~/.dbt cp integration_tests/ci/sample.profiles.yml ~/.dbt/profiles.yml diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 6c4c35a..4d2f863 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -7,7 +7,7 @@ profile: 'integration_tests' vars: amazon_ads: - amazon_ads_schema: "{{ env_var('BUILD_SCHEMA', 'amazon_ads_integration_tests') }}" + amazon_ads_schema: "{{ env_var('BUILD_SCHEMA') }}" amazon_ads_ad_group_history_identifier: "ad_group_history_data" amazon_ads_ad_group_level_report_identifier: "ad_group_level_report_data" amazon_ads_advertised_product_report_identifier: "advertised_product_report_data" From 6b0c0455bedfd5a9daf8c341d25278a792d9fabe Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Fri, 20 Mar 2026 17:17:34 -0500 Subject: [PATCH 07/41] test again --- .buildkite/scripts/run_models.sh | 30 ++++++++++++++++++++++++++---- integration_tests/dbt_project.yml | 2 +- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/.buildkite/scripts/run_models.sh b/.buildkite/scripts/run_models.sh index 0285b28..3856fba 100644 --- a/.buildkite/scripts/run_models.sh +++ b/.buildkite/scripts/run_models.sh @@ -23,9 +23,31 @@ cp integration_tests/ci/sample.profiles.yml ~/.dbt/profiles.yml db=$1 echo `pwd` cd integration_tests + +# Set up base variables +BASE_VARS="" +if [[ -n "${BUILD_SCHEMA:-}" ]]; then + BASE_VARS="\"amazon_ads_schema\": \"${BUILD_SCHEMA}\"" +fi + +# Define test scenarios - easy to add/modify +declare -a test_scenarios=( + # Scenario 1: Default run + "{${BASE_VARS}}" + + # Scenario 2: Portfolio history disabled + "{${BASE_VARS}, \"amazon_ads__portfolio_history_enabled\": false}" +) + dbt deps dbt seed --target "$db" --full-refresh -dbt run --target "$db" --full-refresh -dbt test --target "$db" -dbt run --vars '{amazon_ads__portfolio_history_enabled: false}' --target "$db" --full-refresh -dbt test --vars '{amazon_ads__portfolio_history_enabled: false}' --target "$db" + +# Run each test scenario +for i in "${!test_scenarios[@]}"; do + scenario_num=$((i + 1)) + vars="${test_scenarios[$i]}" + + echo "Running test scenario ${scenario_num}: ${vars}" + dbt run --target "$db" --vars "$vars" --full-refresh + dbt test --target "$db" --vars "$vars" +done diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 4d2f863..fd8b85a 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -7,7 +7,7 @@ profile: 'integration_tests' vars: amazon_ads: - amazon_ads_schema: "{{ env_var('BUILD_SCHEMA') }}" + amazon_ads_schema: "amazon_ads_integration_tests" amazon_ads_ad_group_history_identifier: "ad_group_history_data" amazon_ads_ad_group_level_report_identifier: "ad_group_level_report_data" amazon_ads_advertised_product_report_identifier: "advertised_product_report_data" From 0aa67feca2b183d6400af3f8092e48920092af25 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Fri, 20 Mar 2026 17:33:37 -0500 Subject: [PATCH 08/41] test again --- .buildkite/scripts/run_models.sh | 5 +++- integration_tests/dbt_project.yml | 43 ++++++++++++++++++++++++++----- 2 files changed, 41 insertions(+), 7 deletions(-) diff --git a/.buildkite/scripts/run_models.sh b/.buildkite/scripts/run_models.sh index 3856fba..e7d410a 100644 --- a/.buildkite/scripts/run_models.sh +++ b/.buildkite/scripts/run_models.sh @@ -24,10 +24,13 @@ db=$1 echo `pwd` cd integration_tests +# Get schema variable name from workflow file +SCHEMA_VAR_NAME=$(grep -o 'schema_var_name: .*' ../.github/workflows/generate-docs.yml | cut -d' ' -f2 | tr -d '\r') + # Set up base variables BASE_VARS="" if [[ -n "${BUILD_SCHEMA:-}" ]]; then - BASE_VARS="\"amazon_ads_schema\": \"${BUILD_SCHEMA}\"" + BASE_VARS="\"${SCHEMA_VAR_NAME}\": \"${BUILD_SCHEMA}\"" fi # Define test scenarios - easy to add/modify diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index fd8b85a..8922c82 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -43,12 +43,43 @@ seeds: amazon_ads_integration_tests: +column_types: _fivetran_synced: "timestamp" - campaign_budget_amount: "float" - click_through_rate: "float" - keyword_bid: "float" - clicks: "float" - impressions: "float" - cost: "float" + # Seeds with reporting metrics (clicks, impressions, cost) + ad_group_level_report_data: + +column_types: + _fivetran_synced: "timestamp" + clicks: "float" + impressions: "float" + cost: "float" + campaign_level_report_data: + +column_types: + _fivetran_synced: "timestamp" + campaign_budget_amount: "float" + clicks: "float" + impressions: "float" + cost: "float" + targeting_keyword_report_data: + +column_types: + _fivetran_synced: "timestamp" + campaign_budget_amount: "float" + keyword_bid: "float" + clicks: "float" + impressions: "float" + cost: "float" + search_term_ad_keyword_report_data: + +column_types: + _fivetran_synced: "timestamp" + campaign_budget_amount: "float" + keyword_bid: "float" + clicks: "float" + impressions: "float" + cost: "float" + advertised_product_report_data: + +column_types: + _fivetran_synced: "timestamp" + campaign_budget_amount: "float" + clicks: "float" + impressions: "float" + cost: "float" flags: send_anonymous_usage_stats: False use_colors: True From 8f0fcc73bc331aa53a8d803add0726e0c0bcdd51 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Fri, 20 Mar 2026 17:50:09 -0500 Subject: [PATCH 09/41] test again --- .buildkite/scripts/run_models.sh | 38 ++-------- .buildkite/scripts/run_test_scenarios.py | 93 ++++++++++++++++++++++++ integration_tests/ci/test_scenarios.yml | 11 +++ 3 files changed, 110 insertions(+), 32 deletions(-) create mode 100644 .buildkite/scripts/run_test_scenarios.py create mode 100644 integration_tests/ci/test_scenarios.yml diff --git a/.buildkite/scripts/run_models.sh b/.buildkite/scripts/run_models.sh index e7d410a..ce86737 100644 --- a/.buildkite/scripts/run_models.sh +++ b/.buildkite/scripts/run_models.sh @@ -9,14 +9,9 @@ python3 -m venv venv . venv/bin/activate pip install --upgrade pip setuptools -# Install specific adapter for this warehouse or fall back to full requirements -if [[ -n "${DBT_ADAPTER:-}" ]]; then - echo "Installing specific dbt adapter: ${DBT_ADAPTER}" - pip install "${DBT_ADAPTER}>=1.3.0,<2.0.0" -else - echo "Installing all adapters from requirements.txt" - pip install -r integration_tests/requirements.txt -fi +# Install specific adapter for this warehouse +echo "Installing dbt adapter: ${DBT_ADAPTER}" +pip install "${DBT_ADAPTER}>=1.3.0,<2.0.0" mkdir -p ~/.dbt cp integration_tests/ci/sample.profiles.yml ~/.dbt/profiles.yml @@ -27,30 +22,9 @@ cd integration_tests # Get schema variable name from workflow file SCHEMA_VAR_NAME=$(grep -o 'schema_var_name: .*' ../.github/workflows/generate-docs.yml | cut -d' ' -f2 | tr -d '\r') -# Set up base variables -BASE_VARS="" -if [[ -n "${BUILD_SCHEMA:-}" ]]; then - BASE_VARS="\"${SCHEMA_VAR_NAME}\": \"${BUILD_SCHEMA}\"" -fi - -# Define test scenarios - easy to add/modify -declare -a test_scenarios=( - # Scenario 1: Default run - "{${BASE_VARS}}" - - # Scenario 2: Portfolio history disabled - "{${BASE_VARS}, \"amazon_ads__portfolio_history_enabled\": false}" -) - dbt deps dbt seed --target "$db" --full-refresh +dbt compile --target "$db" -# Run each test scenario -for i in "${!test_scenarios[@]}"; do - scenario_num=$((i + 1)) - vars="${test_scenarios[$i]}" - - echo "Running test scenario ${scenario_num}: ${vars}" - dbt run --target "$db" --vars "$vars" --full-refresh - dbt test --target "$db" --vars "$vars" -done +# Run test scenarios using Python script +python3 .buildkite/scripts/run_test_scenarios.py "$db" "$SCHEMA_VAR_NAME" "$BUILD_SCHEMA" diff --git a/.buildkite/scripts/run_test_scenarios.py b/.buildkite/scripts/run_test_scenarios.py new file mode 100644 index 0000000..fdb7281 --- /dev/null +++ b/.buildkite/scripts/run_test_scenarios.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +""" +Run dbt test scenarios from YAML configuration +""" + +import yaml +import json +import subprocess +import sys +import os +from pathlib import Path + + +def load_scenarios(config_file): + """Load test scenarios from YAML file""" + with open(config_file, 'r') as f: + return yaml.safe_load(f) + + +def run_dbt_command(cmd, cwd=None): + """Run a dbt command and handle errors""" + print(f"Running: {' '.join(cmd)}") + result = subprocess.run(cmd, cwd=cwd) + if result.returncode != 0: + print(f"Command failed: {' '.join(cmd)}") + return False + return True + + +def main(): + if len(sys.argv) != 4: + print("Usage: run_test_scenarios.py ") + sys.exit(1) + + target = sys.argv[1] + schema_var_name = sys.argv[2] + build_schema = sys.argv[3] + + # Load scenarios + config_file = Path('integration_tests/ci/test_scenarios.yml') + if not config_file.exists(): + print(f"Error: {config_file} not found") + sys.exit(1) + + config = load_scenarios(config_file) + + print(f"Running test scenarios for target: {target}") + print(f"Schema variable: {schema_var_name} = {build_schema}") + + # Always run default scenario first + def run_scenario(scenario_vars, scenario_name, full_refresh=True): + print(f"\n=== Running {scenario_name} ===") + + # Build vars dict + vars_dict = scenario_vars.copy() + if build_schema: + vars_dict[schema_var_name] = build_schema + + vars_json = json.dumps(vars_dict) + refresh_flag = "--full-refresh" if full_refresh else "" + print(f"Variables: {vars_json}") + print(f"Full refresh: {full_refresh}") + + # Run dbt commands + run_cmd = ['dbt', 'run', '--target', target, '--vars', vars_json] + if full_refresh: + run_cmd.append('--full-refresh') + + test_cmd = ['dbt', 'test', '--target', target, '--vars', vars_json] + + # Execute commands + if not run_dbt_command(run_cmd, cwd='integration_tests'): + print(f"dbt run failed for {scenario_name}") + sys.exit(1) + + if not run_dbt_command(test_cmd, cwd='integration_tests'): + print(f"dbt test failed for {scenario_name}") + sys.exit(1) + + # Run default scenario first (always full refresh) + run_scenario({}, "default scenario", full_refresh=True) + + # Run additional test scenarios + for i, scenario in enumerate(config.get('test_scenarios', []), 2): + scenario_vars = scenario.get('vars', {}) + full_refresh = scenario.get('full_refresh', True) # Default to true if not specified + run_scenario(scenario_vars, f"test scenario {i}", full_refresh) + + print("\n=== All test scenarios completed successfully! ===") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/integration_tests/ci/test_scenarios.yml b/integration_tests/ci/test_scenarios.yml new file mode 100644 index 0000000..540961e --- /dev/null +++ b/integration_tests/ci/test_scenarios.yml @@ -0,0 +1,11 @@ +# Additional test scenarios (default run happens automatically) +test_scenarios: + - vars: + amazon_ads__portfolio_history_enabled: false + full_refresh: false + +# Add more scenarios as needed: +# - vars: +# amazon_ads__some_other_flag: true +# amazon_ads__another_setting: "custom_value" +# full_refresh: true \ No newline at end of file From 9f6b06b737aa1ca45b0f21c891d889db0a8767d3 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Fri, 20 Mar 2026 18:00:22 -0500 Subject: [PATCH 10/41] test again --- .buildkite/hooks/pre-command | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 1cf0a29..22ea040 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -20,7 +20,7 @@ case "${BUILDKITE_STEP_KEY:-}" in "run_dbt_bigquery") echo "Setting up BigQuery credentials and dependencies" export GCLOUD_SERVICE_KEY=$(gcloud secrets versions access latest --secret="GCLOUD_SERVICE_KEY" --project="dbt-package-testing-363917") - DBT_ADAPTER="dbt-bigquery" + export DBT_ADAPTER="dbt-bigquery" ;; "run-dbt-postgres") echo "Setting up PostgreSQL credentials and dependencies" @@ -28,7 +28,7 @@ case "${BUILDKITE_STEP_KEY:-}" in export CI_POSTGRES_DBT_USER=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_USER" --project="dbt-package-testing-363917") export CI_POSTGRES_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_PASS" --project="dbt-package-testing-363917") export CI_POSTGRES_DBT_DBNAME=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_DBNAME" --project="dbt-package-testing-363917") - DBT_ADAPTER="dbt-postgres" + export DBT_ADAPTER="dbt-postgres" ;; "run_dbt_redshift") echo "Setting up Redshift credentials and dependencies" @@ -36,7 +36,7 @@ case "${BUILDKITE_STEP_KEY:-}" in export CI_REDSHIFT_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_HOST" --project="dbt-package-testing-363917") export CI_REDSHIFT_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_PASS" --project="dbt-package-testing-363917") export CI_REDSHIFT_DBT_USER=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_USER" --project="dbt-package-testing-363917") - DBT_ADAPTER="dbt-redshift" + export DBT_ADAPTER="dbt-redshift" ;; "run_dbt_snowflake") echo "Setting up Snowflake credentials and dependencies" @@ -46,7 +46,7 @@ case "${BUILDKITE_STEP_KEY:-}" in export CI_SNOWFLAKE_DBT_ROLE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_ROLE" --project="dbt-package-testing-363917") export CI_SNOWFLAKE_DBT_USER=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_USER" --project="dbt-package-testing-363917") export CI_SNOWFLAKE_DBT_WAREHOUSE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_WAREHOUSE" --project="dbt-package-testing-363917") - DBT_ADAPTER="dbt-snowflake" + export DBT_ADAPTER="dbt-snowflake" ;; "run_dbt_databricks") echo "Setting up Databricks credentials and dependencies" @@ -54,6 +54,6 @@ case "${BUILDKITE_STEP_KEY:-}" in export CI_DATABRICKS_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HTTP_PATH" --project="dbt-package-testing-363917") export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917") export CI_DATABRICKS_DBT_CATALOG=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_CATALOG" --project="dbt-package-testing-363917") - DBT_ADAPTER="dbt-databricks" + export DBT_ADAPTER="dbt-databricks" ;; esac \ No newline at end of file From 526ceadc62b4211dca5819f0cb403a196cece3d7 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Fri, 20 Mar 2026 18:11:35 -0500 Subject: [PATCH 11/41] test again --- .buildkite/hooks/pre-command | 5 ----- .buildkite/pipeline.yml | 5 +++++ 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 22ea040..2867d14 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -20,7 +20,6 @@ case "${BUILDKITE_STEP_KEY:-}" in "run_dbt_bigquery") echo "Setting up BigQuery credentials and dependencies" export GCLOUD_SERVICE_KEY=$(gcloud secrets versions access latest --secret="GCLOUD_SERVICE_KEY" --project="dbt-package-testing-363917") - export DBT_ADAPTER="dbt-bigquery" ;; "run-dbt-postgres") echo "Setting up PostgreSQL credentials and dependencies" @@ -28,7 +27,6 @@ case "${BUILDKITE_STEP_KEY:-}" in export CI_POSTGRES_DBT_USER=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_USER" --project="dbt-package-testing-363917") export CI_POSTGRES_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_PASS" --project="dbt-package-testing-363917") export CI_POSTGRES_DBT_DBNAME=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_DBNAME" --project="dbt-package-testing-363917") - export DBT_ADAPTER="dbt-postgres" ;; "run_dbt_redshift") echo "Setting up Redshift credentials and dependencies" @@ -36,7 +34,6 @@ case "${BUILDKITE_STEP_KEY:-}" in export CI_REDSHIFT_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_HOST" --project="dbt-package-testing-363917") export CI_REDSHIFT_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_PASS" --project="dbt-package-testing-363917") export CI_REDSHIFT_DBT_USER=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_USER" --project="dbt-package-testing-363917") - export DBT_ADAPTER="dbt-redshift" ;; "run_dbt_snowflake") echo "Setting up Snowflake credentials and dependencies" @@ -46,7 +43,6 @@ case "${BUILDKITE_STEP_KEY:-}" in export CI_SNOWFLAKE_DBT_ROLE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_ROLE" --project="dbt-package-testing-363917") export CI_SNOWFLAKE_DBT_USER=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_USER" --project="dbt-package-testing-363917") export CI_SNOWFLAKE_DBT_WAREHOUSE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_WAREHOUSE" --project="dbt-package-testing-363917") - export DBT_ADAPTER="dbt-snowflake" ;; "run_dbt_databricks") echo "Setting up Databricks credentials and dependencies" @@ -54,6 +50,5 @@ case "${BUILDKITE_STEP_KEY:-}" in export CI_DATABRICKS_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HTTP_PATH" --project="dbt-package-testing-363917") export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917") export CI_DATABRICKS_DBT_CATALOG=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_CATALOG" --project="dbt-package-testing-363917") - export DBT_ADAPTER="dbt-databricks" ;; esac \ No newline at end of file diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index f90c0d3..f1839d1 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -11,6 +11,7 @@ steps: environment: - "BASH_ENV=/tmp/.bashrc" - "BUILD_SCHEMA" + - "DBT_ADAPTER=dbt-postgres" - "CI_POSTGRES_DBT_DBNAME" - "CI_POSTGRES_DBT_HOST" - "CI_POSTGRES_DBT_PASS" @@ -27,6 +28,7 @@ steps: environment: - "BASH_ENV=/tmp/.bashrc" - "BUILD_SCHEMA" + - "DBT_ADAPTER=dbt-snowflake" - "CI_SNOWFLAKE_DBT_ACCOUNT" - "CI_SNOWFLAKE_DBT_DATABASE" - "CI_SNOWFLAKE_DBT_PASS" @@ -45,6 +47,7 @@ steps: environment: - "BASH_ENV=/tmp/.bashrc" - "BUILD_SCHEMA" + - "DBT_ADAPTER=dbt-bigquery" - "GCLOUD_SERVICE_KEY" commands: | bash .buildkite/scripts/run_models.sh bigquery @@ -60,6 +63,7 @@ steps: environment: - "BASH_ENV=/tmp/.bashrc" - "BUILD_SCHEMA" + - "DBT_ADAPTER=dbt-redshift" - "CI_REDSHIFT_DBT_DBNAME" - "CI_REDSHIFT_DBT_HOST" - "CI_REDSHIFT_DBT_PASS" @@ -76,6 +80,7 @@ steps: environment: - "BASH_ENV=/tmp/.bashrc" - "BUILD_SCHEMA" + - "DBT_ADAPTER=dbt-databricks" - "CI_DATABRICKS_DBT_HOST" - "CI_DATABRICKS_DBT_HTTP_PATH" - "CI_DATABRICKS_DBT_TOKEN" From c3584446c3ca5b31d28fe8b1a29a05901559eda6 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Fri, 20 Mar 2026 18:17:26 -0500 Subject: [PATCH 12/41] test again --- .buildkite/scripts/run_models.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/run_models.sh b/.buildkite/scripts/run_models.sh index ce86737..4957290 100644 --- a/.buildkite/scripts/run_models.sh +++ b/.buildkite/scripts/run_models.sh @@ -27,4 +27,4 @@ dbt seed --target "$db" --full-refresh dbt compile --target "$db" # Run test scenarios using Python script -python3 .buildkite/scripts/run_test_scenarios.py "$db" "$SCHEMA_VAR_NAME" "$BUILD_SCHEMA" +python3 ../.buildkite/scripts/run_test_scenarios.py "$db" "$SCHEMA_VAR_NAME" "$BUILD_SCHEMA" From d69c8362d28d14e1f8399224fea4ed50d46476a4 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Fri, 20 Mar 2026 18:28:21 -0500 Subject: [PATCH 13/41] test again --- .buildkite/scripts/run_test_scenarios.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/run_test_scenarios.py b/.buildkite/scripts/run_test_scenarios.py index fdb7281..cb8ff6e 100644 --- a/.buildkite/scripts/run_test_scenarios.py +++ b/.buildkite/scripts/run_test_scenarios.py @@ -37,7 +37,7 @@ def main(): build_schema = sys.argv[3] # Load scenarios - config_file = Path('integration_tests/ci/test_scenarios.yml') + config_file = Path('ci/test_scenarios.yml') if not config_file.exists(): print(f"Error: {config_file} not found") sys.exit(1) From 0938509fb3e3de82fcde9956960bdd6cc3a2b1d6 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Fri, 20 Mar 2026 18:37:15 -0500 Subject: [PATCH 14/41] test again --- .buildkite/scripts/run_test_scenarios.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.buildkite/scripts/run_test_scenarios.py b/.buildkite/scripts/run_test_scenarios.py index cb8ff6e..edc7dfd 100644 --- a/.buildkite/scripts/run_test_scenarios.py +++ b/.buildkite/scripts/run_test_scenarios.py @@ -56,7 +56,7 @@ def run_scenario(scenario_vars, scenario_name, full_refresh=True): if build_schema: vars_dict[schema_var_name] = build_schema - vars_json = json.dumps(vars_dict) + vars_json = f"'{json.dumps(vars_dict)}'" refresh_flag = "--full-refresh" if full_refresh else "" print(f"Variables: {vars_json}") print(f"Full refresh: {full_refresh}") @@ -69,11 +69,11 @@ def run_scenario(scenario_vars, scenario_name, full_refresh=True): test_cmd = ['dbt', 'test', '--target', target, '--vars', vars_json] # Execute commands - if not run_dbt_command(run_cmd, cwd='integration_tests'): + if not run_dbt_command(run_cmd): print(f"dbt run failed for {scenario_name}") sys.exit(1) - if not run_dbt_command(test_cmd, cwd='integration_tests'): + if not run_dbt_command(test_cmd): print(f"dbt test failed for {scenario_name}") sys.exit(1) From 3b59641e378f582074dfaeae943e9116b78324ea Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Fri, 20 Mar 2026 18:40:37 -0500 Subject: [PATCH 15/41] test again --- .buildkite/scripts/run_test_scenarios.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/.buildkite/scripts/run_test_scenarios.py b/.buildkite/scripts/run_test_scenarios.py index edc7dfd..6e7da49 100644 --- a/.buildkite/scripts/run_test_scenarios.py +++ b/.buildkite/scripts/run_test_scenarios.py @@ -56,17 +56,24 @@ def run_scenario(scenario_vars, scenario_name, full_refresh=True): if build_schema: vars_dict[schema_var_name] = build_schema - vars_json = f"'{json.dumps(vars_dict)}'" + # Build YAML format for dbt --vars + vars_yaml_parts = [] + for key, value in vars_dict.items(): + if isinstance(value, str): + vars_yaml_parts.append(f"{key}: {value}") + else: + vars_yaml_parts.append(f"{key}: {value}") + vars_yaml = f"{{{', '.join(vars_yaml_parts)}}}" refresh_flag = "--full-refresh" if full_refresh else "" - print(f"Variables: {vars_json}") + print(f"Variables: {vars_yaml}") print(f"Full refresh: {full_refresh}") # Run dbt commands - run_cmd = ['dbt', 'run', '--target', target, '--vars', vars_json] + run_cmd = ['dbt', 'run', '--target', target, '--vars', vars_yaml] if full_refresh: run_cmd.append('--full-refresh') - test_cmd = ['dbt', 'test', '--target', target, '--vars', vars_json] + test_cmd = ['dbt', 'test', '--target', target, '--vars', vars_yaml] # Execute commands if not run_dbt_command(run_cmd): From 9e163de2b77d8594755cdd54ddcaddef1aaed819 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Sun, 22 Mar 2026 10:43:19 -0500 Subject: [PATCH 16/41] fix dbt compile --- .buildkite/scripts/run_models.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/run_models.sh b/.buildkite/scripts/run_models.sh index 4957290..ee07d8d 100644 --- a/.buildkite/scripts/run_models.sh +++ b/.buildkite/scripts/run_models.sh @@ -24,7 +24,7 @@ SCHEMA_VAR_NAME=$(grep -o 'schema_var_name: .*' ../.github/workflows/generate-do dbt deps dbt seed --target "$db" --full-refresh -dbt compile --target "$db" +dbt compile --target "$db" --vars "{${SCHEMA_VAR_NAME}: ${BUILD_SCHEMA}}" # Run test scenarios using Python script python3 ../.buildkite/scripts/run_test_scenarios.py "$db" "$SCHEMA_VAR_NAME" "$BUILD_SCHEMA" From 94677d9dce6a337d397832232ed1ff177ac6d15e Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Sun, 22 Mar 2026 10:53:19 -0500 Subject: [PATCH 17/41] add compile message --- .buildkite/scripts/run_models.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.buildkite/scripts/run_models.sh b/.buildkite/scripts/run_models.sh index ee07d8d..86eae02 100644 --- a/.buildkite/scripts/run_models.sh +++ b/.buildkite/scripts/run_models.sh @@ -24,6 +24,7 @@ SCHEMA_VAR_NAME=$(grep -o 'schema_var_name: .*' ../.github/workflows/generate-do dbt deps dbt seed --target "$db" --full-refresh +echo "=== Running dbt compile ===" dbt compile --target "$db" --vars "{${SCHEMA_VAR_NAME}: ${BUILD_SCHEMA}}" # Run test scenarios using Python script From 57c8b596757d7bd36d158e589027c319151ce89d Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Sun, 22 Mar 2026 10:58:05 -0500 Subject: [PATCH 18/41] add compile messages --- .buildkite/scripts/run_models.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.buildkite/scripts/run_models.sh b/.buildkite/scripts/run_models.sh index 86eae02..62f530b 100644 --- a/.buildkite/scripts/run_models.sh +++ b/.buildkite/scripts/run_models.sh @@ -25,7 +25,9 @@ SCHEMA_VAR_NAME=$(grep -o 'schema_var_name: .*' ../.github/workflows/generate-do dbt deps dbt seed --target "$db" --full-refresh echo "=== Running dbt compile ===" +echo "Running: dbt compile --target \"$db\" --vars \"{${SCHEMA_VAR_NAME}: ${BUILD_SCHEMA}}\"" dbt compile --target "$db" --vars "{${SCHEMA_VAR_NAME}: ${BUILD_SCHEMA}}" +echo "✓ Successful compile" # Run test scenarios using Python script python3 ../.buildkite/scripts/run_test_scenarios.py "$db" "$SCHEMA_VAR_NAME" "$BUILD_SCHEMA" From 86cb6fbfc64107147ec7c381fbebc6d458828ce2 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Fri, 27 Mar 2026 18:43:01 -0500 Subject: [PATCH 19/41] remove schema cleanup --- .buildkite/hooks/pre-command | 2 +- .buildkite/hooks/pre-exit | 83 ------------------- .buildkite/pipeline.yml | 10 +-- .buildkite/scripts/run_models.sh | 23 +++--- .buildkite/scripts/run_test_scenarios.py | 100 ----------------------- integration_tests/ci/sample.profiles.yml | 10 +-- integration_tests/ci/test_scenarios.yml | 4 +- 7 files changed, 21 insertions(+), 211 deletions(-) delete mode 100644 .buildkite/hooks/pre-exit delete mode 100644 .buildkite/scripts/run_test_scenarios.py diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 2867d14..8fd7eec 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -21,7 +21,7 @@ case "${BUILDKITE_STEP_KEY:-}" in echo "Setting up BigQuery credentials and dependencies" export GCLOUD_SERVICE_KEY=$(gcloud secrets versions access latest --secret="GCLOUD_SERVICE_KEY" --project="dbt-package-testing-363917") ;; - "run-dbt-postgres") + "run_dbt_postgres") echo "Setting up PostgreSQL credentials and dependencies" export CI_POSTGRES_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_HOST" --project="dbt-package-testing-363917") export CI_POSTGRES_DBT_USER=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_USER" --project="dbt-package-testing-363917") diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit deleted file mode 100644 index 896b65a..0000000 --- a/.buildkite/hooks/pre-exit +++ /dev/null @@ -1,83 +0,0 @@ -#!/bin/bash - -# Cleanup hook - drops the build schema when the agent exits -# Runs regardless of build success, failure, cancellation, or interruption - -set +e # Don't exit on errors during cleanup - -echo "Cleaning up build schema: ${BUILD_SCHEMA:-not_set}" - -# Only proceed if BUILD_SCHEMA is set -if [[ -z "${BUILD_SCHEMA}" ]]; then - echo "BUILD_SCHEMA not set, skipping cleanup" - exit 0 -fi - -echo "Running schema cleanup..." - -# Validate schema name (prevent injection) -if [[ ! "${BUILD_SCHEMA}" =~ ^[a-zA-Z0-9_]+$ ]]; then - echo "Invalid schema name format" - exit 0 -fi - -# Schemas to clean -PACKAGE_NAME="${DBT_PACKAGE_NAME:-amazon_ads}" -SCHEMA1="${BUILD_SCHEMA}" -SCHEMA2="${BUILD_SCHEMA}_${PACKAGE_NAME}_dev" - -echo "Cleaning schemas: ${SCHEMA1}, ${SCHEMA2}" -echo "Step key: ${BUILDKITE_STEP_KEY:-unknown}" - -# Clean up only the warehouse for this specific step -case "${BUILDKITE_STEP_KEY:-}" in - "run_dbt_bigquery") - if [[ -n "${GCLOUD_SERVICE_KEY:-}" ]] && command -v bq >/dev/null 2>&1; then - echo "${GCLOUD_SERVICE_KEY}" | gcloud auth activate-service-account --key-file=- --quiet 2>/dev/null - bq --quiet rm -r -f -d "dbt-package-testing:${SCHEMA1}" 2>/dev/null || true - bq --quiet rm -r -f -d "dbt-package-testing:${SCHEMA2}" 2>/dev/null || true - echo "BigQuery cleanup completed" - fi - ;; - "run-dbt-postgres") - if [[ -n "${CI_POSTGRES_DBT_HOST:-}" ]] && command -v psql >/dev/null 2>&1; then - PGPASSWORD="${CI_POSTGRES_DBT_PASS}" psql -h "${CI_POSTGRES_DBT_HOST}" -U "${CI_POSTGRES_DBT_USER}" -d "${CI_POSTGRES_DBT_DBNAME}" -c "DROP SCHEMA IF EXISTS \"${SCHEMA1}\" CASCADE;" -q 2>/dev/null || true - PGPASSWORD="${CI_POSTGRES_DBT_PASS}" psql -h "${CI_POSTGRES_DBT_HOST}" -U "${CI_POSTGRES_DBT_USER}" -d "${CI_POSTGRES_DBT_DBNAME}" -c "DROP SCHEMA IF EXISTS \"${SCHEMA2}\" CASCADE;" -q 2>/dev/null || true - echo "PostgreSQL cleanup completed" - fi - ;; - "run_dbt_redshift") - if [[ -n "${CI_REDSHIFT_DBT_HOST:-}" ]] && command -v psql >/dev/null 2>&1; then - PGPASSWORD="${CI_REDSHIFT_DBT_PASS}" psql -h "${CI_REDSHIFT_DBT_HOST}" -U "${CI_REDSHIFT_DBT_USER}" -d "${CI_REDSHIFT_DBT_DBNAME}" -c "DROP SCHEMA IF EXISTS \"${SCHEMA1}\" CASCADE;" -q 2>/dev/null || true - PGPASSWORD="${CI_REDSHIFT_DBT_PASS}" psql -h "${CI_REDSHIFT_DBT_HOST}" -U "${CI_REDSHIFT_DBT_USER}" -d "${CI_REDSHIFT_DBT_DBNAME}" -c "DROP SCHEMA IF EXISTS \"${SCHEMA2}\" CASCADE;" -q 2>/dev/null || true - echo "Redshift cleanup completed" - fi - ;; - "run_dbt_snowflake") - if [[ -n "${CI_SNOWFLAKE_DBT_ACCOUNT:-}" ]] && command -v snowsql >/dev/null 2>&1; then - snowsql -a "${CI_SNOWFLAKE_DBT_ACCOUNT}" -u "${CI_SNOWFLAKE_DBT_USER}" -p "${CI_SNOWFLAKE_DBT_PASS}" \ - -r "${CI_SNOWFLAKE_DBT_ROLE}" -w "${CI_SNOWFLAKE_DBT_WAREHOUSE}" -d "${CI_SNOWFLAKE_DBT_DATABASE}" \ - -q "DROP SCHEMA IF EXISTS \"${SCHEMA1}\" CASCADE;" --quiet 2>/dev/null || true - snowsql -a "${CI_SNOWFLAKE_DBT_ACCOUNT}" -u "${CI_SNOWFLAKE_DBT_USER}" -p "${CI_SNOWFLAKE_DBT_PASS}" \ - -r "${CI_SNOWFLAKE_DBT_ROLE}" -w "${CI_SNOWFLAKE_DBT_WAREHOUSE}" -d "${CI_SNOWFLAKE_DBT_DATABASE}" \ - -q "DROP SCHEMA IF EXISTS \"${SCHEMA2}\" CASCADE;" --quiet 2>/dev/null || true - echo "Snowflake cleanup completed" - fi - ;; - "run_dbt_databricks") - if [[ -n "${CI_DATABRICKS_DBT_HOST:-}" ]] && command -v curl >/dev/null 2>&1; then - curl -s -X POST -H "Authorization: Bearer ${CI_DATABRICKS_DBT_TOKEN}" -H "Content-Type: application/json" \ - "${CI_DATABRICKS_DBT_HOST}/api/2.0/sql/statements" \ - -d "{\"statement\": \"DROP SCHEMA IF EXISTS \\\"${CI_DATABRICKS_DBT_CATALOG}\\\".\\\"${SCHEMA1}\\\" CASCADE\"}" >/dev/null 2>&1 || true - curl -s -X POST -H "Authorization: Bearer ${CI_DATABRICKS_DBT_TOKEN}" -H "Content-Type: application/json" \ - "${CI_DATABRICKS_DBT_HOST}/api/2.0/sql/statements" \ - -d "{\"statement\": \"DROP SCHEMA IF EXISTS \\\"${CI_DATABRICKS_DBT_CATALOG}\\\".\\\"${SCHEMA2}\\\" CASCADE\"}" >/dev/null 2>&1 || true - echo "Databricks cleanup completed" - fi - ;; - *) - echo "Unknown or no step key, skipping cleanup" - ;; -esac - -echo "Schema cleanup completed" \ No newline at end of file diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index f1839d1..3693ffb 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,9 +1,6 @@ -env: - DBT_PACKAGE_NAME: "amazon_ads" - steps: - label: ":postgres: Run Tests - Postgres" - key: "run-dbt-postgres" + key: "run_dbt_postgres" plugins: - docker#v3.13.0: image: "python:3.13" @@ -11,7 +8,6 @@ steps: environment: - "BASH_ENV=/tmp/.bashrc" - "BUILD_SCHEMA" - - "DBT_ADAPTER=dbt-postgres" - "CI_POSTGRES_DBT_DBNAME" - "CI_POSTGRES_DBT_HOST" - "CI_POSTGRES_DBT_PASS" @@ -28,7 +24,6 @@ steps: environment: - "BASH_ENV=/tmp/.bashrc" - "BUILD_SCHEMA" - - "DBT_ADAPTER=dbt-snowflake" - "CI_SNOWFLAKE_DBT_ACCOUNT" - "CI_SNOWFLAKE_DBT_DATABASE" - "CI_SNOWFLAKE_DBT_PASS" @@ -47,7 +42,6 @@ steps: environment: - "BASH_ENV=/tmp/.bashrc" - "BUILD_SCHEMA" - - "DBT_ADAPTER=dbt-bigquery" - "GCLOUD_SERVICE_KEY" commands: | bash .buildkite/scripts/run_models.sh bigquery @@ -63,7 +57,6 @@ steps: environment: - "BASH_ENV=/tmp/.bashrc" - "BUILD_SCHEMA" - - "DBT_ADAPTER=dbt-redshift" - "CI_REDSHIFT_DBT_DBNAME" - "CI_REDSHIFT_DBT_HOST" - "CI_REDSHIFT_DBT_PASS" @@ -80,7 +73,6 @@ steps: environment: - "BASH_ENV=/tmp/.bashrc" - "BUILD_SCHEMA" - - "DBT_ADAPTER=dbt-databricks" - "CI_DATABRICKS_DBT_HOST" - "CI_DATABRICKS_DBT_HTTP_PATH" - "CI_DATABRICKS_DBT_TOKEN" diff --git a/.buildkite/scripts/run_models.sh b/.buildkite/scripts/run_models.sh index 62f530b..3bc73f2 100644 --- a/.buildkite/scripts/run_models.sh +++ b/.buildkite/scripts/run_models.sh @@ -9,9 +9,12 @@ python3 -m venv venv . venv/bin/activate pip install --upgrade pip setuptools +# Determine warehouse from step key pattern +WAREHOUSE=${BUILDKITE_STEP_KEY#run_dbt_} + # Install specific adapter for this warehouse -echo "Installing dbt adapter: ${DBT_ADAPTER}" -pip install "${DBT_ADAPTER}>=1.3.0,<2.0.0" +echo "Installing dbt adapter: dbt-${WAREHOUSE}" +pip install "dbt-${WAREHOUSE}>=1.3.0,<2.0.0" mkdir -p ~/.dbt cp integration_tests/ci/sample.profiles.yml ~/.dbt/profiles.yml @@ -19,15 +22,13 @@ db=$1 echo `pwd` cd integration_tests -# Get schema variable name from workflow file -SCHEMA_VAR_NAME=$(grep -o 'schema_var_name: .*' ../.github/workflows/generate-docs.yml | cut -d' ' -f2 | tr -d '\r') +SCHEMA_VAR_NAME='amazon_ads_schema' -dbt deps -dbt seed --target "$db" --full-refresh -echo "=== Running dbt compile ===" -echo "Running: dbt compile --target \"$db\" --vars \"{${SCHEMA_VAR_NAME}: ${BUILD_SCHEMA}}\"" -dbt compile --target "$db" --vars "{${SCHEMA_VAR_NAME}: ${BUILD_SCHEMA}}" -echo "✓ Successful compile" +# Fetch central test scenario script +SCRIPT_VERSION="feature/buildkite-scripts" # Use feature/buildkite-scripts branch +mkdir -p ../.buildkite/scripts +curl -f -s -o ../.buildkite/scripts/run_test_scenarios.py \ + "https://raw.githubusercontent.com/fivetran/dbt_package_automations/${SCRIPT_VERSION}/buildkite/scripts/run_test_scenarios.py" -# Run test scenarios using Python script +# Run test scenarios using Python script (includes deps, seed, compile) python3 ../.buildkite/scripts/run_test_scenarios.py "$db" "$SCHEMA_VAR_NAME" "$BUILD_SCHEMA" diff --git a/.buildkite/scripts/run_test_scenarios.py b/.buildkite/scripts/run_test_scenarios.py deleted file mode 100644 index 6e7da49..0000000 --- a/.buildkite/scripts/run_test_scenarios.py +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env python3 -""" -Run dbt test scenarios from YAML configuration -""" - -import yaml -import json -import subprocess -import sys -import os -from pathlib import Path - - -def load_scenarios(config_file): - """Load test scenarios from YAML file""" - with open(config_file, 'r') as f: - return yaml.safe_load(f) - - -def run_dbt_command(cmd, cwd=None): - """Run a dbt command and handle errors""" - print(f"Running: {' '.join(cmd)}") - result = subprocess.run(cmd, cwd=cwd) - if result.returncode != 0: - print(f"Command failed: {' '.join(cmd)}") - return False - return True - - -def main(): - if len(sys.argv) != 4: - print("Usage: run_test_scenarios.py ") - sys.exit(1) - - target = sys.argv[1] - schema_var_name = sys.argv[2] - build_schema = sys.argv[3] - - # Load scenarios - config_file = Path('ci/test_scenarios.yml') - if not config_file.exists(): - print(f"Error: {config_file} not found") - sys.exit(1) - - config = load_scenarios(config_file) - - print(f"Running test scenarios for target: {target}") - print(f"Schema variable: {schema_var_name} = {build_schema}") - - # Always run default scenario first - def run_scenario(scenario_vars, scenario_name, full_refresh=True): - print(f"\n=== Running {scenario_name} ===") - - # Build vars dict - vars_dict = scenario_vars.copy() - if build_schema: - vars_dict[schema_var_name] = build_schema - - # Build YAML format for dbt --vars - vars_yaml_parts = [] - for key, value in vars_dict.items(): - if isinstance(value, str): - vars_yaml_parts.append(f"{key}: {value}") - else: - vars_yaml_parts.append(f"{key}: {value}") - vars_yaml = f"{{{', '.join(vars_yaml_parts)}}}" - refresh_flag = "--full-refresh" if full_refresh else "" - print(f"Variables: {vars_yaml}") - print(f"Full refresh: {full_refresh}") - - # Run dbt commands - run_cmd = ['dbt', 'run', '--target', target, '--vars', vars_yaml] - if full_refresh: - run_cmd.append('--full-refresh') - - test_cmd = ['dbt', 'test', '--target', target, '--vars', vars_yaml] - - # Execute commands - if not run_dbt_command(run_cmd): - print(f"dbt run failed for {scenario_name}") - sys.exit(1) - - if not run_dbt_command(test_cmd): - print(f"dbt test failed for {scenario_name}") - sys.exit(1) - - # Run default scenario first (always full refresh) - run_scenario({}, "default scenario", full_refresh=True) - - # Run additional test scenarios - for i, scenario in enumerate(config.get('test_scenarios', []), 2): - scenario_vars = scenario.get('vars', {}) - full_refresh = scenario.get('full_refresh', True) # Default to true if not specified - run_scenario(scenario_vars, f"test scenario {i}", full_refresh) - - print("\n=== All test scenarios completed successfully! ===") - - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml index db201ec..deac440 100644 --- a/integration_tests/ci/sample.profiles.yml +++ b/integration_tests/ci/sample.profiles.yml @@ -12,13 +12,13 @@ integration_tests: pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}" dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}" port: 5439 - schema: "{{ env_var('BUILD_SCHEMA', 'amazon_ads_integration_tests') }}" + schema: "{{ env_var('BUILD_SCHEMA') }}" threads: 8 bigquery: type: bigquery method: service-account-json project: 'dbt-package-testing' - schema: "{{ env_var('BUILD_SCHEMA', 'amazon_ads_integration_tests') }}" + schema: "{{ env_var('BUILD_SCHEMA') }}" threads: 8 keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}" snowflake: @@ -29,7 +29,7 @@ integration_tests: role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}" database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}" warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}" - schema: "{{ env_var('BUILD_SCHEMA', 'amazon_ads_integration_tests') }}" + schema: "{{ env_var('BUILD_SCHEMA') }}" threads: 8 postgres: type: postgres @@ -38,13 +38,13 @@ integration_tests: pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}" dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}" port: 5432 - schema: "{{ env_var('BUILD_SCHEMA', 'amazon_ads_integration_tests') }}" + schema: "{{ env_var('BUILD_SCHEMA') }}" threads: 8 databricks: catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}" host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}" http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}" - schema: "{{ env_var('BUILD_SCHEMA', 'amazon_ads_integration_tests') }}" + schema: "{{ env_var('BUILD_SCHEMA') }}" threads: 8 token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}" type: databricks diff --git a/integration_tests/ci/test_scenarios.yml b/integration_tests/ci/test_scenarios.yml index 540961e..53c8618 100644 --- a/integration_tests/ci/test_scenarios.yml +++ b/integration_tests/ci/test_scenarios.yml @@ -2,10 +2,10 @@ test_scenarios: - vars: amazon_ads__portfolio_history_enabled: false - full_refresh: false + include_incremental: true # Add more scenarios as needed: # - vars: # amazon_ads__some_other_flag: true # amazon_ads__another_setting: "custom_value" -# full_refresh: true \ No newline at end of file +# include_incremental: false \ No newline at end of file From e467b3beea9d01986c14dad0fe759d65eb34a95c Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Fri, 27 Mar 2026 18:47:25 -0500 Subject: [PATCH 20/41] add auto retry --- .buildkite/pipeline.yml | 20 ++++++++++++++++++++ .buildkite/scripts/run_models.sh | 6 +++--- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 3693ffb..232fd09 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,6 +1,10 @@ steps: - label: ":postgres: Run Tests - Postgres" key: "run_dbt_postgres" + retry: + automatic: + - exit_status: -1 # Timeout + limit: 1 plugins: - docker#v3.13.0: image: "python:3.13" @@ -17,6 +21,10 @@ steps: - label: ":snowflake-db: Run Tests - Snowflake" key: "run_dbt_snowflake" + retry: + automatic: + - exit_status: -1 # Timeout + limit: 1 plugins: - docker#v3.13.0: image: "python:3.13" @@ -35,6 +43,10 @@ steps: - label: ":gcloud: Run Tests - BigQuery" key: "run_dbt_bigquery" + retry: + automatic: + - exit_status: -1 # Timeout + limit: 1 plugins: - docker#v3.13.0: image: "python:3.13" @@ -50,6 +62,10 @@ steps: key: "run_dbt_redshift" concurrency: 3 concurrency_group: "warehouse/redshift" + retry: + automatic: + - exit_status: -1 # Timeout + limit: 1 plugins: - docker#v3.13.0: image: "python:3.13" @@ -66,6 +82,10 @@ steps: - label: ":databricks: Run Tests - Databricks" key: "run_dbt_databricks" + retry: + automatic: + - exit_status: -1 # Timeout + limit: 1 plugins: - docker#v3.13.0: image: "python:3.13" diff --git a/.buildkite/scripts/run_models.sh b/.buildkite/scripts/run_models.sh index 3bc73f2..320b1da 100644 --- a/.buildkite/scripts/run_models.sh +++ b/.buildkite/scripts/run_models.sh @@ -9,8 +9,9 @@ python3 -m venv venv . venv/bin/activate pip install --upgrade pip setuptools -# Determine warehouse from step key pattern -WAREHOUSE=${BUILDKITE_STEP_KEY#run_dbt_} +# Get warehouse from command line argument +db=$1 +WAREHOUSE=$db # Install specific adapter for this warehouse echo "Installing dbt adapter: dbt-${WAREHOUSE}" @@ -18,7 +19,6 @@ pip install "dbt-${WAREHOUSE}>=1.3.0,<2.0.0" mkdir -p ~/.dbt cp integration_tests/ci/sample.profiles.yml ~/.dbt/profiles.yml -db=$1 echo `pwd` cd integration_tests From 6baae089467a78856da5271a92b6f81ace1ffbd6 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Fri, 27 Mar 2026 18:54:13 -0500 Subject: [PATCH 21/41] debug --- .buildkite/scripts/run_models.sh | 38 +++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/.buildkite/scripts/run_models.sh b/.buildkite/scripts/run_models.sh index 320b1da..11f9e0a 100644 --- a/.buildkite/scripts/run_models.sh +++ b/.buildkite/scripts/run_models.sh @@ -16,19 +16,45 @@ WAREHOUSE=$db # Install specific adapter for this warehouse echo "Installing dbt adapter: dbt-${WAREHOUSE}" pip install "dbt-${WAREHOUSE}>=1.3.0,<2.0.0" + +echo "Creating dbt config directory..." mkdir -p ~/.dbt -cp integration_tests/ci/sample.profiles.yml ~/.dbt/profiles.yml -echo `pwd` -cd integration_tests +echo "Copying profiles.yml..." +if [ -f "integration_tests/ci/sample.profiles.yml" ]; then + cp integration_tests/ci/sample.profiles.yml ~/.dbt/profiles.yml + echo "Profiles.yml copied successfully" +else + echo "ERROR: sample.profiles.yml not found!" + exit 1 +fi + +echo "Current directory: $(pwd)" +echo "Changing to integration_tests directory..." +if [ -d "integration_tests" ]; then + cd integration_tests + echo "Successfully changed to integration_tests directory" +else + echo "ERROR: integration_tests directory not found!" + exit 1 +fi SCHEMA_VAR_NAME='amazon_ads_schema' # Fetch central test scenario script -SCRIPT_VERSION="feature/buildkite-scripts" # Use feature/buildkite-scripts branch +echo "Creating scripts directory..." mkdir -p ../.buildkite/scripts -curl -f -s -o ../.buildkite/scripts/run_test_scenarios.py \ - "https://raw.githubusercontent.com/fivetran/dbt_package_automations/${SCRIPT_VERSION}/buildkite/scripts/run_test_scenarios.py" + +echo "Fetching test scenario script from feature/buildkite-scripts branch..." +SCRIPT_URL="https://raw.githubusercontent.com/fivetran/dbt_package_automations/refs/heads/feature/buildkite-scripts/.buildkite/scripts/run_test_scenarios.py" +echo "URL: ${SCRIPT_URL}" + +if curl -f -s -o ../.buildkite/scripts/run_test_scenarios.py "${SCRIPT_URL}"; then + echo "Successfully fetched test scenario script" +else + echo "ERROR: Failed to fetch test scenario script (exit code: $?)" + exit 1 +fi # Run test scenarios using Python script (includes deps, seed, compile) python3 ../.buildkite/scripts/run_test_scenarios.py "$db" "$SCHEMA_VAR_NAME" "$BUILD_SCHEMA" From 399164aa2483672dde6b73c5f43a2e4203744b09 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Fri, 27 Mar 2026 18:55:35 -0500 Subject: [PATCH 22/41] debug --- .buildkite/scripts/run_models.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.buildkite/scripts/run_models.sh b/.buildkite/scripts/run_models.sh index 11f9e0a..f641f8a 100644 --- a/.buildkite/scripts/run_models.sh +++ b/.buildkite/scripts/run_models.sh @@ -9,13 +9,12 @@ python3 -m venv venv . venv/bin/activate pip install --upgrade pip setuptools -# Get warehouse from command line argument +# Get database from command line argument db=$1 -WAREHOUSE=$db -# Install specific adapter for this warehouse -echo "Installing dbt adapter: dbt-${WAREHOUSE}" -pip install "dbt-${WAREHOUSE}>=1.3.0,<2.0.0" +# Install specific adapter for this database +echo "Installing dbt adapter: dbt-${db}" +pip install "dbt-${db}>=1.3.0,<2.0.0" echo "Creating dbt config directory..." mkdir -p ~/.dbt From bdc97ab72e3b2917f61ba6b2449a47e44397bd5e Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Fri, 27 Mar 2026 19:10:07 -0500 Subject: [PATCH 23/41] update generated schema name --- .buildkite/hooks/pre-command | 2 +- .buildkite/scripts/run_models.sh | 36 ++++---------------------------- 2 files changed, 5 insertions(+), 33 deletions(-) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 8fd7eec..30a4402 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -9,7 +9,7 @@ BUILD_NUMBER="${BUILDKITE_BUILD_NUMBER:-0}" # Create unique schema name using commit SHA and build number COMMIT_SHA=$(echo "${BUILDKITE_COMMIT:-unknown}" | cut -c1-7) -BUILD_SCHEMA=$(echo "bk_${COMMIT_SHA}_${BUILD_NUMBER}" | cut -c1-63 | sed 's/_$//') +BUILD_SCHEMA=$(echo "zz_bk_integration_tests_${COMMIT_SHA}_${BUILD_NUMBER}" | cut -c1-63 | sed 's/_$//') export BUILD_SCHEMA echo "Build schema: ${BUILD_SCHEMA}" diff --git a/.buildkite/scripts/run_models.sh b/.buildkite/scripts/run_models.sh index f641f8a..f2ec266 100644 --- a/.buildkite/scripts/run_models.sh +++ b/.buildkite/scripts/run_models.sh @@ -15,45 +15,17 @@ db=$1 # Install specific adapter for this database echo "Installing dbt adapter: dbt-${db}" pip install "dbt-${db}>=1.3.0,<2.0.0" - -echo "Creating dbt config directory..." mkdir -p ~/.dbt +cp integration_tests/ci/sample.profiles.yml ~/.dbt/profiles.yml -echo "Copying profiles.yml..." -if [ -f "integration_tests/ci/sample.profiles.yml" ]; then - cp integration_tests/ci/sample.profiles.yml ~/.dbt/profiles.yml - echo "Profiles.yml copied successfully" -else - echo "ERROR: sample.profiles.yml not found!" - exit 1 -fi - -echo "Current directory: $(pwd)" -echo "Changing to integration_tests directory..." -if [ -d "integration_tests" ]; then - cd integration_tests - echo "Successfully changed to integration_tests directory" -else - echo "ERROR: integration_tests directory not found!" - exit 1 -fi +cd integration_tests SCHEMA_VAR_NAME='amazon_ads_schema' # Fetch central test scenario script -echo "Creating scripts directory..." mkdir -p ../.buildkite/scripts - -echo "Fetching test scenario script from feature/buildkite-scripts branch..." -SCRIPT_URL="https://raw.githubusercontent.com/fivetran/dbt_package_automations/refs/heads/feature/buildkite-scripts/.buildkite/scripts/run_test_scenarios.py" -echo "URL: ${SCRIPT_URL}" - -if curl -f -s -o ../.buildkite/scripts/run_test_scenarios.py "${SCRIPT_URL}"; then - echo "Successfully fetched test scenario script" -else - echo "ERROR: Failed to fetch test scenario script (exit code: $?)" - exit 1 -fi +curl -f -s -o ../.buildkite/scripts/run_test_scenarios.py \ + "https://raw.githubusercontent.com/fivetran/dbt_package_automations/refs/heads/feature/buildkite-scripts/.buildkite/scripts/run_test_scenarios.py" # Run test scenarios using Python script (includes deps, seed, compile) python3 ../.buildkite/scripts/run_test_scenarios.py "$db" "$SCHEMA_VAR_NAME" "$BUILD_SCHEMA" From 29db7c65e563039501c6e21f34a91dc4034e56bc Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Tue, 31 Mar 2026 19:41:22 -0500 Subject: [PATCH 24/41] update where schema var name passed --- .buildkite/scripts/run_models.sh | 4 +--- integration_tests/ci/test_scenarios.yml | 4 ++++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.buildkite/scripts/run_models.sh b/.buildkite/scripts/run_models.sh index f2ec266..928f039 100644 --- a/.buildkite/scripts/run_models.sh +++ b/.buildkite/scripts/run_models.sh @@ -20,12 +20,10 @@ cp integration_tests/ci/sample.profiles.yml ~/.dbt/profiles.yml cd integration_tests -SCHEMA_VAR_NAME='amazon_ads_schema' - # Fetch central test scenario script mkdir -p ../.buildkite/scripts curl -f -s -o ../.buildkite/scripts/run_test_scenarios.py \ "https://raw.githubusercontent.com/fivetran/dbt_package_automations/refs/heads/feature/buildkite-scripts/.buildkite/scripts/run_test_scenarios.py" # Run test scenarios using Python script (includes deps, seed, compile) -python3 ../.buildkite/scripts/run_test_scenarios.py "$db" "$SCHEMA_VAR_NAME" "$BUILD_SCHEMA" +python3 ../.buildkite/scripts/run_test_scenarios.py "$db" "$BUILD_SCHEMA" diff --git a/integration_tests/ci/test_scenarios.yml b/integration_tests/ci/test_scenarios.yml index 53c8618..16973eb 100644 --- a/integration_tests/ci/test_scenarios.yml +++ b/integration_tests/ci/test_scenarios.yml @@ -1,3 +1,7 @@ +# Name of the dbt variable used to set the schema where data lives +# This is the variable name you use in dbt (e.g., --vars amazon_ads_schema: my_schema) +schema_variable_name: "amazon_ads_schema" + # Additional test scenarios (default run happens automatically) test_scenarios: - vars: From a03251be3885c98f1fe46ff7e2d111c302564f86 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Tue, 31 Mar 2026 20:14:49 -0500 Subject: [PATCH 25/41] add comments --- .buildkite/scripts/run_models.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.buildkite/scripts/run_models.sh b/.buildkite/scripts/run_models.sh index 928f039..cdc358c 100644 --- a/.buildkite/scripts/run_models.sh +++ b/.buildkite/scripts/run_models.sh @@ -26,4 +26,5 @@ curl -f -s -o ../.buildkite/scripts/run_test_scenarios.py \ "https://raw.githubusercontent.com/fivetran/dbt_package_automations/refs/heads/feature/buildkite-scripts/.buildkite/scripts/run_test_scenarios.py" # Run test scenarios using Python script (includes deps, seed, compile) +# Test parameters and scenarios are configured in: integration_tests/ci/test_scenarios.yml python3 ../.buildkite/scripts/run_test_scenarios.py "$db" "$BUILD_SCHEMA" From e0eae75423046d32e31b90ca11f262abb7f80049 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Thu, 16 Apr 2026 11:15:03 -0500 Subject: [PATCH 26/41] update names --- .buildkite/scripts/run_models.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.buildkite/scripts/run_models.sh b/.buildkite/scripts/run_models.sh index cdc358c..a73ce6c 100644 --- a/.buildkite/scripts/run_models.sh +++ b/.buildkite/scripts/run_models.sh @@ -26,5 +26,8 @@ curl -f -s -o ../.buildkite/scripts/run_test_scenarios.py \ "https://raw.githubusercontent.com/fivetran/dbt_package_automations/refs/heads/feature/buildkite-scripts/.buildkite/scripts/run_test_scenarios.py" # Run test scenarios using Python script (includes deps, seed, compile) +<<<<<<< Updated upstream # Test parameters and scenarios are configured in: integration_tests/ci/test_scenarios.yml +======= +>>>>>>> Stashed changes python3 ../.buildkite/scripts/run_test_scenarios.py "$db" "$BUILD_SCHEMA" From 3453ff2ba79feef10e027bf7782cd420b1873611 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Thu, 16 Apr 2026 11:21:57 -0500 Subject: [PATCH 27/41] add test names --- .buildkite/scripts/run_models.sh | 3 --- integration_tests/ci/test_scenarios.yml | 9 ++++++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.buildkite/scripts/run_models.sh b/.buildkite/scripts/run_models.sh index a73ce6c..cdc358c 100644 --- a/.buildkite/scripts/run_models.sh +++ b/.buildkite/scripts/run_models.sh @@ -26,8 +26,5 @@ curl -f -s -o ../.buildkite/scripts/run_test_scenarios.py \ "https://raw.githubusercontent.com/fivetran/dbt_package_automations/refs/heads/feature/buildkite-scripts/.buildkite/scripts/run_test_scenarios.py" # Run test scenarios using Python script (includes deps, seed, compile) -<<<<<<< Updated upstream # Test parameters and scenarios are configured in: integration_tests/ci/test_scenarios.yml -======= ->>>>>>> Stashed changes python3 ../.buildkite/scripts/run_test_scenarios.py "$db" "$BUILD_SCHEMA" diff --git a/integration_tests/ci/test_scenarios.yml b/integration_tests/ci/test_scenarios.yml index 16973eb..1c1192b 100644 --- a/integration_tests/ci/test_scenarios.yml +++ b/integration_tests/ci/test_scenarios.yml @@ -1,15 +1,18 @@ # Name of the dbt variable used to set the schema where data lives # This is the variable name you use in dbt (e.g., --vars amazon_ads_schema: my_schema) schema_variable_name: "amazon_ads_schema" +default_include_incremental: false # Whether default scenario should test incremental runs # Additional test scenarios (default run happens automatically) test_scenarios: - - vars: + - name: "portfolio history disabled" + vars: amazon_ads__portfolio_history_enabled: false - include_incremental: true + include_incremental: false # Add more scenarios as needed: -# - vars: +# - name: "some other scenario" +# vars: # amazon_ads__some_other_flag: true # amazon_ads__another_setting: "custom_value" # include_incremental: false \ No newline at end of file From d58904238bcb60f608c79c4f915de71a60264480 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Thu, 16 Apr 2026 17:39:16 -0500 Subject: [PATCH 28/41] add sql warehosue --- .buildkite/hooks/pre-command | 9 ++++++++- integration_tests/ci/test_scenarios.yml | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 30a4402..98a87f1 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -9,7 +9,14 @@ BUILD_NUMBER="${BUILDKITE_BUILD_NUMBER:-0}" # Create unique schema name using commit SHA and build number COMMIT_SHA=$(echo "${BUILDKITE_COMMIT:-unknown}" | cut -c1-7) -BUILD_SCHEMA=$(echo "zz_bk_integration_tests_${COMMIT_SHA}_${BUILD_NUMBER}" | cut -c1-63 | sed 's/_$//') + +# Add databricks-specific identifier for SQL warehouse to differentiate from all purpose cluster +if [[ "${BUILDKITE_STEP_KEY:-}" == "run_dbt_databricks_sql" ]]; then + BUILD_SCHEMA=$(echo "zz_bk_integration_tests_sql_${COMMIT_SHA}_${BUILD_NUMBER}" | cut -c1-63 | sed 's/_$//') +else + BUILD_SCHEMA=$(echo "zz_bk_integration_tests_${COMMIT_SHA}_${BUILD_NUMBER}" | cut -c1-63 | sed 's/_$//') +fi + export BUILD_SCHEMA echo "Build schema: ${BUILD_SCHEMA}" diff --git a/integration_tests/ci/test_scenarios.yml b/integration_tests/ci/test_scenarios.yml index 1c1192b..00e7ff4 100644 --- a/integration_tests/ci/test_scenarios.yml +++ b/integration_tests/ci/test_scenarios.yml @@ -1,5 +1,5 @@ # Name of the dbt variable used to set the schema where data lives -# This is the variable name you use in dbt (e.g., --vars amazon_ads_schema: my_schema) +# This is the variable name you use in dbt (e.g., amazon_ads_schema) schema_variable_name: "amazon_ads_schema" default_include_incremental: false # Whether default scenario should test incremental runs From cd7635fd269843d9cb2079b4cfeec46e9c81c4d5 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Fri, 17 Apr 2026 12:56:39 -0500 Subject: [PATCH 29/41] adjust vars --- .buildkite/scripts/run_models.sh | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.buildkite/scripts/run_models.sh b/.buildkite/scripts/run_models.sh index cdc358c..0718614 100644 --- a/.buildkite/scripts/run_models.sh +++ b/.buildkite/scripts/run_models.sh @@ -9,12 +9,9 @@ python3 -m venv venv . venv/bin/activate pip install --upgrade pip setuptools -# Get database from command line argument -db=$1 - # Install specific adapter for this database -echo "Installing dbt adapter: dbt-${db}" -pip install "dbt-${db}>=1.3.0,<2.0.0" +echo "Installing dbt adapter: dbt-$1" +pip install "dbt-$1>=1.3.0,<2.0.0" mkdir -p ~/.dbt cp integration_tests/ci/sample.profiles.yml ~/.dbt/profiles.yml From 90293dedc8896d3b0ce276c69bf7c102502fe17c Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Fri, 17 Apr 2026 13:04:01 -0500 Subject: [PATCH 30/41] adjust vars --- .buildkite/scripts/run_models.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.buildkite/scripts/run_models.sh b/.buildkite/scripts/run_models.sh index 0718614..d8f1cd2 100644 --- a/.buildkite/scripts/run_models.sh +++ b/.buildkite/scripts/run_models.sh @@ -10,8 +10,8 @@ python3 -m venv venv pip install --upgrade pip setuptools # Install specific adapter for this database -echo "Installing dbt adapter: dbt-$1" -pip install "dbt-$1>=1.3.0,<2.0.0" +echo "Installing dbt adapter: dbt-${1}" +pip install "dbt-${1}>=1.3.0,<2.0.0" mkdir -p ~/.dbt cp integration_tests/ci/sample.profiles.yml ~/.dbt/profiles.yml @@ -24,4 +24,4 @@ curl -f -s -o ../.buildkite/scripts/run_test_scenarios.py \ # Run test scenarios using Python script (includes deps, seed, compile) # Test parameters and scenarios are configured in: integration_tests/ci/test_scenarios.yml -python3 ../.buildkite/scripts/run_test_scenarios.py "$db" "$BUILD_SCHEMA" +python3 ../.buildkite/scripts/run_test_scenarios.py "${1}" "${BUILD_SCHEMA}" From 43b35af68f30ec8dc9df1752d23bcdf964d547e1 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Fri, 17 Apr 2026 13:42:29 -0500 Subject: [PATCH 31/41] move run_models to central --- .buildkite/pipeline.yml | 46 +++++++++++++++++--------------- .buildkite/scripts/run_models.sh | 27 ------------------- 2 files changed, 25 insertions(+), 48 deletions(-) delete mode 100644 .buildkite/scripts/run_models.sh diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 232fd09..a7d4447 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,10 +1,21 @@ +# Each test downloads a centralized test runner script from the dbt_package_automations repo, +# which handles environment setup, dependency installation, and dbt test execution. + +# YAML anchors for reusable configurations +x-runner: &get-runner | + mkdir -p .buildkite/scripts + curl -f -s -o .buildkite/scripts/run_models.sh \ + "https://raw.githubusercontent.com/fivetran/dbt_package_automations/refs/heads/feature/buildkite-scripts/.buildkite/scripts/run_models.sh" + +x-retry-config: &retry-on-timeout + automatic: + - exit_status: -1 # Timeout + limit: 1 + steps: - label: ":postgres: Run Tests - Postgres" key: "run_dbt_postgres" - retry: - automatic: - - exit_status: -1 # Timeout - limit: 1 + retry: *retry-on-timeout plugins: - docker#v3.13.0: image: "python:3.13" @@ -17,14 +28,12 @@ steps: - "CI_POSTGRES_DBT_PASS" - "CI_POSTGRES_DBT_USER" commands: | - bash .buildkite/scripts/run_models.sh postgres + *get-runner + bash .buildkite/scripts/run_models.sh postgres - label: ":snowflake-db: Run Tests - Snowflake" key: "run_dbt_snowflake" - retry: - automatic: - - exit_status: -1 # Timeout - limit: 1 + retry: *retry-on-timeout plugins: - docker#v3.13.0: image: "python:3.13" @@ -39,14 +48,12 @@ steps: - "CI_SNOWFLAKE_DBT_USER" - "CI_SNOWFLAKE_DBT_WAREHOUSE" commands: | + *get-runner bash .buildkite/scripts/run_models.sh snowflake - label: ":gcloud: Run Tests - BigQuery" key: "run_dbt_bigquery" - retry: - automatic: - - exit_status: -1 # Timeout - limit: 1 + retry: *retry-on-timeout plugins: - docker#v3.13.0: image: "python:3.13" @@ -56,16 +63,14 @@ steps: - "BUILD_SCHEMA" - "GCLOUD_SERVICE_KEY" commands: | + *get-runner bash .buildkite/scripts/run_models.sh bigquery - label: ":amazon-redshift: Run Tests - Redshift" key: "run_dbt_redshift" concurrency: 3 concurrency_group: "warehouse/redshift" - retry: - automatic: - - exit_status: -1 # Timeout - limit: 1 + retry: *retry-on-timeout plugins: - docker#v3.13.0: image: "python:3.13" @@ -78,14 +83,12 @@ steps: - "CI_REDSHIFT_DBT_PASS" - "CI_REDSHIFT_DBT_USER" commands: | + *get-runner bash .buildkite/scripts/run_models.sh redshift - label: ":databricks: Run Tests - Databricks" key: "run_dbt_databricks" - retry: - automatic: - - exit_status: -1 # Timeout - limit: 1 + retry: *retry-on-timeout plugins: - docker#v3.13.0: image: "python:3.13" @@ -98,4 +101,5 @@ steps: - "CI_DATABRICKS_DBT_TOKEN" - "CI_DATABRICKS_DBT_CATALOG" commands: | + *get-runner bash .buildkite/scripts/run_models.sh databricks \ No newline at end of file diff --git a/.buildkite/scripts/run_models.sh b/.buildkite/scripts/run_models.sh deleted file mode 100644 index d8f1cd2..0000000 --- a/.buildkite/scripts/run_models.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -apt-get update -apt-get install libsasl2-dev - -python3 -m venv venv -. venv/bin/activate -pip install --upgrade pip setuptools - -# Install specific adapter for this database -echo "Installing dbt adapter: dbt-${1}" -pip install "dbt-${1}>=1.3.0,<2.0.0" -mkdir -p ~/.dbt -cp integration_tests/ci/sample.profiles.yml ~/.dbt/profiles.yml - -cd integration_tests - -# Fetch central test scenario script -mkdir -p ../.buildkite/scripts -curl -f -s -o ../.buildkite/scripts/run_test_scenarios.py \ - "https://raw.githubusercontent.com/fivetran/dbt_package_automations/refs/heads/feature/buildkite-scripts/.buildkite/scripts/run_test_scenarios.py" - -# Run test scenarios using Python script (includes deps, seed, compile) -# Test parameters and scenarios are configured in: integration_tests/ci/test_scenarios.yml -python3 ../.buildkite/scripts/run_test_scenarios.py "${1}" "${BUILD_SCHEMA}" From 8613c23a81db35436d7a0bde1c065eadb4fcda89 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Fri, 17 Apr 2026 14:44:47 -0500 Subject: [PATCH 32/41] try centralizing --- .buildkite/hooks/pre-command | 61 +----------------------------------- .buildkite/pipeline.yml | 20 ++++-------- 2 files changed, 7 insertions(+), 74 deletions(-) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 98a87f1..2a42e7a 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -1,61 +1,2 @@ #!/bin/bash - -set -e - -# Generate unique schema name for this build -# Format: bk__ -# Get package name from environment variable set in pipeline.yml, default to "unknown" if not set -BUILD_NUMBER="${BUILDKITE_BUILD_NUMBER:-0}" - -# Create unique schema name using commit SHA and build number -COMMIT_SHA=$(echo "${BUILDKITE_COMMIT:-unknown}" | cut -c1-7) - -# Add databricks-specific identifier for SQL warehouse to differentiate from all purpose cluster -if [[ "${BUILDKITE_STEP_KEY:-}" == "run_dbt_databricks_sql" ]]; then - BUILD_SCHEMA=$(echo "zz_bk_integration_tests_sql_${COMMIT_SHA}_${BUILD_NUMBER}" | cut -c1-63 | sed 's/_$//') -else - BUILD_SCHEMA=$(echo "zz_bk_integration_tests_${COMMIT_SHA}_${BUILD_NUMBER}" | cut -c1-63 | sed 's/_$//') -fi - -export BUILD_SCHEMA -echo "Build schema: ${BUILD_SCHEMA}" - -# Export secrets only for the warehouse needed by this step -echo "Step key: ${BUILDKITE_STEP_KEY:-unknown}" - -case "${BUILDKITE_STEP_KEY:-}" in - "run_dbt_bigquery") - echo "Setting up BigQuery credentials and dependencies" - export GCLOUD_SERVICE_KEY=$(gcloud secrets versions access latest --secret="GCLOUD_SERVICE_KEY" --project="dbt-package-testing-363917") - ;; - "run_dbt_postgres") - echo "Setting up PostgreSQL credentials and dependencies" - export CI_POSTGRES_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_HOST" --project="dbt-package-testing-363917") - export CI_POSTGRES_DBT_USER=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_USER" --project="dbt-package-testing-363917") - export CI_POSTGRES_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_PASS" --project="dbt-package-testing-363917") - export CI_POSTGRES_DBT_DBNAME=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_DBNAME" --project="dbt-package-testing-363917") - ;; - "run_dbt_redshift") - echo "Setting up Redshift credentials and dependencies" - export CI_REDSHIFT_DBT_DBNAME=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_DBNAME" --project="dbt-package-testing-363917") - export CI_REDSHIFT_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_HOST" --project="dbt-package-testing-363917") - export CI_REDSHIFT_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_PASS" --project="dbt-package-testing-363917") - export CI_REDSHIFT_DBT_USER=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_USER" --project="dbt-package-testing-363917") - ;; - "run_dbt_snowflake") - echo "Setting up Snowflake credentials and dependencies" - export CI_SNOWFLAKE_DBT_ACCOUNT=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_ACCOUNT" --project="dbt-package-testing-363917") - export CI_SNOWFLAKE_DBT_DATABASE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_DATABASE" --project="dbt-package-testing-363917") - export CI_SNOWFLAKE_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_PASS" --project="dbt-package-testing-363917") - export CI_SNOWFLAKE_DBT_ROLE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_ROLE" --project="dbt-package-testing-363917") - export CI_SNOWFLAKE_DBT_USER=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_USER" --project="dbt-package-testing-363917") - export CI_SNOWFLAKE_DBT_WAREHOUSE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_WAREHOUSE" --project="dbt-package-testing-363917") - ;; - "run_dbt_databricks") - echo "Setting up Databricks credentials and dependencies" - export CI_DATABRICKS_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HOST" --project="dbt-package-testing-363917") - export CI_DATABRICKS_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HTTP_PATH" --project="dbt-package-testing-363917") - export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917") - export CI_DATABRICKS_DBT_CATALOG=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_CATALOG" --project="dbt-package-testing-363917") - ;; -esac \ No newline at end of file +curl -s "https://raw.githubusercontent.com/fivetran/dbt_package_automations/refs/heads/feature/buildkite-scripts/.buildkite/scripts/build-setup.sh" | bash \ No newline at end of file diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index a7d4447..a331c95 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -2,10 +2,7 @@ # which handles environment setup, dependency installation, and dbt test execution. # YAML anchors for reusable configurations -x-runner: &get-runner | - mkdir -p .buildkite/scripts - curl -f -s -o .buildkite/scripts/run_models.sh \ - "https://raw.githubusercontent.com/fivetran/dbt_package_automations/refs/heads/feature/buildkite-scripts/.buildkite/scripts/run_models.sh" +x-run-models-url: &run-models-url "https://raw.githubusercontent.com/fivetran/dbt_package_automations/refs/heads/feature/buildkite-scripts/.buildkite/scripts/run_models.sh" x-retry-config: &retry-on-timeout automatic: @@ -28,8 +25,7 @@ steps: - "CI_POSTGRES_DBT_PASS" - "CI_POSTGRES_DBT_USER" commands: | - *get-runner - bash .buildkite/scripts/run_models.sh postgres + curl -s *run-models-url | bash -s postgres - label: ":snowflake-db: Run Tests - Snowflake" key: "run_dbt_snowflake" @@ -48,8 +44,7 @@ steps: - "CI_SNOWFLAKE_DBT_USER" - "CI_SNOWFLAKE_DBT_WAREHOUSE" commands: | - *get-runner - bash .buildkite/scripts/run_models.sh snowflake + curl -s *run-models-url | bash -s snowflake - label: ":gcloud: Run Tests - BigQuery" key: "run_dbt_bigquery" @@ -63,8 +58,7 @@ steps: - "BUILD_SCHEMA" - "GCLOUD_SERVICE_KEY" commands: | - *get-runner - bash .buildkite/scripts/run_models.sh bigquery + curl -s *run-models-url | bash -s bigquery - label: ":amazon-redshift: Run Tests - Redshift" key: "run_dbt_redshift" @@ -83,8 +77,7 @@ steps: - "CI_REDSHIFT_DBT_PASS" - "CI_REDSHIFT_DBT_USER" commands: | - *get-runner - bash .buildkite/scripts/run_models.sh redshift + curl -s *run-models-url | bash -s redshift - label: ":databricks: Run Tests - Databricks" key: "run_dbt_databricks" @@ -101,5 +94,4 @@ steps: - "CI_DATABRICKS_DBT_TOKEN" - "CI_DATABRICKS_DBT_CATALOG" commands: | - *get-runner - bash .buildkite/scripts/run_models.sh databricks \ No newline at end of file + curl -s *run-models-url | bash -s databricks \ No newline at end of file From 384664af9579acab53b9da964ced31dad2be4cb3 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Fri, 17 Apr 2026 14:50:13 -0500 Subject: [PATCH 33/41] try again --- .buildkite/pipeline.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index a331c95..1d0fb1f 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -2,7 +2,8 @@ # which handles environment setup, dependency installation, and dbt test execution. # YAML anchors for reusable configurations -x-run-models-url: &run-models-url "https://raw.githubusercontent.com/fivetran/dbt_package_automations/refs/heads/feature/buildkite-scripts/.buildkite/scripts/run_models.sh" +env: + RUN_MODELS_URL: "https://raw.githubusercontent.com/fivetran/dbt_package_automations/refs/heads/feature/buildkite-scripts/.buildkite/scripts/run_models.sh" x-retry-config: &retry-on-timeout automatic: @@ -25,7 +26,7 @@ steps: - "CI_POSTGRES_DBT_PASS" - "CI_POSTGRES_DBT_USER" commands: | - curl -s *run-models-url | bash -s postgres + curl -s "${RUN_MODELS_URL}" | bash -s postgres - label: ":snowflake-db: Run Tests - Snowflake" key: "run_dbt_snowflake" @@ -44,7 +45,7 @@ steps: - "CI_SNOWFLAKE_DBT_USER" - "CI_SNOWFLAKE_DBT_WAREHOUSE" commands: | - curl -s *run-models-url | bash -s snowflake + curl -s "${RUN_MODELS_URL}" | bash -s snowflake - label: ":gcloud: Run Tests - BigQuery" key: "run_dbt_bigquery" @@ -58,7 +59,7 @@ steps: - "BUILD_SCHEMA" - "GCLOUD_SERVICE_KEY" commands: | - curl -s *run-models-url | bash -s bigquery + curl -s "${RUN_MODELS_URL}" | bash -s bigquery - label: ":amazon-redshift: Run Tests - Redshift" key: "run_dbt_redshift" @@ -77,7 +78,7 @@ steps: - "CI_REDSHIFT_DBT_PASS" - "CI_REDSHIFT_DBT_USER" commands: | - curl -s *run-models-url | bash -s redshift + curl -s "${RUN_MODELS_URL}" | bash -s redshift - label: ":databricks: Run Tests - Databricks" key: "run_dbt_databricks" @@ -94,4 +95,4 @@ steps: - "CI_DATABRICKS_DBT_TOKEN" - "CI_DATABRICKS_DBT_CATALOG" commands: | - curl -s *run-models-url | bash -s databricks \ No newline at end of file + curl -s "${RUN_MODELS_URL}" | bash -s databricks \ No newline at end of file From 61d8df63569072b6bbc67c0e1957341eebfe86dd Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Fri, 17 Apr 2026 15:00:49 -0500 Subject: [PATCH 34/41] try again --- .buildkite/hooks/pre-command | 3 ++- .buildkite/pipeline.yml | 31 ++++++++++++++++++++----------- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 2a42e7a..9e783d1 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -1,2 +1,3 @@ #!/bin/bash -curl -s "https://raw.githubusercontent.com/fivetran/dbt_package_automations/refs/heads/feature/buildkite-scripts/.buildkite/scripts/build-setup.sh" | bash \ No newline at end of file +curl -s -o /tmp/build-setup.sh "https://raw.githubusercontent.com/fivetran/dbt_package_automations/refs/heads/feature/buildkite-scripts/.buildkite/scripts/build-setup.sh" +source /tmp/build-setup.sh \ No newline at end of file diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 1d0fb1f..a9866c6 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,19 +1,16 @@ # Each test downloads a centralized test runner script from the dbt_package_automations repo, # which handles environment setup, dependency installation, and dbt test execution. -# YAML anchors for reusable configurations env: RUN_MODELS_URL: "https://raw.githubusercontent.com/fivetran/dbt_package_automations/refs/heads/feature/buildkite-scripts/.buildkite/scripts/run_models.sh" -x-retry-config: &retry-on-timeout - automatic: - - exit_status: -1 # Timeout - limit: 1 - steps: - label: ":postgres: Run Tests - Postgres" key: "run_dbt_postgres" - retry: *retry-on-timeout + retry: + automatic: + - exit_status: -1 # Timeout + limit: 1 plugins: - docker#v3.13.0: image: "python:3.13" @@ -30,7 +27,10 @@ steps: - label: ":snowflake-db: Run Tests - Snowflake" key: "run_dbt_snowflake" - retry: *retry-on-timeout + retry: + automatic: + - exit_status: -1 # Timeout + limit: 1 plugins: - docker#v3.13.0: image: "python:3.13" @@ -49,7 +49,10 @@ steps: - label: ":gcloud: Run Tests - BigQuery" key: "run_dbt_bigquery" - retry: *retry-on-timeout + retry: + automatic: + - exit_status: -1 # Timeout + limit: 1 plugins: - docker#v3.13.0: image: "python:3.13" @@ -65,7 +68,10 @@ steps: key: "run_dbt_redshift" concurrency: 3 concurrency_group: "warehouse/redshift" - retry: *retry-on-timeout + retry: + automatic: + - exit_status: -1 # Timeout + limit: 1 plugins: - docker#v3.13.0: image: "python:3.13" @@ -82,7 +88,10 @@ steps: - label: ":databricks: Run Tests - Databricks" key: "run_dbt_databricks" - retry: *retry-on-timeout + retry: + automatic: + - exit_status: -1 # Timeout + limit: 1 plugins: - docker#v3.13.0: image: "python:3.13" From 4745c31b9bfd29dd13f9bd88aaed262683d834b1 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Fri, 17 Apr 2026 15:19:16 -0500 Subject: [PATCH 35/41] more consolidaton --- .buildkite/hooks/pre-command | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 .buildkite/hooks/pre-command diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command deleted file mode 100644 index 9e783d1..0000000 --- a/.buildkite/hooks/pre-command +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -curl -s -o /tmp/build-setup.sh "https://raw.githubusercontent.com/fivetran/dbt_package_automations/refs/heads/feature/buildkite-scripts/.buildkite/scripts/build-setup.sh" -source /tmp/build-setup.sh \ No newline at end of file From 5229fd460c3375864022c02717ce1b2ec365b3d9 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Fri, 17 Apr 2026 15:27:40 -0500 Subject: [PATCH 36/41] update --- .buildkite/pipeline.yml | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index a9866c6..3ff9eff 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -17,11 +17,9 @@ steps: shell: [ "/bin/bash", "-e", "-c" ] environment: - "BASH_ENV=/tmp/.bashrc" - - "BUILD_SCHEMA" - - "CI_POSTGRES_DBT_DBNAME" - - "CI_POSTGRES_DBT_HOST" - - "CI_POSTGRES_DBT_PASS" - - "CI_POSTGRES_DBT_USER" + - "BUILDKITE_BUILD_NUMBER" + - "BUILDKITE_COMMIT" + - "BUILDKITE_STEP_KEY" commands: | curl -s "${RUN_MODELS_URL}" | bash -s postgres @@ -37,13 +35,9 @@ steps: shell: [ "/bin/bash", "-e", "-c" ] environment: - "BASH_ENV=/tmp/.bashrc" - - "BUILD_SCHEMA" - - "CI_SNOWFLAKE_DBT_ACCOUNT" - - "CI_SNOWFLAKE_DBT_DATABASE" - - "CI_SNOWFLAKE_DBT_PASS" - - "CI_SNOWFLAKE_DBT_ROLE" - - "CI_SNOWFLAKE_DBT_USER" - - "CI_SNOWFLAKE_DBT_WAREHOUSE" + - "BUILDKITE_BUILD_NUMBER" + - "BUILDKITE_COMMIT" + - "BUILDKITE_STEP_KEY" commands: | curl -s "${RUN_MODELS_URL}" | bash -s snowflake @@ -59,8 +53,9 @@ steps: shell: [ "/bin/bash", "-e", "-c" ] environment: - "BASH_ENV=/tmp/.bashrc" - - "BUILD_SCHEMA" - - "GCLOUD_SERVICE_KEY" + - "BUILDKITE_BUILD_NUMBER" + - "BUILDKITE_COMMIT" + - "BUILDKITE_STEP_KEY" commands: | curl -s "${RUN_MODELS_URL}" | bash -s bigquery @@ -78,6 +73,9 @@ steps: shell: [ "/bin/bash", "-e", "-c" ] environment: - "BASH_ENV=/tmp/.bashrc" + - "BUILDKITE_BUILD_NUMBER" + - "BUILDKITE_COMMIT" + - "BUILDKITE_STEP_KEY" - "BUILD_SCHEMA" - "CI_REDSHIFT_DBT_DBNAME" - "CI_REDSHIFT_DBT_HOST" @@ -98,6 +96,9 @@ steps: shell: [ "/bin/bash", "-e", "-c" ] environment: - "BASH_ENV=/tmp/.bashrc" + - "BUILDKITE_BUILD_NUMBER" + - "BUILDKITE_COMMIT" + - "BUILDKITE_STEP_KEY" - "BUILD_SCHEMA" - "CI_DATABRICKS_DBT_HOST" - "CI_DATABRICKS_DBT_HTTP_PATH" From c363f5108949fbd09d12489d33ec1b4fb6198af0 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Fri, 17 Apr 2026 15:34:51 -0500 Subject: [PATCH 37/41] fix it --- .buildkite/hooks/pre-command | 3 +++ .buildkite/pipeline.yml | 39 ++++++++++++++++++++++-------------- 2 files changed, 27 insertions(+), 15 deletions(-) create mode 100644 .buildkite/hooks/pre-command diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command new file mode 100644 index 0000000..9e783d1 --- /dev/null +++ b/.buildkite/hooks/pre-command @@ -0,0 +1,3 @@ +#!/bin/bash +curl -s -o /tmp/build-setup.sh "https://raw.githubusercontent.com/fivetran/dbt_package_automations/refs/heads/feature/buildkite-scripts/.buildkite/scripts/build-setup.sh" +source /tmp/build-setup.sh \ No newline at end of file diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 3ff9eff..e0d7d5e 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -17,9 +17,11 @@ steps: shell: [ "/bin/bash", "-e", "-c" ] environment: - "BASH_ENV=/tmp/.bashrc" - - "BUILDKITE_BUILD_NUMBER" - - "BUILDKITE_COMMIT" - - "BUILDKITE_STEP_KEY" + - "BUILD_SCHEMA" + - "CI_POSTGRES_DBT_DBNAME" + - "CI_POSTGRES_DBT_HOST" + - "CI_POSTGRES_DBT_PASS" + - "CI_POSTGRES_DBT_USER" commands: | curl -s "${RUN_MODELS_URL}" | bash -s postgres @@ -35,9 +37,13 @@ steps: shell: [ "/bin/bash", "-e", "-c" ] environment: - "BASH_ENV=/tmp/.bashrc" - - "BUILDKITE_BUILD_NUMBER" - - "BUILDKITE_COMMIT" - - "BUILDKITE_STEP_KEY" + - "BUILD_SCHEMA" + - "CI_SNOWFLAKE_DBT_ACCOUNT" + - "CI_SNOWFLAKE_DBT_DATABASE" + - "CI_SNOWFLAKE_DBT_PASS" + - "CI_SNOWFLAKE_DBT_ROLE" + - "CI_SNOWFLAKE_DBT_USER" + - "CI_SNOWFLAKE_DBT_WAREHOUSE" commands: | curl -s "${RUN_MODELS_URL}" | bash -s snowflake @@ -53,9 +59,8 @@ steps: shell: [ "/bin/bash", "-e", "-c" ] environment: - "BASH_ENV=/tmp/.bashrc" - - "BUILDKITE_BUILD_NUMBER" - - "BUILDKITE_COMMIT" - - "BUILDKITE_STEP_KEY" + - "BUILD_SCHEMA" + - "GCLOUD_SERVICE_KEY" commands: | curl -s "${RUN_MODELS_URL}" | bash -s bigquery @@ -73,9 +78,11 @@ steps: shell: [ "/bin/bash", "-e", "-c" ] environment: - "BASH_ENV=/tmp/.bashrc" - - "BUILDKITE_BUILD_NUMBER" - - "BUILDKITE_COMMIT" - - "BUILDKITE_STEP_KEY" + - "BUILD_SCHEMA" + - "CI_REDSHIFT_DBT_DBNAME" + - "CI_REDSHIFT_DBT_HOST" + - "CI_REDSHIFT_DBT_PASS" + - "CI_REDSHIFT_DBT_USER" - "BUILD_SCHEMA" - "CI_REDSHIFT_DBT_DBNAME" - "CI_REDSHIFT_DBT_HOST" @@ -96,9 +103,11 @@ steps: shell: [ "/bin/bash", "-e", "-c" ] environment: - "BASH_ENV=/tmp/.bashrc" - - "BUILDKITE_BUILD_NUMBER" - - "BUILDKITE_COMMIT" - - "BUILDKITE_STEP_KEY" + - "BUILD_SCHEMA" + - "CI_DATABRICKS_DBT_HOST" + - "CI_DATABRICKS_DBT_HTTP_PATH" + - "CI_DATABRICKS_DBT_TOKEN" + - "CI_DATABRICKS_DBT_CATALOG" - "BUILD_SCHEMA" - "CI_DATABRICKS_DBT_HOST" - "CI_DATABRICKS_DBT_HTTP_PATH" From dc8dcfabf16f888d3ff8f05f2915ff2681c7e09a Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Fri, 17 Apr 2026 15:48:47 -0500 Subject: [PATCH 38/41] file rename --- .buildkite/hooks/pre-command | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 9e783d1..9114a83 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -1,3 +1,3 @@ #!/bin/bash -curl -s -o /tmp/build-setup.sh "https://raw.githubusercontent.com/fivetran/dbt_package_automations/refs/heads/feature/buildkite-scripts/.buildkite/scripts/build-setup.sh" +curl -s -o /tmp/build-setup.sh "https://raw.githubusercontent.com/fivetran/dbt_package_automations/refs/heads/feature/buildkite-scripts/.buildkite/scripts/build_setup.sh" source /tmp/build-setup.sh \ No newline at end of file From af9b389012f633076fadfe7ef273b8c1df9880c4 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Fri, 17 Apr 2026 17:15:49 -0500 Subject: [PATCH 39/41] update --- .buildkite/pipeline.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index e0d7d5e..c833f01 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -83,11 +83,6 @@ steps: - "CI_REDSHIFT_DBT_HOST" - "CI_REDSHIFT_DBT_PASS" - "CI_REDSHIFT_DBT_USER" - - "BUILD_SCHEMA" - - "CI_REDSHIFT_DBT_DBNAME" - - "CI_REDSHIFT_DBT_HOST" - - "CI_REDSHIFT_DBT_PASS" - - "CI_REDSHIFT_DBT_USER" commands: | curl -s "${RUN_MODELS_URL}" | bash -s redshift From 2ff6358d8f51987a8755ac511d760e6bb96d28b6 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Fri, 17 Apr 2026 17:17:21 -0500 Subject: [PATCH 40/41] update --- .buildkite/pipeline.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index c833f01..a9866c6 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -103,10 +103,5 @@ steps: - "CI_DATABRICKS_DBT_HTTP_PATH" - "CI_DATABRICKS_DBT_TOKEN" - "CI_DATABRICKS_DBT_CATALOG" - - "BUILD_SCHEMA" - - "CI_DATABRICKS_DBT_HOST" - - "CI_DATABRICKS_DBT_HTTP_PATH" - - "CI_DATABRICKS_DBT_TOKEN" - - "CI_DATABRICKS_DBT_CATALOG" commands: | curl -s "${RUN_MODELS_URL}" | bash -s databricks \ No newline at end of file From f4ded3d7b13062245d21137bbde9b25c85c57dc6 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Fri, 17 Apr 2026 17:21:03 -0500 Subject: [PATCH 41/41] update --- integration_tests/ci/sample.profiles.yml | 50 ------------------------ 1 file changed, 50 deletions(-) delete mode 100644 integration_tests/ci/sample.profiles.yml diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml deleted file mode 100644 index deac440..0000000 --- a/integration_tests/ci/sample.profiles.yml +++ /dev/null @@ -1,50 +0,0 @@ - -# HEY! This file is used in the dbt package integrations tests with Buildkite. -# You should __NEVER__ check credentials into version control. Thanks for reading :) - -integration_tests: - target: redshift - outputs: - redshift: - type: redshift - host: "{{ env_var('CI_REDSHIFT_DBT_HOST') }}" - user: "{{ env_var('CI_REDSHIFT_DBT_USER') }}" - pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}" - dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}" - port: 5439 - schema: "{{ env_var('BUILD_SCHEMA') }}" - threads: 8 - bigquery: - type: bigquery - method: service-account-json - project: 'dbt-package-testing' - schema: "{{ env_var('BUILD_SCHEMA') }}" - threads: 8 - keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}" - snowflake: - type: snowflake - account: "{{ env_var('CI_SNOWFLAKE_DBT_ACCOUNT') }}" - user: "{{ env_var('CI_SNOWFLAKE_DBT_USER') }}" - password: "{{ env_var('CI_SNOWFLAKE_DBT_PASS') }}" - role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}" - database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}" - warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}" - schema: "{{ env_var('BUILD_SCHEMA') }}" - threads: 8 - postgres: - type: postgres - host: "{{ env_var('CI_POSTGRES_DBT_HOST') }}" - user: "{{ env_var('CI_POSTGRES_DBT_USER') }}" - pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}" - dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}" - port: 5432 - schema: "{{ env_var('BUILD_SCHEMA') }}" - threads: 8 - databricks: - catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}" - host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}" - http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}" - schema: "{{ env_var('BUILD_SCHEMA') }}" - threads: 8 - token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}" - type: databricks