diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 81b7b00..6534925 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -1,25 +1,3 @@ #!/bin/bash - -set -e - -# Export secrets for Docker containers. -# Restrict exposing secrets only to the steps that need them -export GCLOUD_SERVICE_KEY=$(gcloud secrets versions access latest --secret="GCLOUD_SERVICE_KEY" --project="dbt-package-testing-363917") -export CI_POSTGRES_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_HOST" --project="dbt-package-testing-363917") -export CI_POSTGRES_DBT_USER=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_USER" --project="dbt-package-testing-363917") -export CI_POSTGRES_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_PASS" --project="dbt-package-testing-363917") -export CI_POSTGRES_DBT_DBNAME=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_DBNAME" --project="dbt-package-testing-363917") -export CI_REDSHIFT_DBT_DBNAME=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_DBNAME" --project="dbt-package-testing-363917") -export CI_REDSHIFT_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_HOST" --project="dbt-package-testing-363917") -export CI_REDSHIFT_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_PASS" --project="dbt-package-testing-363917") -export CI_REDSHIFT_DBT_USER=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_USER" --project="dbt-package-testing-363917") -export CI_SNOWFLAKE_DBT_ACCOUNT=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_ACCOUNT" --project="dbt-package-testing-363917") -export CI_SNOWFLAKE_DBT_DATABASE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_DATABASE" --project="dbt-package-testing-363917") -export CI_SNOWFLAKE_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_PASS" --project="dbt-package-testing-363917") -export CI_SNOWFLAKE_DBT_ROLE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_ROLE" --project="dbt-package-testing-363917") -export CI_SNOWFLAKE_DBT_USER=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_USER" --project="dbt-package-testing-363917") -export CI_SNOWFLAKE_DBT_WAREHOUSE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_WAREHOUSE" --project="dbt-package-testing-363917") -export CI_DATABRICKS_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HOST" --project="dbt-package-testing-363917") -export CI_DATABRICKS_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HTTP_PATH" --project="dbt-package-testing-363917") -export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917") -export CI_DATABRICKS_DBT_CATALOG=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_CATALOG" --project="dbt-package-testing-363917") \ No newline at end of file +curl -s -o /tmp/build_setup.sh "https://raw.githubusercontent.com/fivetran/dbt_package_automations/refs/heads/feature/buildkite-scripts/.buildkite/scripts/build_setup.sh" +source /tmp/build_setup.sh \ No newline at end of file diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 2603602..6a98d46 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -1,27 +1,40 @@ +env: + RUN_MODELS_URL: "https://raw.githubusercontent.com/fivetran/dbt_package_automations/refs/heads/feature/buildkite-scripts/.buildkite/scripts/run_models.sh" + steps: - label: ":postgres: Run Tests - Postgres" - key: "run-dbt-postgres" + key: "run_dbt_postgres" + retry: + automatic: + - exit_status: -1 # Timeout + limit: 1 plugins: - docker#v3.13.0: - image: "python:3.10.13" + image: "python:3.13" shell: [ "/bin/bash", "-e", "-c" ] environment: - "BASH_ENV=/tmp/.bashrc" + - "BUILD_SCHEMA" - "CI_POSTGRES_DBT_DBNAME" - "CI_POSTGRES_DBT_HOST" - "CI_POSTGRES_DBT_PASS" - "CI_POSTGRES_DBT_USER" commands: | - bash .buildkite/scripts/run_models.sh postgres + curl -s "${RUN_MODELS_URL}" | bash -s postgres - label: ":snowflake-db: Run Tests - Snowflake" key: "run_dbt_snowflake" + retry: + automatic: + - exit_status: -1 # Timeout + limit: 1 plugins: - docker#v3.13.0: - image: "python:3.10.13" + image: "python:3.13" shell: [ "/bin/bash", "-e", "-c" ] environment: - "BASH_ENV=/tmp/.bashrc" + - "BUILD_SCHEMA" - "CI_SNOWFLAKE_DBT_ACCOUNT" - "CI_SNOWFLAKE_DBT_DATABASE" - "CI_SNOWFLAKE_DBT_PASS" @@ -29,46 +42,63 @@ steps: - "CI_SNOWFLAKE_DBT_USER" - "CI_SNOWFLAKE_DBT_WAREHOUSE" commands: | - bash .buildkite/scripts/run_models.sh snowflake + curl -s "${RUN_MODELS_URL}" | bash -s snowflake - label: ":gcloud: Run Tests - BigQuery" key: "run_dbt_bigquery" + retry: + automatic: + - exit_status: -1 # Timeout + limit: 1 plugins: - docker#v3.13.0: - image: "python:3.10.13" + image: "python:3.13" shell: [ "/bin/bash", "-e", "-c" ] environment: - "BASH_ENV=/tmp/.bashrc" + - "BUILD_SCHEMA" - "GCLOUD_SERVICE_KEY" commands: | - bash .buildkite/scripts/run_models.sh bigquery + curl -s "${RUN_MODELS_URL}" | bash -s bigquery - label: ":amazon-redshift: Run Tests - Redshift" key: "run_dbt_redshift" + concurrency: 3 + concurrency_group: "warehouse/redshift" + retry: + automatic: + - exit_status: -1 # Timeout + limit: 1 plugins: - docker#v3.13.0: - image: "python:3.10.13" + image: "python:3.13" shell: [ "/bin/bash", "-e", "-c" ] environment: - "BASH_ENV=/tmp/.bashrc" + - "BUILD_SCHEMA" - "CI_REDSHIFT_DBT_DBNAME" - "CI_REDSHIFT_DBT_HOST" - "CI_REDSHIFT_DBT_PASS" - "CI_REDSHIFT_DBT_USER" commands: | - bash .buildkite/scripts/run_models.sh redshift + curl -s "${RUN_MODELS_URL}" | bash -s redshift - label: ":databricks: Run Tests - Databricks" key: "run_dbt_databricks" + retry: + automatic: + - exit_status: -1 # Timeout + limit: 1 plugins: - docker#v3.13.0: - image: "python:3.10.13" + image: "python:3.13" shell: [ "/bin/bash", "-e", "-c" ] environment: - "BASH_ENV=/tmp/.bashrc" + - "BUILD_SCHEMA" - "CI_DATABRICKS_DBT_HOST" - "CI_DATABRICKS_DBT_HTTP_PATH" - "CI_DATABRICKS_DBT_TOKEN" - "CI_DATABRICKS_DBT_CATALOG" commands: | - bash .buildkite/scripts/run_models.sh databricks \ No newline at end of file + curl -s "${RUN_MODELS_URL}" | bash -s databricks \ No newline at end of file diff --git a/.buildkite/scripts/run_models.sh b/.buildkite/scripts/run_models.sh deleted file mode 100644 index fb75f9b..0000000 --- a/.buildkite/scripts/run_models.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -apt-get update -apt-get install libsasl2-dev - -python3 -m venv venv -. venv/bin/activate -pip install --upgrade pip setuptools -pip install -r integration_tests/requirements.txt -mkdir -p ~/.dbt -cp integration_tests/ci/sample.profiles.yml ~/.dbt/profiles.yml - -db=$1 -echo `pwd` -cd integration_tests -dbt deps -dbt seed --target "$db" --full-refresh -dbt run --target "$db" --full-refresh -dbt test --target "$db" -dbt run --vars '{amazon_ads__portfolio_history_enabled: false}' --target "$db" --full-refresh -dbt test --vars '{amazon_ads__portfolio_history_enabled: false}' --target "$db" - -dbt run-operation fivetran_utils.drop_schemas_automation --target "$db" diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml deleted file mode 100644 index e7c86b2..0000000 --- a/integration_tests/ci/sample.profiles.yml +++ /dev/null @@ -1,50 +0,0 @@ - -# HEY! This file is used in the dbt package integrations tests with Buildkite. -# You should __NEVER__ check credentials into version control. Thanks for reading :) - -integration_tests: - target: redshift - outputs: - redshift: - type: redshift - host: "{{ env_var('CI_REDSHIFT_DBT_HOST') }}" - user: "{{ env_var('CI_REDSHIFT_DBT_USER') }}" - pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}" - dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}" - port: 5439 - schema: amazon_ads_integration_tests_03 - threads: 8 - bigquery: - type: bigquery - method: service-account-json - project: 'dbt-package-testing' - schema: amazon_ads_integration_tests_03 - threads: 8 - keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}" - snowflake: - type: snowflake - account: "{{ env_var('CI_SNOWFLAKE_DBT_ACCOUNT') }}" - user: "{{ env_var('CI_SNOWFLAKE_DBT_USER') }}" - password: "{{ env_var('CI_SNOWFLAKE_DBT_PASS') }}" - role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}" - database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}" - warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}" - schema: amazon_ads_integration_tests_03 - threads: 8 - postgres: - type: postgres - host: "{{ env_var('CI_POSTGRES_DBT_HOST') }}" - user: "{{ env_var('CI_POSTGRES_DBT_USER') }}" - pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}" - dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}" - port: 5432 - schema: amazon_ads_integration_tests_03 - threads: 8 - databricks: - catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}" - host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}" - http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}" - schema: amazon_ads_integration_tests_03 - threads: 8 - token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}" - type: databricks diff --git a/integration_tests/ci/test_scenarios.yml b/integration_tests/ci/test_scenarios.yml new file mode 100644 index 0000000..bce0176 --- /dev/null +++ b/integration_tests/ci/test_scenarios.yml @@ -0,0 +1,8 @@ +schema_variable_name: "amazon_ads_schema" +default_include_incremental: false + +test_scenarios: + - name: "portfolio history disabled" + vars: + amazon_ads__portfolio_history_enabled: false + include_incremental: false \ No newline at end of file diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 4b069ae..d166df7 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -1,54 +1,74 @@ config-version: 2 - -name: 'amazon_ads_integration_tests' -version: '1.2.0' - -profile: 'integration_tests' - +name: amazon_ads_integration_tests +version: 1.2.0 +profile: integration_tests vars: amazon_ads: - amazon_ads_schema: amazon_ads_integration_tests_03 - amazon_ads_ad_group_history_identifier: "ad_group_history_data" - amazon_ads_ad_group_level_report_identifier: "ad_group_level_report_data" - amazon_ads_advertised_product_report_identifier: "advertised_product_report_data" - amazon_ads_campaign_history_identifier: "campaign_history_data" - amazon_ads_campaign_level_report_identifier: "campaign_level_report_data" - amazon_ads_portfolio_history_identifier: "portfolio_history_data" - amazon_ads_product_ad_history_identifier: "product_ad_history_data" - amazon_ads_profile_identifier: "profile_data" - amazon_ads_keyword_history_identifier: "keyword_history_data" - amazon_ads_targeting_keyword_report_identifier: "targeting_keyword_report_data" - amazon_ads_search_term_ad_keyword_report_identifier: "search_term_ad_keyword_report_data" - + amazon_ads_schema: amazon_ads_integration_tests + amazon_ads_ad_group_history_identifier: ad_group_history_data + amazon_ads_ad_group_level_report_identifier: ad_group_level_report_data + amazon_ads_advertised_product_report_identifier: advertised_product_report_data + amazon_ads_campaign_history_identifier: campaign_history_data + amazon_ads_campaign_level_report_identifier: campaign_level_report_data + amazon_ads_portfolio_history_identifier: portfolio_history_data + amazon_ads_product_ad_history_identifier: product_ad_history_data + amazon_ads_profile_identifier: profile_data + amazon_ads_keyword_history_identifier: keyword_history_data + amazon_ads_targeting_keyword_report_identifier: targeting_keyword_report_data + amazon_ads_search_term_ad_keyword_report_identifier: search_term_ad_keyword_report_data amazon_ads__campaign_passthrough_metrics: - - name: sales_7_d - - name: purchases_30_d - alias: purchases_alias - - name: purchases_14_d - + - name: sales_7_d + - name: purchases_30_d + alias: purchases_alias + - name: purchases_14_d models: +persist_docs: relation: true - columns: "{{ false if target.type in ('databricks') else true }}" - +schema: "amazon_ads_{{ var('directed_schema','dev') }}" - + columns: '{{ false if target.type in (''databricks'') else true }}' + +schema: amazon_ads_{{ var('directed_schema','dev') }} dispatch: - - macro_namespace: dbt_utils - search_order: ['spark_utils', 'dbt_utils'] - +- macro_namespace: dbt_utils + search_order: + - spark_utils + - dbt_utils seeds: +docs: - show: False - +quote_columns: "{{ true if target.type == 'redshift' else false }}" + show: false + +quote_columns: '{{ true if target.type == ''redshift'' else false }}' + +column_types: + _fivetran_synced: timestamp amazon_ads_integration_tests: - +column_types: - _fivetran_synced: "timestamp" - campaign_budget_amount: "float" - click_through_rate: "float" - keyword_bid: "float" - clicks: "float" - impressions: "float" - cost: "float" + advertised_product_report_data: + +column_types: + campaign_budget_amount: float + clicks: float + impressions: float + cost: float + campaign_level_report_data: + +column_types: + campaign_budget_amount: float + clicks: float + impressions: float + cost: float + ad_group_level_report_data: + +column_types: + clicks: float + cost: float + impressions: float + search_term_ad_keyword_report_data: + +column_types: + campaign_budget_amount: float + clicks: float + cost: float + impressions: float + keyword_bid: float + targeting_keyword_report_data: + +column_types: + campaign_budget_amount: float + clicks: float + cost: float + impressions: float + keyword_bid: float flags: - send_anonymous_usage_stats: False - use_colors: True + send_anonymous_usage_stats: false + use_colors: true diff --git a/integration_tests/requirements.txt b/integration_tests/requirements.txt deleted file mode 100644 index 8c55c23..0000000 --- a/integration_tests/requirements.txt +++ /dev/null @@ -1,8 +0,0 @@ -dbt-snowflake>=1.3.0,<2.0.0 -dbt-bigquery>=1.3.0,<2.0.0 -dbt-redshift>=1.3.0,<2.0.0 -dbt-postgres>=1.3.0,<2.0.0 -dbt-spark>=1.3.0,<2.0.0 -dbt-spark[PyHive]>=1.3.0,<2.0.0 -dbt-databricks>=1.6.0,<2.0.0 -certifi==2025.1.31 \ No newline at end of file