Skip to content

Buildkite improvements FY27 Q2 #61

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
fivetran-data-model-bot wants to merge 1 commit into from
Closed

Buildkite improvements FY27 Q2 #61

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 2 additions & 24 deletions .buildkite/hooks/pre-command
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,25 +1,3 @@
#!/bin/bash

set -e

# Export secrets for Docker containers.
# Restrict exposing secrets only to the steps that need them
export GCLOUD_SERVICE_KEY=$(gcloud secrets versions access latest --secret="GCLOUD_SERVICE_KEY" --project="dbt-package-testing-363917")
export CI_POSTGRES_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_HOST" --project="dbt-package-testing-363917")
export CI_POSTGRES_DBT_USER=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_USER" --project="dbt-package-testing-363917")
export CI_POSTGRES_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_PASS" --project="dbt-package-testing-363917")
export CI_POSTGRES_DBT_DBNAME=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_DBNAME" --project="dbt-package-testing-363917")
export CI_REDSHIFT_DBT_DBNAME=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_DBNAME" --project="dbt-package-testing-363917")
export CI_REDSHIFT_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_HOST" --project="dbt-package-testing-363917")
export CI_REDSHIFT_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_PASS" --project="dbt-package-testing-363917")
export CI_REDSHIFT_DBT_USER=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_USER" --project="dbt-package-testing-363917")
export CI_SNOWFLAKE_DBT_ACCOUNT=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_ACCOUNT" --project="dbt-package-testing-363917")
export CI_SNOWFLAKE_DBT_DATABASE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_DATABASE" --project="dbt-package-testing-363917")
export CI_SNOWFLAKE_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_PASS" --project="dbt-package-testing-363917")
export CI_SNOWFLAKE_DBT_ROLE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_ROLE" --project="dbt-package-testing-363917")
export CI_SNOWFLAKE_DBT_USER=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_USER" --project="dbt-package-testing-363917")
export CI_SNOWFLAKE_DBT_WAREHOUSE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_WAREHOUSE" --project="dbt-package-testing-363917")
export CI_DATABRICKS_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HOST" --project="dbt-package-testing-363917")
export CI_DATABRICKS_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HTTP_PATH" --project="dbt-package-testing-363917")
export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917")
export CI_DATABRICKS_DBT_CATALOG=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_CATALOG" --project="dbt-package-testing-363917")
curl -s -o /tmp/build_setup.sh "https://raw.githubusercontent.com/fivetran/dbt_package_automations/refs/heads/feature/buildkite-scripts/.buildkite/scripts/build_setup.sh"
source /tmp/build_setup.sh
52 changes: 41 additions & 11 deletions .buildkite/pipeline.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,74 +1,104 @@
env:
RUN_MODELS_URL: "https://raw.githubusercontent.com/fivetran/dbt_package_automations/refs/heads/feature/buildkite-scripts/.buildkite/scripts/run_models.sh"

steps:
- label: ":postgres: Run Tests - Postgres"
key: "run-dbt-postgres"
key: "run_dbt_postgres"
retry:
automatic:
- exit_status: -1 # Timeout
limit: 1
plugins:
- docker#v3.13.0:
image: "python:3.10.13"
image: "python:3.13"
shell: [ "/bin/bash", "-e", "-c" ]
environment:
- "BASH_ENV=/tmp/.bashrc"
- "BUILD_SCHEMA"
- "CI_POSTGRES_DBT_DBNAME"
- "CI_POSTGRES_DBT_HOST"
- "CI_POSTGRES_DBT_PASS"
- "CI_POSTGRES_DBT_USER"
commands: |
bash .buildkite/scripts/run_models.sh postgres
curl -s "${RUN_MODELS_URL}" | bash -s postgres

- label: ":snowflake-db: Run Tests - Snowflake"
key: "run_dbt_snowflake"
retry:
automatic:
- exit_status: -1 # Timeout
limit: 1
plugins:
- docker#v3.13.0:
image: "python:3.10.13"
image: "python:3.13"
shell: [ "/bin/bash", "-e", "-c" ]
environment:
- "BASH_ENV=/tmp/.bashrc"
- "BUILD_SCHEMA"
- "CI_SNOWFLAKE_DBT_ACCOUNT"
- "CI_SNOWFLAKE_DBT_DATABASE"
- "CI_SNOWFLAKE_DBT_PASS"
- "CI_SNOWFLAKE_DBT_ROLE"
- "CI_SNOWFLAKE_DBT_USER"
- "CI_SNOWFLAKE_DBT_WAREHOUSE"
commands: |
bash .buildkite/scripts/run_models.sh snowflake
curl -s "${RUN_MODELS_URL}" | bash -s snowflake

- label: ":gcloud: Run Tests - BigQuery"
key: "run_dbt_bigquery"
retry:
automatic:
- exit_status: -1 # Timeout
limit: 1
plugins:
- docker#v3.13.0:
image: "python:3.10.13"
image: "python:3.13"
shell: [ "/bin/bash", "-e", "-c" ]
environment:
- "BASH_ENV=/tmp/.bashrc"
- "BUILD_SCHEMA"
- "GCLOUD_SERVICE_KEY"
commands: |
bash .buildkite/scripts/run_models.sh bigquery
curl -s "${RUN_MODELS_URL}" | bash -s bigquery

- label: ":amazon-redshift: Run Tests - Redshift"
key: "run_dbt_redshift"
concurrency: 3
concurrency_group: "warehouse/redshift"
retry:
automatic:
- exit_status: -1 # Timeout
limit: 1
plugins:
- docker#v3.13.0:
image: "python:3.10.13"
image: "python:3.13"
shell: [ "/bin/bash", "-e", "-c" ]
environment:
- "BASH_ENV=/tmp/.bashrc"
- "BUILD_SCHEMA"
- "CI_REDSHIFT_DBT_DBNAME"
- "CI_REDSHIFT_DBT_HOST"
- "CI_REDSHIFT_DBT_PASS"
- "CI_REDSHIFT_DBT_USER"
commands: |
bash .buildkite/scripts/run_models.sh redshift
curl -s "${RUN_MODELS_URL}" | bash -s redshift

- label: ":databricks: Run Tests - Databricks"
key: "run_dbt_databricks"
retry:
automatic:
- exit_status: -1 # Timeout
limit: 1
plugins:
- docker#v3.13.0:
image: "python:3.10.13"
image: "python:3.13"
shell: [ "/bin/bash", "-e", "-c" ]
environment:
- "BASH_ENV=/tmp/.bashrc"
- "BUILD_SCHEMA"
- "CI_DATABRICKS_DBT_CATALOG"
- "CI_DATABRICKS_DBT_HOST"
- "CI_DATABRICKS_DBT_HTTP_PATH"
- "CI_DATABRICKS_DBT_TOKEN"
commands: |
bash .buildkite/scripts/run_models.sh databricks
curl -s "${RUN_MODELS_URL}" | bash -s databricks
24 changes: 0 additions & 24 deletions .buildkite/scripts/run_models.sh
View file
Open in desktop

This file was deleted.

50 changes: 0 additions & 50 deletions integration_tests/ci/sample.profiles.yml
View file
Open in desktop

This file was deleted.

12 changes: 12 additions & 0 deletions integration_tests/ci/test_scenarios.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
schema_variable_name: "linkedin_ads_schema"
default_include_incremental: false

test_scenarios:
- name: "local currency with geo and analytics disabled"
vars:
linkedin_ads__use_local_currency: true
ad_reporting__url_report__using_null_filter: false
linkedin_ads__using_geo: false
linkedin_ads__using_monthly_ad_analytics_by_member_country: false
linkedin_ads__using_monthly_ad_analytics_by_member_region: false
include_incremental: false
2 changes: 1 addition & 1 deletion integration_tests/dbt_project.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ config-version: 2

vars:
linkedin:
linkedin_ads_schema: linkedin_integration_tests_5
linkedin_ads_schema: linkedin_integration_tests
linkedin_ads_account_history_identifier: "linkedin_account_history_data"
linkedin_ads_ad_analytics_by_creative_identifier: "linkedin_ad_analytics_by_creative_data"
linkedin_ads_campaign_group_history_identifier: "linkedin_campaign_group_history_data"
Expand Down
9 changes: 0 additions & 9 deletions integration_tests/requirements.txt
View file
Open in desktop

This file was deleted.