From 859767c0c1fd5a5e9a50937c5d375cab83a1988f Mon Sep 17 00:00:00 2001 From: Joe Markiewicz <74217849+fivetran-joemarkiewicz@users.noreply.github.com> Date: Mon, 23 Jun 2025 09:43:04 -0500 Subject: [PATCH 1/9] Reorder breaking changes in changelog --- CHANGELOG.md | 24 ++++++++++++++++++++---- README.md | 3 ++- dbt_project.yml | 2 +- integration_tests/dbt_project.yml | 2 +- models/src_pendo.yml | 8 +++++--- 5 files changed, 29 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6616f9a..ce31a79 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,27 @@ -# dbt_pendo_source version.version +# dbt_pendo_source v0.6.0 + +## Breaking Change for dbt Core < 1.9.5 +> *Note: This is not relevant to Fivetran Quickstart users.* +Migrated `freshness` from a top-level source property to a source `config` in alignment with [recent updates](https://github.com/dbt-labs/dbt-core/issues/11506) from dbt Core. This will resolve the following deprecation warning that users running dbt >= 1.9.5 may have received: + +``` +[WARNING]: Deprecated functionality +Found `freshness` as a top-level property of `pendo` in file +`models/src_pendo.yml`. The `freshness` top-level property should be moved +into the `config` of `pendo`. +``` + +**IMPORTANT:** Users running dbt Core < 1.9.5 will not be able to utilize freshness tests in this release or any subsequent releases, as older versions of dbt will not recognize freshness as a source `config` and therefore not run the tests. + +If you are using dbt Core < 1.9.5 and want to continue running Pendo Source freshness tests, please elect **one** of the following options: + 1. (Recommended) Upgrade to dbt Core >= 1.9.5 + 2. Do not upgrade your installed version of the `pendo_source` package. Pin your dependency on v0.5.0 in your `packages.yml` file. + 3. Utilize a dbt [override](https://docs.getdbt.com/reference/resource-properties/overrides) to overwrite the package's `pendo` source and apply freshness via the [old](https://github.com/fivetran/dbt_pendo_source/blob/v0.5.0/models/src_pendo.yml#L11-L13) top-level property route. This will require you to copy and paste the entirety of the `src_pendo.yml` [file](https://github.com/fivetran/dbt_pendo_source/blob/v0.5.0/models/src_pendo.yml#L18-L732) and add an `overrides: pendo_source` property. ## Documentation - Corrected references to connectors and connections in the README. ([#27](https://github.com/fivetran/dbt_pendo_source/pull/27)) -# dbt_pendo_source v0.UPDATE.UPDATE - - ## Under the Hood: +## Under the Hood - Incorporated the new `fivetran_utils.drop_schemas_automation` macro into the end of each Buildkite integration test job. - Updated the pull request [templates](/.github). diff --git a/README.md b/README.md index c774733..eb19d00 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ - Materializes [Pendo staging tables](https://fivetran.github.io/dbt_pendo_source/#!/overview/pendo_source/models/?g_v=1) which beverage data in the format described by [this ERD](https://fivetran.com/docs/applications/pendo/#schemainformation). These staging tables clean, test, and prepare your Pendo data from [Fivetran's connector](https://fivetran.com/docs/applications/pendo) for analysis by doing the following: - Name columns for consistency across all packages and for easier analysis - Adds freshness tests to source data + - dbt Core >= 1.9.5 is required to run freshness tests out of the box. - Adds column-level testing where applicable. For example, all primary keys are tested for uniqueness and non-null values. - Generates a comprehensive data dictionary of your Pendo data through the [dbt docs site](https://fivetran.github.io/dbt_pendo_source/). - These tables are designed to work simultaneously with our [Pendo transformation package](https://github.com/fivetran/dbt_pendo). @@ -39,7 +40,7 @@ If you are **not** using the [Pendo transformation package](https://github.com/f ```yaml packages: - package: fivetran/pendo_source - version: [">=0.5.0", "<0.6.0"] # we recommend using ranges to capture non-breaking changes automatically + version: [">=0.6.0", "<0.7.0"] # we recommend using ranges to capture non-breaking changes automatically ``` ### Step 3: Define database and schema variables diff --git a/dbt_project.yml b/dbt_project.yml index 8543686..c790499 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,6 +1,6 @@ config-version: 2 name: 'pendo_source' -version: '0.5.0' +version: '0.6.0' require-dbt-version: [">=1.3.0", "<2.0.0"] models: pendo_source: diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 9fc67e7..783e8c1 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -1,7 +1,7 @@ config-version: 2 name: 'pendo_source_integration_tests' -version: '0.5.0' +version: '0.6.0' profile: 'integration_tests' diff --git a/models/src_pendo.yml b/models/src_pendo.yml index 1a0d2b4..8bd12a9 100644 --- a/models/src_pendo.yml +++ b/models/src_pendo.yml @@ -8,9 +8,11 @@ sources: loader: fivetran loaded_at_field: _fivetran_synced - freshness: - warn_after: {count: 72, period: hour} - error_after: {count: 168, period: hour} + + config: + freshness: + warn_after: {count: 72, period: hour} + error_after: {count: 168, period: hour} tables: From 18f4fd6871e91dbc89bc84314e9c27d782df675c Mon Sep 17 00:00:00 2001 From: Joe Markiewicz <74217849+fivetran-joemarkiewicz@users.noreply.github.com> Date: Mon, 23 Jun 2025 10:55:45 -0500 Subject: [PATCH 2/9] Update requirements.txt --- integration_tests/requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/integration_tests/requirements.txt b/integration_tests/requirements.txt index 87ae0f9..c9f6024 100644 --- a/integration_tests/requirements.txt +++ b/integration_tests/requirements.txt @@ -4,4 +4,5 @@ dbt-redshift>=1.3.0,<2.0.0 dbt-postgres>=1.3.0,<2.0.0 dbt-spark>=1.3.0,<2.0.0 dbt-spark[PyHive]>=1.3.0,<2.0.0 -dbt-databricks>=1.3.0,<2.0.0 \ No newline at end of file +dbt-databricks>=1.3.0,<2.0.0 +certifi==2025.1.31 From 533b45dbcd5128de7bb56899110e1a529d630884 Mon Sep 17 00:00:00 2001 From: Joe Markiewicz <74217849+fivetran-joemarkiewicz@users.noreply.github.com> Date: Tue, 24 Jun 2025 14:22:29 -0500 Subject: [PATCH 3/9] Update run_models.sh --- .buildkite/scripts/run_models.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.buildkite/scripts/run_models.sh b/.buildkite/scripts/run_models.sh index 5250872..224eadc 100644 --- a/.buildkite/scripts/run_models.sh +++ b/.buildkite/scripts/run_models.sh @@ -20,5 +20,6 @@ dbt seed --target "$db" --full-refresh dbt compile --target "$db" dbt run --target "$db" --full-refresh dbt test --target "$db" +dbt source freshness --target "$db" || echo "...Only verifying freshness runs..." dbt run-operation fivetran_utils.drop_schemas_automation --target "$db" From 4f1f44356778c3eaec8140c971748d2ce9cea873 Mon Sep 17 00:00:00 2001 From: Joe Markiewicz <74217849+fivetran-joemarkiewicz@users.noreply.github.com> Date: Tue, 24 Jun 2025 14:52:59 -0500 Subject: [PATCH 4/9] Update requirements.txt --- integration_tests/requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/integration_tests/requirements.txt b/integration_tests/requirements.txt index c9f6024..40b0633 100644 --- a/integration_tests/requirements.txt +++ b/integration_tests/requirements.txt @@ -3,6 +3,5 @@ dbt-bigquery>=1.3.0,<2.0.0 dbt-redshift>=1.3.0,<2.0.0 dbt-postgres>=1.3.0,<2.0.0 dbt-spark>=1.3.0,<2.0.0 -dbt-spark[PyHive]>=1.3.0,<2.0.0 dbt-databricks>=1.3.0,<2.0.0 certifi==2025.1.31 From 02a3135ec4500bd7b40a1d1e1391ec8a8a6eff90 Mon Sep 17 00:00:00 2001 From: Joe Markiewicz <74217849+fivetran-joemarkiewicz@users.noreply.github.com> Date: Tue, 24 Jun 2025 15:31:03 -0500 Subject: [PATCH 5/9] Update requirements.txt --- integration_tests/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/integration_tests/requirements.txt b/integration_tests/requirements.txt index 40b0633..da17c67 100644 --- a/integration_tests/requirements.txt +++ b/integration_tests/requirements.txt @@ -1,3 +1,4 @@ +dbt-core>=1.3.0,<2.0.0 dbt-snowflake>=1.3.0,<2.0.0 dbt-bigquery>=1.3.0,<2.0.0 dbt-redshift>=1.3.0,<2.0.0 From daed3793827ddf9575e3ae67d5e238500cf825b9 Mon Sep 17 00:00:00 2001 From: Joe Markiewicz <74217849+fivetran-joemarkiewicz@users.noreply.github.com> Date: Tue, 24 Jun 2025 15:46:58 -0500 Subject: [PATCH 6/9] Update run_models.sh --- .buildkite/scripts/run_models.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/run_models.sh b/.buildkite/scripts/run_models.sh index 224eadc..d13fe77 100644 --- a/.buildkite/scripts/run_models.sh +++ b/.buildkite/scripts/run_models.sh @@ -8,7 +8,7 @@ apt-get install libsasl2-dev python3 -m venv venv . venv/bin/activate pip install --upgrade pip setuptools -pip install -r integration_tests/requirements.txt +pip install --no-pre -r integration_tests/requirements.txt mkdir -p ~/.dbt cp integration_tests/ci/sample.profiles.yml ~/.dbt/profiles.yml From 202c30c5f9493b7eb37d92ba7cf81ea64250da15 Mon Sep 17 00:00:00 2001 From: Joe Markiewicz <74217849+fivetran-joemarkiewicz@users.noreply.github.com> Date: Tue, 24 Jun 2025 16:11:47 -0500 Subject: [PATCH 7/9] image changes --- .buildkite/pipeline.yml | 10 +++++----- .buildkite/scripts/run_models.sh | 2 +- integration_tests/requirements.txt | 1 - 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 281f1d4..2603602 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -3,7 +3,7 @@ steps: key: "run-dbt-postgres" plugins: - docker#v3.13.0: - image: "python:3.8" + image: "python:3.10.13" shell: [ "/bin/bash", "-e", "-c" ] environment: - "BASH_ENV=/tmp/.bashrc" @@ -18,7 +18,7 @@ steps: key: "run_dbt_snowflake" plugins: - docker#v3.13.0: - image: "python:3.8" + image: "python:3.10.13" shell: [ "/bin/bash", "-e", "-c" ] environment: - "BASH_ENV=/tmp/.bashrc" @@ -35,7 +35,7 @@ steps: key: "run_dbt_bigquery" plugins: - docker#v3.13.0: - image: "python:3.8" + image: "python:3.10.13" shell: [ "/bin/bash", "-e", "-c" ] environment: - "BASH_ENV=/tmp/.bashrc" @@ -47,7 +47,7 @@ steps: key: "run_dbt_redshift" plugins: - docker#v3.13.0: - image: "python:3.8" + image: "python:3.10.13" shell: [ "/bin/bash", "-e", "-c" ] environment: - "BASH_ENV=/tmp/.bashrc" @@ -62,7 +62,7 @@ steps: key: "run_dbt_databricks" plugins: - docker#v3.13.0: - image: "python:3.8" + image: "python:3.10.13" shell: [ "/bin/bash", "-e", "-c" ] environment: - "BASH_ENV=/tmp/.bashrc" diff --git a/.buildkite/scripts/run_models.sh b/.buildkite/scripts/run_models.sh index d13fe77..224eadc 100644 --- a/.buildkite/scripts/run_models.sh +++ b/.buildkite/scripts/run_models.sh @@ -8,7 +8,7 @@ apt-get install libsasl2-dev python3 -m venv venv . venv/bin/activate pip install --upgrade pip setuptools -pip install --no-pre -r integration_tests/requirements.txt +pip install -r integration_tests/requirements.txt mkdir -p ~/.dbt cp integration_tests/ci/sample.profiles.yml ~/.dbt/profiles.yml diff --git a/integration_tests/requirements.txt b/integration_tests/requirements.txt index da17c67..40b0633 100644 --- a/integration_tests/requirements.txt +++ b/integration_tests/requirements.txt @@ -1,4 +1,3 @@ -dbt-core>=1.3.0,<2.0.0 dbt-snowflake>=1.3.0,<2.0.0 dbt-bigquery>=1.3.0,<2.0.0 dbt-redshift>=1.3.0,<2.0.0 From ed9378418ed082adf6c0b9ac6f6e8beb335c6593 Mon Sep 17 00:00:00 2001 From: Joe Markiewicz <74217849+fivetran-joemarkiewicz@users.noreply.github.com> Date: Tue, 24 Jun 2025 16:23:42 -0500 Subject: [PATCH 8/9] remove flag --- integration_tests/ci/sample.profiles.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml index 89f76e6..e794bd8 100644 --- a/integration_tests/ci/sample.profiles.yml +++ b/integration_tests/ci/sample.profiles.yml @@ -4,7 +4,6 @@ config: send_anonymous_usage_stats: False - use_colors: True integration_tests: target: redshift From 5b70e2cd603121ef214a68854d9e998fd9eeff5b Mon Sep 17 00:00:00 2001 From: Joe Markiewicz <74217849+fivetran-joemarkiewicz@users.noreply.github.com> Date: Tue, 24 Jun 2025 16:34:09 -0500 Subject: [PATCH 9/9] moved config --- integration_tests/ci/sample.profiles.yml | 3 --- integration_tests/dbt_project.yml | 5 ++++- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml index e794bd8..06a2e6d 100644 --- a/integration_tests/ci/sample.profiles.yml +++ b/integration_tests/ci/sample.profiles.yml @@ -2,9 +2,6 @@ # HEY! This file is used in the dbt package integrations tests with Buildkite. # You should __NEVER__ check credentials into version control. Thanks for reading :) -config: - send_anonymous_usage_stats: False - integration_tests: target: redshift outputs: diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 783e8c1..6cea547 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -75,4 +75,7 @@ seeds: last_updated_by_user_id: "{{ 'int64' if target.name == 'bigquery' else 'bigint' }}" visitor_history: +column_types: - id_hash: "{{ 'int64' if target.name == 'bigquery' else 'bigint' }}" \ No newline at end of file + id_hash: "{{ 'int64' if target.name == 'bigquery' else 'bigint' }}" + +flags: + send_anonymous_usage_stats: False \ No newline at end of file