From 67dd603876b7d1480261d75d90e7799a3843b4cf Mon Sep 17 00:00:00 2001 From: talhaahmad-supp Date: Mon, 8 Jun 2026 16:42:48 +0100 Subject: [PATCH] fix(cdp_monitoring): skip 422 errors per item instead of crashing A single 422 response from the CDP API was raising an unhandled HTTPError and aborting the entire ingest loop. Wrap each per-item call in try/except so bad items are skipped and logged while all other records continue to ingest normally. Co-Authored-By: Claude Sonnet 4.6 --- .../cdp_monitoring/scripts/ingest_activation.py | 5 ++++- .../cdp_monitoring/scripts/ingest_journey_activation.py | 9 ++++++--- .../cdp_monitoring/scripts/ingest_journey_summary.py | 9 ++++++--- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/scenarios/monitoring/cdp_monitoring/scripts/ingest_activation.py b/scenarios/monitoring/cdp_monitoring/scripts/ingest_activation.py index 240f0398..cdd3aa1a 100644 --- a/scenarios/monitoring/cdp_monitoring/scripts/ingest_activation.py +++ b/scenarios/monitoring/cdp_monitoring/scripts/ingest_activation.py @@ -18,7 +18,10 @@ def get_activations_per_audience(base_url, headers, id): def get_all_activations(base_url, headers, id_list): l = [] for i in id_list: - l.extend(get_activations_per_audience(base_url=base_url, headers=headers, id=i)) + try: + l.extend(get_activations_per_audience(base_url=base_url, headers=headers, id=i)) + except requests.exceptions.HTTPError as e: + print(f'Skipping audience {i}: {e}') return l def insert_activations(import_unixtime, endpoint, apikey, dest_db, dest_table, activations): diff --git a/scenarios/monitoring/cdp_monitoring/scripts/ingest_journey_activation.py b/scenarios/monitoring/cdp_monitoring/scripts/ingest_journey_activation.py index 0ec811f7..fd806ecb 100644 --- a/scenarios/monitoring/cdp_monitoring/scripts/ingest_journey_activation.py +++ b/scenarios/monitoring/cdp_monitoring/scripts/ingest_journey_activation.py @@ -22,9 +22,12 @@ def get_journey_activation(base_url, headers, id): def get_all_journey_activation(base_url, headers, ids): l = [] for i in ids: - d = get_journey_activation(base_url, headers, i) - if d != None: - l.extend(d) + try: + d = get_journey_activation(base_url, headers, i) + if d != None: + l.extend(d) + except requests.exceptions.HTTPError as e: + print(f'Skipping journey {i}: {e}') return l def run(session_unixtime, dest_db, dest_table, journey_ids, api_endpoint='api.treasuredata.com', cdp_api_endpoint='api-cdp.treasuredata.com'): diff --git a/scenarios/monitoring/cdp_monitoring/scripts/ingest_journey_summary.py b/scenarios/monitoring/cdp_monitoring/scripts/ingest_journey_summary.py index 444888d3..ee03b4d5 100644 --- a/scenarios/monitoring/cdp_monitoring/scripts/ingest_journey_summary.py +++ b/scenarios/monitoring/cdp_monitoring/scripts/ingest_journey_summary.py @@ -24,9 +24,12 @@ def get_journey_summary(base_url, headers, id): def get_all_journey_summary(base_url, headers, ids): l = [] for i in ids: - d = get_journey_summary(base_url, headers, i) - if d != None: - l.append(d) + try: + d = get_journey_summary(base_url, headers, i) + if d != None: + l.append(d) + except requests.exceptions.HTTPError as e: + print(f'Skipping journey {i}: {e}') return l def run(session_unixtime, dest_db, dest_table, journey_ids, api_endpoint='api.treasuredata.com', cdp_api_endpoint='api-cdp.treasuredata.com'):