From fceb9dbed1f7d060b9ad47ba5d590800c4d5e8a7 Mon Sep 17 00:00:00 2001 From: Lukasz Gryglicki Date: Thu, 5 Mar 2026 09:49:54 +0100 Subject: [PATCH 1/4] Add env/stage filter for DDog and collapse more API URLs into single templates Signed-off-by: Lukasz Gryglicki Assisted by [OpenAI](https://platform.openai.com/) Assisted by [GitHub Copilot](https://github.com/features/copilot) --- cla-backend-go/telemetry/datadog_otlp.go | 1 + cla-backend/cla/routes.py | 1 + utils/otel_dd/api_usage_stats_ddog.py | 14 ++++- utils/otel_dd/check_spans_in_ddog.sh | 73 ++++++++++++++++++------ 4 files changed, 68 insertions(+), 21 deletions(-) diff --git a/cla-backend-go/telemetry/datadog_otlp.go b/cla-backend-go/telemetry/datadog_otlp.go index a1b37127a..8b176fd06 100644 --- a/cla-backend-go/telemetry/datadog_otlp.go +++ b/cla-backend-go/telemetry/datadog_otlp.go @@ -191,6 +191,7 @@ func WrapHTTPHandler(next http.Handler) http.Handler { p = reUUIDLike.ReplaceAllString(p, "/{invalid-uuid}$1") p = reUUIDHexDash36.ReplaceAllString(p, "/{invalid-uuid}$1") p = reNumericID.ReplaceAllString(p, "/{id}$1") + p = reNumericID.ReplaceAllString(p, "/{id}$1") // Salesforce IDs: valid vs invalid p = reSFIDValid.ReplaceAllString(p, "/{sfid}$1") p = reSFIDLike.ReplaceAllString(p, "/{invalid-sfid}$1") diff --git a/cla-backend/cla/routes.py b/cla-backend/cla/routes.py index f3fb2bcd9..8e7908c85 100755 --- a/cla-backend/cla/routes.py +++ b/cla-backend/cla/routes.py @@ -108,6 +108,7 @@ def _sanitize_api_path(path: str) -> str: p = _RE_UUID_LIKE.sub(r"/{invalid-uuid}\1", p) p = _RE_UUID_HEXDASH_36.sub(r"/{invalid-uuid}\1", p) p = _RE_NUMERIC_ID.sub(r"/{id}\1", p) + p = _RE_NUMERIC_ID.sub(r"/{id}\1", p) p = _RE_SFID_VALID.sub(r"/{sfid}\1", p) p = _RE_SFID_LIKE.sub(r"/{invalid-sfid}\1", p) p = _RE_LFXID_VALID.sub(r"/{lfxid}\1", p) diff --git a/utils/otel_dd/api_usage_stats_ddog.py b/utils/otel_dd/api_usage_stats_ddog.py index 003d6232c..6b8086765 100755 --- a/utils/otel_dd/api_usage_stats_ddog.py +++ b/utils/otel_dd/api_usage_stats_ddog.py @@ -26,12 +26,18 @@ import datetime as dt import json import os +import re import sys import urllib.error import urllib.request from typing import Any, Dict, List, Optional, Tuple +_ENV_RE = re.compile(r"(^|\s)env\s*:") + +def has_env_filter(query: str) -> bool: + return bool(_ENV_RE.search(query)) + def eprint(*args: Any) -> None: print(*args, file=sys.stderr) @@ -193,11 +199,15 @@ def main() -> int: p.add_argument("--from", dest="time_from", default="now-60m", help='Time range start (Datadog format), default "now-60m"') p.add_argument("--to", dest="time_to", default="now", help='Time range end (Datadog format), default "now"') - p.add_argument("--query", default="service:easycla-backend env:dev", help='Datadog query string (default: "service:easycla-backend env:dev")') p.add_argument("--limit", type=int, default=5000, help="Page limit per request (default: 5000)") p.add_argument("--verbose", action="store_true", help="Log progress to stderr") + p.add_argument("--env", "--environment", "--stage", dest="env", default=os.getenv("DD_ENV") or os.getenv("ENV") or os.getenv("STAGE") or "dev", help='Datadog env tag value (default: DD_ENV/ENV/STAGE or "dev")') + p.add_argument("--query", default="service:easycla-backend", help='Datadog query string WITHOUT env (env is appended unless query already contains env:...) (default: "service:easycla-backend")') args = p.parse_args() + query = (args.query or "").strip() + if args.env and not has_env_filter(query): + query = f"{query} env:{args.env}".strip() if query else f"env:{args.env}" # Default skip-e2e unless explicitly --no-skip-e2e skip_e2e = True @@ -217,7 +227,7 @@ def main() -> int: dd_site=dd_site, # type: ignore[arg-type] dd_api_key=dd_api_key, # type: ignore[arg-type] dd_app_key=dd_app_key, # type: ignore[arg-type] - query=args.query, + query=query, time_from=args.time_from, time_to=args.time_to, limit=args.limit, diff --git a/utils/otel_dd/check_spans_in_ddog.sh b/utils/otel_dd/check_spans_in_ddog.sh index 4d0e4658f..9fd421902 100755 --- a/utils/otel_dd/check_spans_in_ddog.sh +++ b/utils/otel_dd/check_spans_in_ddog.sh @@ -1,32 +1,67 @@ #!/bin/bash +set -euo pipefail + # Example: -# ./utils/otel_dd/check_spans_in_ddog.sh --skip-e2e | jq -r '.data[].attributes.custom.http.route' | sort | uniq +# ./utils/otel_dd/check_spans_in_ddog.sh --env prod --skip-e2e \ +# | jq -r '.data[].attributes.custom.http.route' | sort | uniq + +usage() { + cat <<'EOF' >&2 +Usage: + check_spans_in_ddog.sh [--env ] [--stage ] [--skip-e2e|--no-skip-e2e] + +Env vars: + DD_SITE, DD_API_KEY, DD_APP_KEY (required) + DD_ENV / ENV / STAGE (optional; default "dev") + DD_SERVICE (optional; default "easycla-backend") + +Notes: + - Filters Datadog span events by: service: env: + - Skips spans where attributes.custom.easycla.e2e == "true" when --skip-e2e +EOF +} SKIP_E2E=0 -for arg in "$@"; do - case "$arg" in - --skip-e2e) SKIP_E2E=1 ;; - --no-skip-e2e) SKIP_E2E=0 ;; - *) ;; +DD_ENV="${DD_ENV:-${ENV:-${STAGE:-dev}}}" +DD_SERVICE="${DD_SERVICE:-easycla-backend}" + +while [[ $# -gt 0 ]]; do + case "$1" in + --skip-e2e) SKIP_E2E=1; shift ;; + --no-skip-e2e) SKIP_E2E=0; shift ;; + --env|--environment|--stage) + [[ $# -ge 2 ]] || { echo "ERROR: $1 requires a value" >&2; usage; exit 2; } + DD_ENV="$2" + shift 2 + ;; + --service) + [[ $# -ge 2 ]] || { echo "ERROR: --service requires a value" >&2; usage; exit 2; } + DD_SERVICE="$2" + shift 2 + ;; + -h|--help) usage; exit 0 ;; + *) + echo "ERROR: unknown arg: $1" >&2 + usage + exit 2 + ;; esac done -payload='{ - "data": { - "type": "search_request", - "attributes": { - "filter": { - "from": "now-60m", - "to": "now", - "query": "service:easycla-backend env:dev" - }, - "sort": "timestamp", - "page": { "limit": 5000 } +QUERY="service:${DD_SERVICE} env:${DD_ENV}" + +# Build request JSON safely with jq (avoids quoting bugs) +payload="$(jq -n --arg query "$QUERY" '{ + data: { + type: "search_request", + attributes: { + filter: { from: "now-60m", to: "now", query: $query }, + sort: "timestamp", + page: { limit: 5000 } } } -}' +}')" -# jq helper: treat missing as false; accept "true"/true jq_filter=' def is_e2e: (.attributes.custom.easycla.e2e // false) From 9bfadf3f7233c247875bf15b13dbdcd92f84dd18 Mon Sep 17 00:00:00 2001 From: Lukasz Gryglicki Date: Thu, 5 Mar 2026 10:23:57 +0100 Subject: [PATCH 2/4] Add missing license header Signed-off-by: Lukasz Gryglicki Assisted by [OpenAI](https://platform.openai.com/) Assisted by [GitHub Copilot](https://github.com/features/copilot) --- cla-backend/cla/tests/unit/test_jwt_auth.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cla-backend/cla/tests/unit/test_jwt_auth.py b/cla-backend/cla/tests/unit/test_jwt_auth.py index 4d04fe26d..df1441a3e 100644 --- a/cla-backend/cla/tests/unit/test_jwt_auth.py +++ b/cla-backend/cla/tests/unit/test_jwt_auth.py @@ -1,3 +1,6 @@ +# Copyright The Linux Foundation and each contributor to CommunityBridge. +# SPDX-License-Identifier: MIT + """Unit tests validating the PyJWT migration does not break JWT/auth processing. These tests are intentionally offline: From e584048421da1c5574b9a750f496f2b4f98dc08e Mon Sep 17 00:00:00 2001 From: Lukasz Gryglicki Date: Thu, 5 Mar 2026 10:25:17 +0100 Subject: [PATCH 3/4] Add missing license header 2 Signed-off-by: Lukasz Gryglicki Assisted by [OpenAI](https://platform.openai.com/) Assisted by [GitHub Copilot](https://github.com/features/copilot) --- utils/otel_dd/api_usage_stats_ddog.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/utils/otel_dd/api_usage_stats_ddog.py b/utils/otel_dd/api_usage_stats_ddog.py index 6b8086765..cef507142 100755 --- a/utils/otel_dd/api_usage_stats_ddog.py +++ b/utils/otel_dd/api_usage_stats_ddog.py @@ -1,3 +1,6 @@ +# Copyright The Linux Foundation and each contributor to CommunityBridge. +# SPDX-License-Identifier: MIT + #!/usr/bin/env python3 """ Query Datadog span events and output per-route API usage statistics as CSV. From f7730ba091083c0982a468a68638c9ae6eb79cd7 Mon Sep 17 00:00:00 2001 From: Lukasz Gryglicki Date: Thu, 5 Mar 2026 10:27:06 +0100 Subject: [PATCH 4/4] Add missing license header 3 Signed-off-by: Lukasz Gryglicki Assisted by [OpenAI](https://platform.openai.com/) Assisted by [GitHub Copilot](https://github.com/features/copilot) --- utils/otel_dd/api_usage_stats_ddog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/otel_dd/api_usage_stats_ddog.py b/utils/otel_dd/api_usage_stats_ddog.py index cef507142..941216dbf 100755 --- a/utils/otel_dd/api_usage_stats_ddog.py +++ b/utils/otel_dd/api_usage_stats_ddog.py @@ -1,7 +1,7 @@ +#!/usr/bin/env python3 # Copyright The Linux Foundation and each contributor to CommunityBridge. # SPDX-License-Identifier: MIT -#!/usr/bin/env python3 """ Query Datadog span events and output per-route API usage statistics as CSV.