MPT-12684 otel config as library. better config handling

Author: ffaraone

.dockerignore

@@ -1,6 +1,7 @@
 .git
 .idea
 .vscode
+.venv
 
 **/.env.sample
 **/.storybook

.github/workflows/ffc_pr_build_and_test.yaml

@@ -48,6 +48,7 @@ jobs:
       trapper_changed: ${{ steps.check_trapper.outputs.changed }}
       users_dataset_generator_changed: ${{ steps.check_users_dataset_generator.outputs.changed }}
       webhook_executor_changed: ${{ steps.check_webhook_executor.outputs.changed }}
+      tools_changed: ${{ steps.check_tools.outputs.changed }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -362,6 +363,15 @@ jobs:
             echo "changed=false" >> $GITHUB_OUTPUT
           fi
 
+      - name: Check if 'tools/**' changed
+        id: check_tools
+        run: |
+          if git diff --name-only origin/${{ github.base_ref }}...HEAD | grep -q '^tools/'; then
+            echo "changed=true" >> $GITHUB_OUTPUT
+          else
+            echo "changed=false" >> $GITHUB_OUTPUT
+          fi
+
   auth_tests:
     if: needs.determine_changes.outputs.auth_changed == 'true'
     needs: determine_changes
@@ -817,6 +827,16 @@ jobs:
         run: bash -x build.sh webhook_executor build
       - name: Build test image and run tests
         run: bash -x docker_images/run_test.sh webhook_executor
+  tools_tests:
+    if: needs.determine_changes.outputs.tools_changed == 'true'
+    needs: determine_changes
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Build test image and run tests
+        run: bash -x tools/run_test.sh
 
   pr_notification:
     needs:
@@ -858,6 +878,7 @@ jobs:
       - trapper_tests
       - users_dataset_generator_tests
       - webhook_executor_tests
+      - tools_tests
     runs-on: ubuntu-latest
     if: always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled')
     steps:

.gitignore

@@ -21,6 +21,10 @@ rest_api/.clickhouse
 **/dist
 **/*egg-info/
 **/*.tar.gz
+htmlcov
+.coverage
+coverage.xml
+.cache
 
 # ngui
 ngui/server/.env
@@ -34,14 +38,6 @@ ngui/*/storybook
 ngui/server/package-lock.json
 ngui/ui/package-lock.json
 
-# ffc_ngui
-ffc_ngui/server/.env
-ffc_ngui/ui/.env
-ffc_ngui/*/node_modules
-ffc_ngui/*/build/
-ffc_ngui/*/dist/
-ffc_ngui/*/storybook
-
 **/npm-debug.log*
 **/yarn-debug.log*
 **/yarn-error.log*
@@ -62,3 +58,4 @@ optscale-deploy/.vagrant/
 /e2etests/.cache/
 /e2etests/.cache/*
 /e2etests/tests/downloads/*
+

auth/Dockerfile

@@ -26,6 +26,9 @@ COPY auth/auth_server/*.py ./auth/auth_server/
 COPY auth/auth_server/alembic.template auth/auth_server/alembic.template
 COPY auth/auth_server/swagger auth/auth_server/swagger
 
+COPY auth/pyproject.toml auth/pyproject.toml
+COPY auth/uv.lock auth/uv.lock
+
 RUN uv --project auth sync --locked --no-dev
 RUN uv --project auth run python -u auth/auth_server/write_spec.py
 

auth/auth_server/handlers/v1/base.py

@@ -10,6 +10,7 @@
 from tools.optscale_exceptions.common_exc import (WrongArgumentsException,
                                                   UnauthorizedException)
 from tools.optscale_exceptions.http_exc import OptHTTPError
+from tools.optscale_telemetry import get_trace_headers
 
 LOG = logging.getLogger(__name__)
 
@@ -60,6 +61,10 @@ def put(self, *args, **kwargs):
     def options(self, *args, **kwargs):
         self.raise405()
 
+    def prepare(self):
+        for name, value in get_trace_headers().items():
+            self.set_header(name, value)
+
     def _get_request(self):
         return self.request
 

auth/auth_server/server.py

@@ -13,6 +13,7 @@
 from auth.auth_server.handlers.v1.base import DefaultHandler
 from auth.auth_server.handlers.v1.swagger import SwaggerStaticFileHandler
 from auth.auth_server.models.db_factory import DBType, DBFactory
+from tools.optscale_telemetry import OpenTelemetryConfig
 
 import optscale_client.config_client.client
 
@@ -103,6 +104,16 @@ def make_app(db_type, etcd_host, etcd_port, wait=False):
         config_cl.wait_configured()
     db = DBFactory(db_type, config_cl).db
     db.create_schema()
+
+    config = OpenTelemetryConfig(
+        service_name=os.getenv("OTEL_SERVICE_NAME", "auth"),
+        service_version=os.getenv("OTEL_SERVICE_VERSION", "local"),
+        otel_config=config_cl.read_branch("/opentelemetry"),
+        service_config=config_cl.read_branch("auth/opentelemetry"),
+        sqlalchemy_engine=db.engine,
+    )
+    config.setup_open_telemetry()
+
     handler_kwargs = {
         "engine": db.engine,
         "config": config_cl,

auth/auth_server/tests/unittests/test_api_base.py

@@ -38,6 +38,10 @@ def get_auth_client(version="v1"):
         }.get(version)
 
     def setUp(self, version='v1'):
+        patch(
+            'optscale_client.config_client.client.Client.read_branch',
+            return_value=None,
+        ).start()
         super().setUp()
         secret = gen_id()
         patch('optscale_client.config_client.client.Client.cluster_secret',

auth/pyproject.toml

@@ -12,6 +12,7 @@ dependencies = [
     "optscale-exceptions",
     "optscale-password",
     "optscale-time",
+    "optscale-telemetry",
     "ordered-set==4.1.0",
     "pyjwt>=2.4.0",
     "pymacaroons==0.9.2",
@@ -39,6 +40,7 @@ config-client = { path = "../optscale_client/config_client" }
 restapi-client = { path = "../optscale_client/rest_api_client" }
 auth-client = { path = "../optscale_client/auth_client" }
 check-alembic-down-revisions = { path = "../tools/check_alembic_down_revisions" }
+optscale-telemetry = { path = "../tools/optscale_telemetry" }
 
 [dependency-groups]
 dev = [
@@ -48,4 +50,6 @@ dev = [
     "freezegun==0.3.8",
     "pycodestyle==2.11.1",
     "pylint==3.0.3",
+    "pytest==8.4.2",
+    "pytest-xdist==3.6.1",
 ]

auth/run_test.sh

@@ -23,7 +23,7 @@ echo "<<Alembic down revision tests"
 
 echo "Unit tests>>>"
 docker run -i --rm ${TEST_IMAGE} \
-    bash -c "uv --project auth run python -m unittest discover ./auth/auth_server/tests"
+    bash -c "uv --project auth run pytest -n auto auth --disable-warnings"
 echo "<<Unit tests"
 
 docker rmi ${TEST_IMAGE}