From 35e79578eef8cc818c5b11835bb05bb9a6f4ba80 Mon Sep 17 00:00:00 2001 From: Jitendra Dhawan Date: Wed, 8 Apr 2026 08:48:25 +0530 Subject: [PATCH 01/24] Add metrics instrumentation for service health checks and upstream availability --- pom.xml | 6 ++++ ranger-core/pom.xml | 10 +++++++ .../appform/ranger/core/util/FinderUtils.java | 1 + .../ranger/core/util/MetricRecorder.java | 29 +++++++++++++++++++ .../ranger/discovery/core/Constants.java | 2 ++ .../core/ServiceDiscoveryConfiguration.java | 3 ++ .../drove/config/DroveUpstreamConfig.java | 4 +++ .../ranger/http/config/HttpClientConfig.java | 1 + .../servicefinder/HttpApiCommunicator.java | 15 +++++----- .../servicefinder/HttpNodeDataSource.java | 7 ++++- .../HttpServiceDataSource.java | 2 ++ .../serviceprovider/HttpNodeDataSink.java | 2 ++ .../RangerZkUpstreamConfiguration.java | 4 +++ .../common/ZkNodeDataStoreConnector.java | 5 +++- .../servicefinder/ZkNodeDataSource.java | 3 ++ .../servicefinderhub/ZkServiceDataSource.java | 2 ++ .../serviceprovider/ZkNodeDataSink.java | 2 ++ 17 files changed, 88 insertions(+), 10 deletions(-) create mode 100644 ranger-core/src/main/java/io/appform/ranger/core/util/MetricRecorder.java diff --git a/pom.xml b/pom.xml index a53d3b75..f92aa02c 100644 --- a/pom.xml +++ b/pom.xml @@ -147,6 +147,7 @@ 1.1.10.8 4.2.0 3.3.2 + 1.0.16 @@ -229,6 +230,11 @@ ${xmlunit-core.version} test + + io.appform.functionmetrics + function-metrics + ${function.metrics.version} + diff --git a/ranger-core/pom.xml b/ranger-core/pom.xml index a25aa8d4..af87a831 100644 --- a/ranger-core/pom.xml +++ b/ranger-core/pom.xml @@ -47,6 +47,16 @@ failsafe ${failsafe.version} + + io.dropwizard + dropwizard-metrics + ${dropwizard.version} + provided + + + io.appform.functionmetrics + function-metrics + diff --git a/ranger-core/src/main/java/io/appform/ranger/core/util/FinderUtils.java b/ranger-core/src/main/java/io/appform/ranger/core/util/FinderUtils.java index 412486b4..e1ba1c76 100644 --- a/ranger-core/src/main/java/io/appform/ranger/core/util/FinderUtils.java +++ b/ranger-core/src/main/java/io/appform/ranger/core/util/FinderUtils.java @@ -62,6 +62,7 @@ public static boolean isValidNode( return false; } if(serviceNode.getLastUpdatedTimeStamp() < healthcheckZombieCheckThresholdTime) { + MetricRecorder.recordZombieNodeFound(service.getServiceName()); log.warn("Zombie node [{}:{}] found for [{}]", serviceNode.getHost(), serviceNode.getPort(), service.getServiceName()); return false; diff --git a/ranger-core/src/main/java/io/appform/ranger/core/util/MetricRecorder.java b/ranger-core/src/main/java/io/appform/ranger/core/util/MetricRecorder.java new file mode 100644 index 00000000..060fd2f6 --- /dev/null +++ b/ranger-core/src/main/java/io/appform/ranger/core/util/MetricRecorder.java @@ -0,0 +1,29 @@ +package io.appform.ranger.core.util; + +import com.codahale.metrics.MetricRegistry; +import lombok.experimental.UtilityClass; + +@UtilityClass +public class MetricRecorder { + + private static final String PACKAGE_PREFIX = "io.appform.ranger"; + + private static MetricRegistry metricRegistry = new MetricRegistry(); + + public static void initialize(MetricRegistry registry) { + metricRegistry = registry; + } + + public static void recordZombieNodeFound(String serviceName) { + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, "zombieNodes")).mark(); + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, "zombieNodes", "service", serviceName)).mark(); + } + + public static void recordZkConnection(boolean zkConnectionActive) { + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX,"zkConnection", zkConnectionActive ? "active" : "inactive")).mark(); + } + + public static void recordHttpUpstreamAvailability(String clientId, boolean httpUpstreamAvailable) { + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX,"httpUpstream", clientId, httpUpstreamAvailable ? "active" : "inactive")).mark(); + } +} diff --git a/ranger-discovery-core/src/main/java/io/appform/ranger/discovery/core/Constants.java b/ranger-discovery-core/src/main/java/io/appform/ranger/discovery/core/Constants.java index 0ea6e15a..50b83eb3 100644 --- a/ranger-discovery-core/src/main/java/io/appform/ranger/discovery/core/Constants.java +++ b/ranger-discovery-core/src/main/java/io/appform/ranger/discovery/core/Constants.java @@ -24,6 +24,8 @@ @UtilityClass public class Constants { public static final String DEFAULT_NAMESPACE = "default"; + public static final String DEFAULT_ID = "RANGER_DISCOVERY"; + public static final String DEFAULT_HOST = "__DEFAULT_SERVICE_HOST"; public static final int DEFAULT_PORT = -1; public static final int DEFAULT_DW_CHECK_INTERVAL = 15; diff --git a/ranger-discovery-core/src/main/java/io/appform/ranger/discovery/core/ServiceDiscoveryConfiguration.java b/ranger-discovery-core/src/main/java/io/appform/ranger/discovery/core/ServiceDiscoveryConfiguration.java index 2abf66cd..dbd66084 100644 --- a/ranger-discovery-core/src/main/java/io/appform/ranger/discovery/core/ServiceDiscoveryConfiguration.java +++ b/ranger-discovery-core/src/main/java/io/appform/ranger/discovery/core/ServiceDiscoveryConfiguration.java @@ -38,6 +38,9 @@ @NoArgsConstructor public class ServiceDiscoveryConfiguration { + @NotNull + @NotEmpty + private String id = Constants.DEFAULT_ID; @NotNull @NotEmpty diff --git a/ranger-drove/src/main/java/io/appform/ranger/drove/config/DroveUpstreamConfig.java b/ranger-drove/src/main/java/io/appform/ranger/drove/config/DroveUpstreamConfig.java index 60882748..b866f635 100644 --- a/ranger-drove/src/main/java/io/appform/ranger/drove/config/DroveUpstreamConfig.java +++ b/ranger-drove/src/main/java/io/appform/ranger/drove/config/DroveUpstreamConfig.java @@ -16,6 +16,7 @@ package io.appform.ranger.drove.config; import io.dropwizard.util.Duration; +import javax.validation.constraints.NotBlank; import lombok.Builder; import lombok.Value; import lombok.extern.jackson.Jacksonized; @@ -40,6 +41,9 @@ public class DroveUpstreamConfig { public static final Duration DEFAULT_OPERATION_TIMEOUT = Duration.seconds(5); public static final Duration DEFAULT_EVENT_POLLING_INTERVAL = Duration.seconds(5); + @NotBlank + String id; + @Valid @NotEmpty List endpoints; diff --git a/ranger-http/src/main/java/io/appform/ranger/http/config/HttpClientConfig.java b/ranger-http/src/main/java/io/appform/ranger/http/config/HttpClientConfig.java index dbcac3a8..f7397342 100644 --- a/ranger-http/src/main/java/io/appform/ranger/http/config/HttpClientConfig.java +++ b/ranger-http/src/main/java/io/appform/ranger/http/config/HttpClientConfig.java @@ -28,6 +28,7 @@ @Jacksonized @AllArgsConstructor public class HttpClientConfig { + String id; String host; int port; boolean secure; diff --git a/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpApiCommunicator.java b/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpApiCommunicator.java index b5ee80f9..3dc180de 100644 --- a/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpApiCommunicator.java +++ b/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpApiCommunicator.java @@ -22,14 +22,6 @@ import io.appform.ranger.http.config.HttpClientConfig; import io.appform.ranger.http.model.ServiceDataSourceResponse; import io.appform.ranger.http.serde.HTTPResponseDataDeserializer; -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; -import lombok.val; -import okhttp3.HttpUrl; -import okhttp3.OkHttpClient; -import okhttp3.Request; -import okhttp3.Response; - import java.util.List; import java.util.Objects; import java.util.Set; @@ -38,6 +30,13 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Supplier; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import lombok.val; +import okhttp3.HttpUrl; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.Response; /** * Direct api based communication diff --git a/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpNodeDataSource.java b/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpNodeDataSource.java index d88f94a6..980dc54f 100644 --- a/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpNodeDataSource.java +++ b/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpNodeDataSource.java @@ -15,9 +15,11 @@ */ package io.appform.ranger.http.servicefinder; +import io.appform.functionmetrics.MonitoredFunction; import io.appform.ranger.core.model.NodeDataSource; import io.appform.ranger.core.model.Service; import io.appform.ranger.core.model.ServiceNode; +import io.appform.ranger.core.util.MetricRecorder; import io.appform.ranger.http.common.HttpNodeDataStoreConnector; import io.appform.ranger.http.config.HttpClientConfig; import io.appform.ranger.http.serde.HTTPResponseDataDeserializer; @@ -53,12 +55,15 @@ public HttpNodeDataSource( } @Override + @MonitoredFunction public Optional>> refresh(D deserializer) { return Optional.of(httpCommunicator.listNodes(service, deserializer)); } @Override public boolean isActive() { - return upstreamAvailable.get(); + var httpUpstreamAvailable = upstreamAvailable.get(); + MetricRecorder.recordHttpUpstreamAvailability(this.config.getId(), httpUpstreamAvailable); + return httpUpstreamAvailable; } } diff --git a/ranger-http/src/main/java/io/appform/ranger/http/servicefinderhub/HttpServiceDataSource.java b/ranger-http/src/main/java/io/appform/ranger/http/servicefinderhub/HttpServiceDataSource.java index a52cc214..c5bf0805 100644 --- a/ranger-http/src/main/java/io/appform/ranger/http/servicefinderhub/HttpServiceDataSource.java +++ b/ranger-http/src/main/java/io/appform/ranger/http/servicefinderhub/HttpServiceDataSource.java @@ -15,6 +15,7 @@ */ package io.appform.ranger.http.servicefinderhub; +import io.appform.functionmetrics.MonitoredFunction; import io.appform.ranger.core.finderhub.ServiceDataSource; import io.appform.ranger.core.model.Service; import io.appform.ranger.http.common.HttpNodeDataStoreConnector; @@ -33,6 +34,7 @@ public HttpServiceDataSource(HttpClientConfig config, HttpCommunicator httpCl } @Override + @MonitoredFunction public Collection services() { Objects.requireNonNull(config, "client config has not been set for node data"); return httpCommunicator.services(); diff --git a/ranger-http/src/main/java/io/appform/ranger/http/serviceprovider/HttpNodeDataSink.java b/ranger-http/src/main/java/io/appform/ranger/http/serviceprovider/HttpNodeDataSink.java index f8506aa3..96964be4 100644 --- a/ranger-http/src/main/java/io/appform/ranger/http/serviceprovider/HttpNodeDataSink.java +++ b/ranger-http/src/main/java/io/appform/ranger/http/serviceprovider/HttpNodeDataSink.java @@ -17,6 +17,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; +import io.appform.functionmetrics.MonitoredFunction; import io.appform.ranger.core.model.NodeDataSink; import io.appform.ranger.core.model.Service; import io.appform.ranger.core.model.ServiceNode; @@ -50,6 +51,7 @@ public HttpNodeDataSink(Service service, HttpClientConfig config, ObjectMapper m } @Override + @MonitoredFunction public void updateState(S serializer, ServiceNode serviceNode) { requireNonNull(config, "client config has not been set for node data"); requireNonNull(mapper, "mapper has not been set for node data"); diff --git a/ranger-hub-server-bundle-core/src/main/java/io/appform/ranger/hub/server/bundle/configuration/RangerZkUpstreamConfiguration.java b/ranger-hub-server-bundle-core/src/main/java/io/appform/ranger/hub/server/bundle/configuration/RangerZkUpstreamConfiguration.java index e1a3a650..a69374f5 100644 --- a/ranger-hub-server-bundle-core/src/main/java/io/appform/ranger/hub/server/bundle/configuration/RangerZkUpstreamConfiguration.java +++ b/ranger-hub-server-bundle-core/src/main/java/io/appform/ranger/hub/server/bundle/configuration/RangerZkUpstreamConfiguration.java @@ -18,6 +18,7 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import io.appform.ranger.core.model.HubConstants; import io.appform.ranger.hub.server.bundle.models.BackendType; +import javax.validation.constraints.NotBlank; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.ToString; @@ -34,6 +35,9 @@ @JsonIgnoreProperties(ignoreUnknown = true) public class RangerZkUpstreamConfiguration extends RangerUpstreamConfiguration { + @NotBlank + private String id; + @NotEmpty @Valid private List zookeepers; diff --git a/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/common/ZkNodeDataStoreConnector.java b/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/common/ZkNodeDataStoreConnector.java index 3f3f7a5e..4208fbd2 100644 --- a/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/common/ZkNodeDataStoreConnector.java +++ b/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/common/ZkNodeDataStoreConnector.java @@ -21,6 +21,7 @@ import io.appform.ranger.core.model.NodeDataStoreConnector; import io.appform.ranger.core.model.Service; import io.appform.ranger.core.util.Exceptions; +import io.appform.ranger.core.util.MetricRecorder; import io.appform.ranger.zookeeper.util.PathBuilder; import lombok.AccessLevel; import lombok.Getter; @@ -162,8 +163,10 @@ public void stop() { @Override public boolean isActive() { - return curatorFramework != null && curatorFramework.getZookeeperClient() != null + var zkConnectionActive = curatorFramework != null && curatorFramework.getZookeeperClient() != null && curatorFramework.getZookeeperClient().isConnected(); + MetricRecorder.recordZkConnection(zkConnectionActive); + return zkConnectionActive; } protected boolean isStarted() { diff --git a/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinder/ZkNodeDataSource.java b/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinder/ZkNodeDataSource.java index 28cce6a0..34e53857 100644 --- a/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinder/ZkNodeDataSource.java +++ b/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinder/ZkNodeDataSource.java @@ -15,9 +15,11 @@ */ package io.appform.ranger.zookeeper.servicefinder; +import io.appform.functionmetrics.MonitoredFunction; import io.appform.ranger.core.model.NodeDataSource; import io.appform.ranger.core.model.Service; import io.appform.ranger.core.model.ServiceNode; +import io.appform.ranger.core.util.MetricRecorder; import io.appform.ranger.zookeeper.common.ZkNodeDataStoreConnector; import io.appform.ranger.zookeeper.common.ZkStoreType; import io.appform.ranger.zookeeper.serde.ZkNodeDataDeserializer; @@ -48,6 +50,7 @@ public ZkNodeDataSource( } @Override + @MonitoredFunction public Optional>> refresh(D deserializer) { return checkForUpdateOnZookeeper(deserializer); } diff --git a/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinderhub/ZkServiceDataSource.java b/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinderhub/ZkServiceDataSource.java index 37d043a1..5437d2f9 100644 --- a/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinderhub/ZkServiceDataSource.java +++ b/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinderhub/ZkServiceDataSource.java @@ -15,6 +15,7 @@ */ package io.appform.ranger.zookeeper.servicefinderhub; +import io.appform.functionmetrics.MonitoredFunction; import io.appform.ranger.core.finderhub.ServiceDataSource; import io.appform.ranger.core.model.Service; import io.appform.ranger.zookeeper.util.PathBuilder; @@ -52,6 +53,7 @@ public ZkServiceDataSource(String namespace, @Override @SneakyThrows + @MonitoredFunction public Collection services() { val children = curatorFramework.getChildren() .forPath(PathBuilder.REGISTERED_SERVICES_PATH); diff --git a/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/serviceprovider/ZkNodeDataSink.java b/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/serviceprovider/ZkNodeDataSink.java index cac71fee..8b982271 100644 --- a/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/serviceprovider/ZkNodeDataSink.java +++ b/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/serviceprovider/ZkNodeDataSink.java @@ -15,6 +15,7 @@ */ package io.appform.ranger.zookeeper.serviceprovider; +import io.appform.functionmetrics.MonitoredFunction; import io.appform.ranger.core.model.NodeDataSink; import io.appform.ranger.core.model.Service; import io.appform.ranger.core.model.ServiceNode; @@ -43,6 +44,7 @@ public ZkNodeDataSink( } @Override + @MonitoredFunction public void updateState(S serializer, ServiceNode serviceNode) { if (isStopped()) { log.warn("Node has been stopped already for service: {}. No update will be possible.", From b22adac15373fb351a2d59abfbd4e91b9954b1ea Mon Sep 17 00:00:00 2001 From: Jitendra Dhawan Date: Wed, 8 Apr 2026 17:28:52 +0530 Subject: [PATCH 02/24] Refactor RangerZkUpstreamConfiguration to use a single zookeeper string instead of a list --- .../discovery/core/ServiceDiscoveryConfiguration.java | 4 ++-- .../ranger/hub/server/bundle/RangerHubServerBundle.java | 4 +--- .../configuration/RangerZkUpstreamConfiguration.java | 8 ++------ .../ranger/hub/server/bundle/RangerHubServerBundle.java | 4 +--- .../ranger/hub/server/bundle/RangerHubServerBundle.java | 4 +--- ranger-server/src/main/resources/local.yml | 2 +- 6 files changed, 8 insertions(+), 18 deletions(-) diff --git a/ranger-discovery-core/src/main/java/io/appform/ranger/discovery/core/ServiceDiscoveryConfiguration.java b/ranger-discovery-core/src/main/java/io/appform/ranger/discovery/core/ServiceDiscoveryConfiguration.java index dbd66084..eb503a05 100644 --- a/ranger-discovery-core/src/main/java/io/appform/ranger/discovery/core/ServiceDiscoveryConfiguration.java +++ b/ranger-discovery-core/src/main/java/io/appform/ranger/discovery/core/ServiceDiscoveryConfiguration.java @@ -17,6 +17,7 @@ package io.appform.ranger.discovery.core; import com.google.common.base.Strings; +import javax.validation.constraints.NotBlank; import lombok.Builder; import lombok.Data; import lombok.EqualsAndHashCode; @@ -38,8 +39,7 @@ @NoArgsConstructor public class ServiceDiscoveryConfiguration { - @NotNull - @NotEmpty + @NotBlank private String id = Constants.DEFAULT_ID; @NotNull diff --git a/ranger-hub-server-bundle-core/ranger-hub-server-bundle/src/main/java/io/appform/ranger/hub/server/bundle/RangerHubServerBundle.java b/ranger-hub-server-bundle-core/ranger-hub-server-bundle/src/main/java/io/appform/ranger/hub/server/bundle/RangerHubServerBundle.java index 473f7b53..50196088 100644 --- a/ranger-hub-server-bundle-core/ranger-hub-server-bundle/src/main/java/io/appform/ranger/hub/server/bundle/RangerHubServerBundle.java +++ b/ranger-hub-server-bundle-core/ranger-hub-server-bundle/src/main/java/io/appform/ranger/hub/server/bundle/RangerHubServerBundle.java @@ -197,9 +197,7 @@ public List>> vis @Override public List>> visit(RangerZkUpstreamConfiguration rangerZkConfiguration) { - return rangerZkConfiguration.getZookeepers().stream() - .map(zk -> addCuratorAndGetZkHubClient(zk, rangerZkConfiguration)) - .toList(); + return List.of(addCuratorAndGetZkHubClient(rangerZkConfiguration.getZookeeper(), rangerZkConfiguration)); } @Override diff --git a/ranger-hub-server-bundle-core/src/main/java/io/appform/ranger/hub/server/bundle/configuration/RangerZkUpstreamConfiguration.java b/ranger-hub-server-bundle-core/src/main/java/io/appform/ranger/hub/server/bundle/configuration/RangerZkUpstreamConfiguration.java index a69374f5..1071b828 100644 --- a/ranger-hub-server-bundle-core/src/main/java/io/appform/ranger/hub/server/bundle/configuration/RangerZkUpstreamConfiguration.java +++ b/ranger-hub-server-bundle-core/src/main/java/io/appform/ranger/hub/server/bundle/configuration/RangerZkUpstreamConfiguration.java @@ -23,11 +23,8 @@ import lombok.EqualsAndHashCode; import lombok.ToString; -import javax.validation.Valid; import javax.validation.constraints.Max; import javax.validation.constraints.Min; -import javax.validation.constraints.NotEmpty; -import java.util.List; @Data @EqualsAndHashCode(callSuper = true) @@ -38,9 +35,8 @@ public class RangerZkUpstreamConfiguration extends RangerUpstreamConfiguration { @NotBlank private String id; - @NotEmpty - @Valid - private List zookeepers; + @NotBlank + private String zookeeper; private boolean disablePushUpdaters; diff --git a/ranger-hub-server-bundle/src/main/java/io/appform/ranger/hub/server/bundle/RangerHubServerBundle.java b/ranger-hub-server-bundle/src/main/java/io/appform/ranger/hub/server/bundle/RangerHubServerBundle.java index 473f7b53..50196088 100644 --- a/ranger-hub-server-bundle/src/main/java/io/appform/ranger/hub/server/bundle/RangerHubServerBundle.java +++ b/ranger-hub-server-bundle/src/main/java/io/appform/ranger/hub/server/bundle/RangerHubServerBundle.java @@ -197,9 +197,7 @@ public List>> vis @Override public List>> visit(RangerZkUpstreamConfiguration rangerZkConfiguration) { - return rangerZkConfiguration.getZookeepers().stream() - .map(zk -> addCuratorAndGetZkHubClient(zk, rangerZkConfiguration)) - .toList(); + return List.of(addCuratorAndGetZkHubClient(rangerZkConfiguration.getZookeeper(), rangerZkConfiguration)); } @Override diff --git a/ranger-hub-server-dw5-bundle/src/main/java/io/appform/ranger/hub/server/bundle/RangerHubServerBundle.java b/ranger-hub-server-dw5-bundle/src/main/java/io/appform/ranger/hub/server/bundle/RangerHubServerBundle.java index 2d6affec..e6d240bf 100644 --- a/ranger-hub-server-dw5-bundle/src/main/java/io/appform/ranger/hub/server/bundle/RangerHubServerBundle.java +++ b/ranger-hub-server-dw5-bundle/src/main/java/io/appform/ranger/hub/server/bundle/RangerHubServerBundle.java @@ -197,9 +197,7 @@ public List>> vis @Override public List>> visit(RangerZkUpstreamConfiguration rangerZkConfiguration) { - return rangerZkConfiguration.getZookeepers().stream() - .map(zk -> addCuratorAndGetZkHubClient(zk, rangerZkConfiguration)) - .toList(); + return List.of(addCuratorAndGetZkHubClient(rangerZkConfiguration.getZookeeper(), rangerZkConfiguration)); } @Override diff --git a/ranger-server/src/main/resources/local.yml b/ranger-server/src/main/resources/local.yml index 8e0c6012..181ebbd3 100644 --- a/ranger-server/src/main/resources/local.yml +++ b/ranger-server/src/main/resources/local.yml @@ -12,7 +12,7 @@ ranger: nodeRefreshTimeMs: 5000 serviceRefreshTimeoutMs: 3000 hubStartTimeoutMs: 5000 - zookeepers: [ "localhost:2181" ] + zookeeper: "localhost:2181" disablePushUpdaters: true From 07cb447fcfd4ad810c39e524a3a457929e4ded19 Mon Sep 17 00:00:00 2001 From: Himneesh Goyal Date: Thu, 9 Apr 2026 14:41:20 +0530 Subject: [PATCH 03/24] [Fix] Collision Check concurrency handling --- .../discovery/bundle/id/CollisionChecker.java | 14 ++--- .../bundle/id/nonce/RandomNonceGenerator.java | 9 ++-- .../bundle/id/CollisionCheckerTest.java | 51 ++++++++++++++++--- 3 files changed, 56 insertions(+), 18 deletions(-) diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/CollisionChecker.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/CollisionChecker.java index baff2f1a..145aecc6 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/CollisionChecker.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/CollisionChecker.java @@ -44,22 +44,22 @@ public CollisionChecker(@NonNull TimeUnit resolution) { this.resolution = resolution; } - public boolean check(long timeInMillis, int location) { + public long checkAndGetTime(int location) { dataLock.lock(); try { - long resolvedTime = resolution.convert(timeInMillis, TimeUnit.MILLISECONDS); + long resolvedTime = resolution.convert( + System.currentTimeMillis(), TimeUnit.MILLISECONDS + ); if (currentInstant != resolvedTime) { currentInstant = resolvedTime; bitSet.clear(); } - if (bitSet.get(location)) { - return false; + return -1; } bitSet.set(location); - return true; - } - finally { + return resolvedTime; + } finally { dataLock.unlock(); } } diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/nonce/RandomNonceGenerator.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/nonce/RandomNonceGenerator.java index 5d2e8fa5..ac4f74b2 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/nonce/RandomNonceGenerator.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/nonce/RandomNonceGenerator.java @@ -41,12 +41,11 @@ public void retryEventListener(final ExecutionAttemptedEvent e private NonceInfo random(final CollisionChecker collisionChecker) { int randomGen; - long curTimeMs; + long resolvedTime; do { - curTimeMs = System.currentTimeMillis(); randomGen = secureRandom.nextInt(Constants.MAX_ID_PER_MS); - } while (!collisionChecker.check(curTimeMs, randomGen)); - return new NonceInfo(randomGen, curTimeMs); + resolvedTime = collisionChecker.checkAndGetTime(randomGen); + } while (resolvedTime == -1); + return new NonceInfo(randomGen, resolvedTime); } - } diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/CollisionCheckerTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/CollisionCheckerTest.java index 3f29c2e7..11eee465 100644 --- a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/CollisionCheckerTest.java +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/CollisionCheckerTest.java @@ -17,24 +17,63 @@ package io.appform.ranger.discovery.bundle.id; +import java.util.concurrent.TimeUnit; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; - import java.util.stream.IntStream; +import static org.awaitility.Awaitility.await; + + /** * Test on {@link CollisionChecker} */ class CollisionCheckerTest { @Test - void testCheck() { + void testCheckAndGetTimeAndCollision() { + CollisionChecker collisionChecker = new CollisionChecker(); + + long firstAttempt = collisionChecker.checkAndGetTime(1); + long secondAttempt = collisionChecker.checkAndGetTime(1); + + Assertions.assertTrue(firstAttempt > 0, "Should successfully return a valid timestamp"); + + // the second attempt MUST be a collision if still we are on same millisecond (-1) + if (System.currentTimeMillis() == firstAttempt) { + Assertions.assertEquals(-1L, secondAttempt, "Should return -1 on collision"); + } + } + + @Test + void testTimeRolloverClearsBitSet() { CollisionChecker collisionChecker = new CollisionChecker(); - Assertions.assertTrue(collisionChecker.check(100, 1)); - Assertions.assertFalse(collisionChecker.check(100, 1)); + + long firstTime = collisionChecker.checkAndGetTime(10); + Assertions.assertTrue(firstTime > 0); + + await().atMost(101, TimeUnit.MILLISECONDS) + .until(() -> System.currentTimeMillis() > firstTime); + + long secondTime = collisionChecker.checkAndGetTime(10); + + Assertions.assertTrue(secondTime > firstTime, "Time should have moved forward"); + Assertions.assertNotEquals(-1L, secondTime, "Should successfully acquire location 10 in the new millisecond"); + } + + @Test + void testHighVolumeCheckAndGetTimeInSingleMillisecond() { + CollisionChecker collisionChecker = new CollisionChecker(); + + // Generate 1000 IDs as fast as possible. + // We assert that any duplicate location requested in the exact same ms returns -1 IntStream.range(0, 1000).forEach(i -> { - Assertions.assertTrue(collisionChecker.check(101, i)); - Assertions.assertFalse(collisionChecker.check(101, i)); + long time1 = collisionChecker.checkAndGetTime(i); + long time2 = collisionChecker.checkAndGetTime(i); + + if (time1 == System.currentTimeMillis()) { + Assertions.assertEquals(-1L, time2); + } }); } } \ No newline at end of file From ac60486ddcec4cadfc9513c51e20a54079cef460 Mon Sep 17 00:00:00 2001 From: Himneesh Goyal Date: Thu, 9 Apr 2026 16:43:17 +0530 Subject: [PATCH 04/24] [Fix] Use of Nanoseconds in Id generator --- .../discovery/bundle/id/CollisionChecker.java | 16 +++++++++++++--- .../bundle/id/CollisionCheckerTest.java | 5 +---- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/CollisionChecker.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/CollisionChecker.java index 145aecc6..ebebe828 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/CollisionChecker.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/id/CollisionChecker.java @@ -31,8 +31,18 @@ public class CollisionChecker { private final BitSet bitSet = new BitSet(1000); private long currentInstant = 0; - private final Lock dataLock = new ReentrantLock(); + private static final long NANO_TIME_MS; + private static final long CURRENT_TIME_MS; + private static final long NANO_TO_EPOCH_OFFSET_MS; + + static { + NANO_TIME_MS = TimeUnit.NANOSECONDS.toMillis(System.nanoTime()); + CURRENT_TIME_MS = System.currentTimeMillis(); + NANO_TO_EPOCH_OFFSET_MS = NANO_TIME_MS - CURRENT_TIME_MS; + log.info("CollisionChecker init: NANO_TIME_MS={}, CURRENT_TIME_MS={}, NANO_TO_EPOCH_OFFSET_MS={}", + NANO_TIME_MS, CURRENT_TIME_MS, NANO_TO_EPOCH_OFFSET_MS); + } private final TimeUnit resolution; @@ -48,8 +58,8 @@ public long checkAndGetTime(int location) { dataLock.lock(); try { long resolvedTime = resolution.convert( - System.currentTimeMillis(), TimeUnit.MILLISECONDS - ); + System.nanoTime(), TimeUnit.NANOSECONDS + ) - resolution.convert(NANO_TO_EPOCH_OFFSET_MS, TimeUnit.MILLISECONDS); if (currentInstant != resolvedTime) { currentInstant = resolvedTime; bitSet.clear(); diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/CollisionCheckerTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/CollisionCheckerTest.java index 11eee465..78ae6b4f 100644 --- a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/CollisionCheckerTest.java +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/id/CollisionCheckerTest.java @@ -70,10 +70,7 @@ void testHighVolumeCheckAndGetTimeInSingleMillisecond() { IntStream.range(0, 1000).forEach(i -> { long time1 = collisionChecker.checkAndGetTime(i); long time2 = collisionChecker.checkAndGetTime(i); - - if (time1 == System.currentTimeMillis()) { - Assertions.assertEquals(-1L, time2); - } + Assertions.assertNotEquals(time2, time1); }); } } \ No newline at end of file From a59e0e1409605e59fa9c7e5e956464cbe79c74df Mon Sep 17 00:00:00 2001 From: Himneesh Goyal Date: Wed, 15 Apr 2026 11:10:42 +0530 Subject: [PATCH 05/24] Version Bump --- pom.xml | 2 +- ranger-bom/pom.xml | 2 +- ranger-client/pom.xml | 2 +- ranger-core/pom.xml | 2 +- ranger-discovery-bundle/pom.xml | 2 +- ranger-drove-client/pom.xml | 2 +- ranger-drove/pom.xml | 2 +- ranger-http-client/pom.xml | 2 +- ranger-http-model/pom.xml | 2 +- ranger-http/pom.xml | 2 +- ranger-hub-server-bundle/pom.xml | 2 +- ranger-server-bundle/pom.xml | 2 +- ranger-server-common/pom.xml | 2 +- ranger-server/pom.xml | 2 +- ranger-zk-client/pom.xml | 2 +- ranger-zookeeper/pom.xml | 2 +- 16 files changed, 16 insertions(+), 16 deletions(-) diff --git a/pom.xml b/pom.xml index c4f0e3c3..2a7b2995 100644 --- a/pom.xml +++ b/pom.xml @@ -23,7 +23,7 @@ io.appform.ranger ranger pom - 1.1.3 + 1.1.4 Ranger Service Discovery https://github.com/appform-io/ranger Service Discovery for Java diff --git a/ranger-bom/pom.xml b/ranger-bom/pom.xml index f4272886..2331b93d 100644 --- a/ranger-bom/pom.xml +++ b/ranger-bom/pom.xml @@ -6,7 +6,7 @@ io.appform.ranger ranger - 1.1.3 + 1.1.4 ranger-bom diff --git a/ranger-client/pom.xml b/ranger-client/pom.xml index 80b02038..3851e5ed 100644 --- a/ranger-client/pom.xml +++ b/ranger-client/pom.xml @@ -21,7 +21,7 @@ ranger io.appform.ranger - 1.1.3 + 1.1.4 4.0.0 diff --git a/ranger-core/pom.xml b/ranger-core/pom.xml index e6991f23..19abf890 100644 --- a/ranger-core/pom.xml +++ b/ranger-core/pom.xml @@ -21,7 +21,7 @@ ranger io.appform.ranger - 1.1.3 + 1.1.4 4.0.0 diff --git a/ranger-discovery-bundle/pom.xml b/ranger-discovery-bundle/pom.xml index 2c087f43..026c4da2 100644 --- a/ranger-discovery-bundle/pom.xml +++ b/ranger-discovery-bundle/pom.xml @@ -21,7 +21,7 @@ ranger io.appform.ranger - 1.1.3 + 1.1.4 4.0.0 diff --git a/ranger-drove-client/pom.xml b/ranger-drove-client/pom.xml index 9328ec5f..9c4d5aca 100644 --- a/ranger-drove-client/pom.xml +++ b/ranger-drove-client/pom.xml @@ -21,7 +21,7 @@ ranger io.appform.ranger - 1.1.3 + 1.1.4 4.0.0 diff --git a/ranger-drove/pom.xml b/ranger-drove/pom.xml index 96c8ddcc..211f05be 100644 --- a/ranger-drove/pom.xml +++ b/ranger-drove/pom.xml @@ -21,7 +21,7 @@ ranger io.appform.ranger - 1.1.3 + 1.1.4 1.30 diff --git a/ranger-http-client/pom.xml b/ranger-http-client/pom.xml index b8649787..700d0015 100644 --- a/ranger-http-client/pom.xml +++ b/ranger-http-client/pom.xml @@ -21,7 +21,7 @@ ranger io.appform.ranger - 1.1.3 + 1.1.4 4.0.0 diff --git a/ranger-http-model/pom.xml b/ranger-http-model/pom.xml index 53d6b262..6647b193 100644 --- a/ranger-http-model/pom.xml +++ b/ranger-http-model/pom.xml @@ -21,7 +21,7 @@ ranger io.appform.ranger - 1.1.3 + 1.1.4 4.0.0 diff --git a/ranger-http/pom.xml b/ranger-http/pom.xml index 6c65d0d2..1de734e7 100644 --- a/ranger-http/pom.xml +++ b/ranger-http/pom.xml @@ -21,7 +21,7 @@ ranger io.appform.ranger - 1.1.3 + 1.1.4 4.0.0 diff --git a/ranger-hub-server-bundle/pom.xml b/ranger-hub-server-bundle/pom.xml index e27365e3..c2c4c257 100644 --- a/ranger-hub-server-bundle/pom.xml +++ b/ranger-hub-server-bundle/pom.xml @@ -21,7 +21,7 @@ ranger io.appform.ranger - 1.1.3 + 1.1.4 diff --git a/ranger-server-bundle/pom.xml b/ranger-server-bundle/pom.xml index 504ce8ef..2b43b112 100644 --- a/ranger-server-bundle/pom.xml +++ b/ranger-server-bundle/pom.xml @@ -21,7 +21,7 @@ ranger io.appform.ranger - 1.1.3 + 1.1.4 4.0.0 diff --git a/ranger-server-common/pom.xml b/ranger-server-common/pom.xml index a81559e3..e9da029c 100644 --- a/ranger-server-common/pom.xml +++ b/ranger-server-common/pom.xml @@ -21,7 +21,7 @@ ranger io.appform.ranger - 1.1.3 + 1.1.4 4.0.0 diff --git a/ranger-server/pom.xml b/ranger-server/pom.xml index bb123575..9ae309a0 100644 --- a/ranger-server/pom.xml +++ b/ranger-server/pom.xml @@ -22,7 +22,7 @@ io.appform.ranger ranger - 1.1.3 + 1.1.4 ranger-server diff --git a/ranger-zk-client/pom.xml b/ranger-zk-client/pom.xml index 2005ddbe..723a2467 100644 --- a/ranger-zk-client/pom.xml +++ b/ranger-zk-client/pom.xml @@ -21,7 +21,7 @@ ranger io.appform.ranger - 1.1.3 + 1.1.4 4.0.0 diff --git a/ranger-zookeeper/pom.xml b/ranger-zookeeper/pom.xml index 0ae4ffba..26297c3f 100644 --- a/ranger-zookeeper/pom.xml +++ b/ranger-zookeeper/pom.xml @@ -21,7 +21,7 @@ ranger io.appform.ranger - 1.1.3 + 1.1.4 4.0.0 From 4945fbc67d9cb53743e41d43c58857e6a8bbc6db Mon Sep 17 00:00:00 2001 From: Santanu Sinha Date: Wed, 8 Apr 2026 11:55:25 +0530 Subject: [PATCH 06/24] Removed legacy nexus plugin --- pom.xml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/pom.xml b/pom.xml index 2a7b2995..aef6adfc 100644 --- a/pom.xml +++ b/pom.xml @@ -115,7 +115,6 @@ 0.8.13 3.3.1 3.12.0 - 1.7.0 3.16.2 2.9.0 @@ -352,18 +351,6 @@ - - org.sonatype.plugins - nexus-staging-maven-plugin - ${nexus-staging-maven-plugin.version} - true - - ossrh - https://ossrh-staging-api.central.sonatype.com - true - 10 - - org.apache.maven.plugins maven-enforcer-plugin From e990fc755c2c6787241fc292a2c6b56d7e1ebcb0 Mon Sep 17 00:00:00 2001 From: Jitendra Dhawan Date: Sun, 31 May 2026 15:23:04 +0530 Subject: [PATCH 07/24] Add metrics instrumentation to service data sources and update related classes --- .../client/AbstractRangerHubClient.java | 6 +- .../ranger/client/stubs/RangerTestHub.java | 2 +- .../TestSimpleUnshardedServiceFinder.java | 17 +- .../core/finder/BaseServiceFinderBuilder.java | 11 +- .../ServiceRegistryUpdater.java | 12 +- .../core/healthcheck/HealthChecker.java | 7 +- .../ranger/core/model/DataStoreType.java | 22 +++ .../ranger/core/model/NodeDataSource.java | 4 + .../BaseServiceProviderBuilder.java | 13 +- .../ranger/core/util/MetricRecorder.java | 149 +++++++++++++++++- .../core/finderhub/ServiceFinderHubTest.java | 12 +- .../serviceprovider/ServiceProviderTest.java | 2 +- .../ranger/discovery/bundle/Constants.java | 2 - .../bundle/ServiceDiscoveryBundle.java | 19 ++- ...viceDiscoveryBundleCustomHostPortTest.java | 1 + .../ServiceDiscoveryBundleDwMonitorTest.java | 1 + ...DiscoveryBundleDwStalenessMonitorTest.java | 1 + ...scoveryBundleHierarchicalSelectorTest.java | 1 + ...rviceDiscoveryBundleLocalHostPortTest.java | 2 + .../ServiceDiscoveryBundleRotationTest.java | 1 + .../bundle/ServiceDiscoveryBundleTest.java | 1 + .../DefaultNodeInfoResolverTest.java | 1 + .../ranger/discovery/core/Constants.java | 2 +- .../core/ServiceDiscoveryConfiguration.java | 9 +- .../bundle/ServiceDiscoveryBundle.java | 11 +- ...viceDiscoveryBundleCustomHostPortTest.java | 2 + .../ServiceDiscoveryBundleDwMonitorTest.java | 2 + ...DiscoveryBundleDwStalenessMonitorTest.java | 2 + ...scoveryBundleHierarchicalSelectorTest.java | 2 + ...rviceDiscoveryBundleLocalHostPortTest.java | 7 + .../ServiceDiscoveryBundleRotationTest.java | 2 + .../bundle/ServiceDiscoveryBundleTest.java | 2 + .../DefaultNodeInfoResolverTest.java | 2 + .../drove/AbstractRangerDroveHubClient.java | 4 +- .../client/drove/SimpleRangerDroveClient.java | 1 + .../drove/common/DroveApiCommunicator.java | 66 ++++++-- .../common/DroveCachingCommunicator.java | 13 +- .../common/DroveNodeDataStoreConnector.java | 9 +- .../drove/common/DroveOkHttpTransport.java | 10 +- .../servicefinder/DroveNodeDataSource.java | 24 ++- .../DroveShardedServiceFinderBuilder.java | 10 +- .../DroveUnshardedServiceFinderBuilider.java | 7 +- .../DroveServiceDataSource.java | 25 ++- .../DroveShardedServiceFinderFactory.java | 1 + .../DroveUnshardedServiceFinderFactory.java | 1 + .../ranger/drove/utils/RangerDroveUtils.java | 4 +- .../DroveNodeDataSourceTest.java | 10 +- .../DroveServiceDataSourceTest.java | 2 +- .../http/AbstractRangerHttpHubClient.java | 4 +- .../client/http/SimpleRangerHttpClient.java | 1 + ranger-http/pom.xml | 9 ++ .../common/HttpNodeDataStoreConnector.java | 3 + .../ranger/http/config/HttpClientConfig.java | 4 + .../servicefinder/HttpApiCommunicator.java | 40 ++++- .../servicefinder/HttpNodeDataSource.java | 15 +- .../HttpShardedServiceFinderBuilder.java | 4 +- .../HttpUnshardedServiceFinderBuilider.java | 4 +- .../HttpServiceDataSource.java | 21 ++- .../HttpShardedServiceFinderFactory.java | 1 + .../HttpUnshardedServiceFinderFactory.java | 1 + .../serviceprovider/HttpNodeDataSink.java | 45 +++++- .../HttpShardedServiceProviderBuilder.java | 4 +- .../HttpServiceDataSourceTest.java | 4 +- .../server/bundle/RangerHubServerBundle.java | 3 + .../server/bundle/RangerServerBundle.java | 14 +- .../server/bundle/RangerServerBundle.java | 13 +- .../java/io/appform/ranger/server/App.java | 2 + .../client/zk/AbstractRangerZKHubClient.java | 4 +- .../client/zk/SimpleRangerZKClient.java | 3 + .../client/zk/ShardedZKRangerClientTest.java | 1 + .../common/ZkNodeDataStoreConnector.java | 7 +- .../servicefinder/ZkNodeDataSource.java | 43 ++++- .../ZkSimpleShardedServiceFinderBuilder.java | 6 +- ...ZkSimpleUnshardedServiceFinderBuilder.java | 4 +- .../servicefinderhub/ZkServiceDataSource.java | 35 +++- .../serviceprovider/ZkNodeDataSink.java | 29 +++- .../ZkServiceProviderBuilder.java | 4 +- .../zookeeper/servicehub/ServiceHubTest.java | 2 +- 78 files changed, 701 insertions(+), 146 deletions(-) create mode 100644 ranger-core/src/main/java/io/appform/ranger/core/model/DataStoreType.java diff --git a/ranger-client/src/main/java/io/appform/ranger/client/AbstractRangerHubClient.java b/ranger-client/src/main/java/io/appform/ranger/client/AbstractRangerHubClient.java index 8eac12a6..0028508a 100644 --- a/ranger-client/src/main/java/io/appform/ranger/client/AbstractRangerHubClient.java +++ b/ranger-client/src/main/java/io/appform/ranger/client/AbstractRangerHubClient.java @@ -37,6 +37,7 @@ @SuperBuilder public abstract class AbstractRangerHubClient, D extends Deserializer> implements RangerHubClient { + private final String metricId; private final String namespace; private final ObjectMapper mapper; private final D deserializer; @@ -60,6 +61,7 @@ public abstract class AbstractRangerHubClient, D @Override public void start() { + requireNonNull(metricId, "metricId can't be null"); requireNonNull(mapper, "Mapper can't be null"); requireNonNull(namespace, "namespace can't be null"); requireNonNull(deserializer, "deserializer can't be null"); @@ -88,7 +90,7 @@ public void start() { this.excludedServices = Objects.requireNonNullElseGet(this.excludedServices, Set::of); if(null == this.serviceDataSource){ - this.serviceDataSource = getDefaultDataSource(); + this.serviceDataSource = getDefaultDataSource(metricId); } this.hub = buildHub(); @@ -193,7 +195,7 @@ public CompletableFuture addService(Service service) { } - protected abstract ServiceDataSource getDefaultDataSource(); + protected abstract ServiceDataSource getDefaultDataSource(String metricId); protected abstract ServiceFinderFactory getFinderFactory(); diff --git a/ranger-client/src/test/java/io/appform/ranger/client/stubs/RangerTestHub.java b/ranger-client/src/test/java/io/appform/ranger/client/stubs/RangerTestHub.java index 34b4b4c8..25970cbd 100644 --- a/ranger-client/src/test/java/io/appform/ranger/client/stubs/RangerTestHub.java +++ b/ranger-client/src/test/java/io/appform/ranger/client/stubs/RangerTestHub.java @@ -52,7 +52,7 @@ protected void postBuild(ServiceFinderHub build() { } @Override - protected NodeDataSource> dataSource(Service service) { + protected NodeDataSource> dataSource(String metricId, Service service) { return new TestDataSource(); } static class TestDataSource implements NodeDataSource>{ + @Override + public String getMetricId() { + return "testDataSource"; + } + + @Override + public DataStoreType getDataStoreType() { + return DataStoreType.HTTP; + } + @Override public Optional>> refresh(Deserializer deserializer) { return Optional.of(Collections.singletonList( diff --git a/ranger-core/src/main/java/io/appform/ranger/core/finder/BaseServiceFinderBuilder.java b/ranger-core/src/main/java/io/appform/ranger/core/finder/BaseServiceFinderBuilder.java index 83cffe51..60d1ecfc 100644 --- a/ranger-core/src/main/java/io/appform/ranger/core/finder/BaseServiceFinderBuilder.java +++ b/ranger-core/src/main/java/io/appform/ranger/core/finder/BaseServiceFinderBuilder.java @@ -42,6 +42,7 @@ public abstract class BaseServiceFinderBuilder B extends BaseServiceFinderBuilder, D extends Deserializer> { + protected String metricId; protected String namespace; protected String serviceName; protected int nodeRefreshIntervalMs; @@ -53,6 +54,11 @@ public abstract class BaseServiceFinderBuilder protected final List> startSignalHandlers = new ArrayList<>(); protected final List> stopSignalHandlers = new ArrayList<>(); + public B withMetricId(final String metricId) { + this.metricId = metricId; + return (B)this; + } + public B withNamespace(final String namespace) { this.namespace = namespace; return (B)this; @@ -131,6 +137,7 @@ public B withStopSignalHandlers(List> stopSignalHandlers) { public abstract F build(); protected F buildFinder() { + requireNonNull(metricId); requireNonNull(namespace); requireNonNull(serviceName); requireNonNull(deserializer); @@ -144,7 +151,7 @@ protected F buildFinder() { val finder = buildFinder(service, shardSelector, nodeSelector); val registry = finder.getServiceRegistry(); val signalGenerators = new ArrayList>(); - val nodeDataSource = dataSource(service); + val nodeDataSource = dataSource(metricId, service); signalGenerators.add(new ScheduledRegistryUpdateSignal<>(service, nodeRefreshIntervalMs)); additionalRefreshSignals.addAll(implementationSpecificRefreshSignals(service, nodeDataSource)); @@ -173,7 +180,7 @@ protected List> implementationSpecificRefreshSignals(Service service, return Collections.emptyList(); } - protected abstract NodeDataSource dataSource(Service service); + protected abstract NodeDataSource dataSource(String metricId, Service service); protected abstract F buildFinder( Service service, diff --git a/ranger-core/src/main/java/io/appform/ranger/core/finder/serviceregistry/ServiceRegistryUpdater.java b/ranger-core/src/main/java/io/appform/ranger/core/finder/serviceregistry/ServiceRegistryUpdater.java index cf7963ee..b2f25ebc 100644 --- a/ranger-core/src/main/java/io/appform/ranger/core/finder/serviceregistry/ServiceRegistryUpdater.java +++ b/ranger-core/src/main/java/io/appform/ranger/core/finder/serviceregistry/ServiceRegistryUpdater.java @@ -27,6 +27,7 @@ import io.appform.ranger.core.signals.Signal; import io.appform.ranger.core.util.Exceptions; import io.appform.ranger.core.util.FinderUtils; +import io.appform.ranger.core.util.MetricRecorder; import java.util.concurrent.atomic.AtomicBoolean; import lombok.extern.slf4j.Slf4j; import lombok.val; @@ -139,8 +140,8 @@ private void updateRegistry() throws InterruptedException { log.debug("Checking for updates on data source for service: {}", serviceRegistry.getService().getServiceName()); var callFailed = false; - if (nodeDataSource.isActive()) { //Source should implement circuit breaker to fail fast and reopen after some - // time + if (nodeDataSource.isActive()) { //Source should implement circuit breaker to fail fast and reopen after some time + val stopwatch = Stopwatch.createStarted(); try { val nodeList = nodeDataSource.refresh(deserializer).orElse(null); if (null != nodeList) { @@ -150,6 +151,8 @@ private void updateRegistry() throws InterruptedException { //Remove all stale nodes before updating. This is done centrally to ensure some data sources //don't skip this check. Some control is still provided so that they can overload. serviceRegistry.updateNodes(FinderUtils.filterValidNodes(serviceRegistry.getService(), nodeList, livenessCheckMaxAge)); + MetricRecorder.recordNodeDataRefreshSuccess(nodeDataSource.getDataStoreType(), nodeDataSource.getMetricId(), + stopwatch.elapsed(TimeUnit.MILLISECONDS)); } else { log.warn("Empty list returned from node data source. We are in a weird state. Keeping old list for {}", @@ -161,6 +164,10 @@ private void updateRegistry() throws InterruptedException { e.getClass().getSimpleName(), e.getMessage()); callFailed = true; + MetricRecorder.recordNodeDataRefreshFailure(nodeDataSource.getDataStoreType(), nodeDataSource.getMetricId(), + stopwatch.elapsed(TimeUnit.MILLISECONDS)); + } finally { + stopwatch.stop(); } } if (!nodeDataSource.isActive() || callFailed) { @@ -168,6 +175,7 @@ private void updateRegistry() throws InterruptedException { log.warn("Node data source seems to be down. Keeping old list for {}." + " Will update timestamp to keep stale date relevant.", serviceRegistry.getService().getServiceName()); + MetricRecorder.recordStaleDataRetained(nodeDataSource.getDataStoreType(), nodeDataSource.getMetricId()); serviceRegistry.updateNodes(serviceRegistry.nodeList() .stream() .filter(node -> HealthcheckStatus.healthy == node.getHealthcheckStatus()) diff --git a/ranger-core/src/main/java/io/appform/ranger/core/healthcheck/HealthChecker.java b/ranger-core/src/main/java/io/appform/ranger/core/healthcheck/HealthChecker.java index be9a1f79..9f1c37f8 100644 --- a/ranger-core/src/main/java/io/appform/ranger/core/healthcheck/HealthChecker.java +++ b/ranger-core/src/main/java/io/appform/ranger/core/healthcheck/HealthChecker.java @@ -15,6 +15,7 @@ */ package io.appform.ranger.core.healthcheck; +import io.appform.ranger.core.util.MetricRecorder; import lombok.extern.slf4j.Slf4j; import lombok.val; @@ -27,12 +28,14 @@ @Slf4j public class HealthChecker implements Supplier { + private final String metricId; private final List healthChecks; private final int staleUpdateThreshold; private HealthcheckStatus lastHealthcheckStatus; private long lastUpdatedTime; - public HealthChecker(List healthChecks, int staleUpdateThreshold) { + public HealthChecker(String metricId, List healthChecks, int staleUpdateThreshold) { + this.metricId = metricId; this.healthChecks = healthChecks; this.staleUpdateThreshold = staleUpdateThreshold; } @@ -57,11 +60,13 @@ private boolean refreshHealth() { catch (Exception e) { log.error("Error running healthcheck. Setting node to unhealthy", e); healthcheckStatus = HealthcheckStatus.unhealthy; + MetricRecorder.recordHealthcheckFailure(metricId); } if (HealthcheckStatus.unhealthy == healthcheckStatus) { break; } } + MetricRecorder.recordHealthcheckStatus(metricId, HealthcheckStatus.healthy == healthcheckStatus); //Trigger update only if state change has happened //Conditions on which update will be triggered //1. First time diff --git a/ranger-core/src/main/java/io/appform/ranger/core/model/DataStoreType.java b/ranger-core/src/main/java/io/appform/ranger/core/model/DataStoreType.java new file mode 100644 index 00000000..4ff1b84f --- /dev/null +++ b/ranger-core/src/main/java/io/appform/ranger/core/model/DataStoreType.java @@ -0,0 +1,22 @@ +/* + * Copyright 2024 Authors, Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.appform.ranger.core.model; + +public enum DataStoreType { + ZK, + HTTP, + DROVE, +} diff --git a/ranger-core/src/main/java/io/appform/ranger/core/model/NodeDataSource.java b/ranger-core/src/main/java/io/appform/ranger/core/model/NodeDataSource.java index 86276b2e..14d5e93d 100644 --- a/ranger-core/src/main/java/io/appform/ranger/core/model/NodeDataSource.java +++ b/ranger-core/src/main/java/io/appform/ranger/core/model/NodeDataSource.java @@ -26,6 +26,10 @@ @SuppressWarnings("unused") public interface NodeDataSource> extends NodeDataStoreConnector { + String getMetricId(); + + DataStoreType getDataStoreType(); + Optional>> refresh(D deserializer) throws CommunicationException; default long healthcheckZombieCheckThresholdTime(Service service) { diff --git a/ranger-core/src/main/java/io/appform/ranger/core/serviceprovider/BaseServiceProviderBuilder.java b/ranger-core/src/main/java/io/appform/ranger/core/serviceprovider/BaseServiceProviderBuilder.java index 0ad17db8..5c4fd1f9 100644 --- a/ranger-core/src/main/java/io/appform/ranger/core/serviceprovider/BaseServiceProviderBuilder.java +++ b/ranger-core/src/main/java/io/appform/ranger/core/serviceprovider/BaseServiceProviderBuilder.java @@ -48,6 +48,7 @@ @SuppressWarnings({"unchecked", "unused", "UnusedReturnValue"}) public abstract class BaseServiceProviderBuilder, S extends Serializer> { + protected String metricId; protected String namespace; protected String serviceName; protected S serializer; @@ -66,6 +67,11 @@ public abstract class BaseServiceProviderBuilder> isolatedMonitors = new ArrayList<>(); + public B withMetricId(final String metricId) { + this.metricId = metricId; + return (B)this; + } + public BaseServiceProviderBuilder withNamespace(final String namespace) { this.namespace = namespace; return this; @@ -166,6 +172,7 @@ public B withAdditionalRefreshSignals(List> additional } protected final ServiceProvider buildProvider() { + requireNonNull(metricId); requireNonNull(namespace); requireNonNull(serviceName); requireNonNull(serializer); @@ -190,12 +197,12 @@ protected final ServiceProvider buildProvider() { healthchecks.add(serviceHealthAggregator); val service = Service.builder().namespace(namespace).serviceName(serviceName).build(); - val usableNodeDataSource = dataSink(service); + val usableNodeDataSource = dataSink(metricId, service); val healthcheckUpdateSignalGenerator = new ScheduledSignal<>( service, - new HealthChecker(healthchecks, staleUpdateThresholdMs), + new HealthChecker(metricId, healthchecks, staleUpdateThresholdMs), Collections.emptyList(), healthUpdateIntervalMs ); @@ -236,5 +243,5 @@ protected final ServiceProvider buildProvider() { public abstract ServiceProvider build(); - protected abstract NodeDataSink dataSink(final Service service); + protected abstract NodeDataSink dataSink(String metricId, final Service service); } diff --git a/ranger-core/src/main/java/io/appform/ranger/core/util/MetricRecorder.java b/ranger-core/src/main/java/io/appform/ranger/core/util/MetricRecorder.java index 060fd2f6..a0e61a14 100644 --- a/ranger-core/src/main/java/io/appform/ranger/core/util/MetricRecorder.java +++ b/ranger-core/src/main/java/io/appform/ranger/core/util/MetricRecorder.java @@ -1,12 +1,39 @@ package io.appform.ranger.core.util; import com.codahale.metrics.MetricRegistry; +import io.appform.ranger.core.model.DataStoreType; import lombok.experimental.UtilityClass; +import static java.util.concurrent.TimeUnit.*; + @UtilityClass public class MetricRecorder { private static final String PACKAGE_PREFIX = "io.appform.ranger"; + private static final String ACTIVE = "active"; + private static final String INACTIVE = "inactive"; + private static final String NULL_OR_EMPTY_RESPONSE = "nullOrEmptyResponse"; + private static final String DATA_STORE_TYPE = "dataStoreType"; + private static final String DATA_SOURCE = "dataSource"; + private static final String SERVICE_NAME = "serviceName"; + private static final String ZOMBIE_NODES = "zombieNodes"; + private static final String HTTP_CALL = "httpCall"; + private static final String UNKNOWN_FAILURE = "unknownFailure"; + private static final String RESPONSE_PARSE_FAILURE = "responseParseFailure"; + private static final String NODE_DATA_REFRESH = "nodeDataRefresh"; + private static final String HEALTHY = "healthy"; + private static final String UNHEALTHY = "unhealthy"; + private static final String UPDATE = "update"; + + public static final String SERVICES_LIST = "services"; + public static final String LIST_NODES = "listNodes"; + public static final String SUCCESS = "success"; + public static final String FAILURE = "failure"; + public static final String HEALTHCHECK = "healthcheck"; + public static final String NODE_DATA_SINK = "nodeDataSink"; + public static final String REGISTER_SERVICE = "registerService"; + public static final String SERIALIZAION = "serialization"; + public static final String DESERIALIZATION = "deserialization"; private static MetricRegistry metricRegistry = new MetricRegistry(); @@ -15,15 +42,125 @@ public static void initialize(MetricRegistry registry) { } public static void recordZombieNodeFound(String serviceName) { - metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, "zombieNodes")).mark(); - metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, "zombieNodes", "service", serviceName)).mark(); + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, ZOMBIE_NODES)).mark(); + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, ZOMBIE_NODES, SERVICE_NAME, serviceName)).mark(); + } + + public static void recordNoteDataSourceStatus(DataStoreType dataStoreType, String metricId, boolean active) { + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_SOURCE, dataStoreType.name(), metricId, + active ? ACTIVE : INACTIVE)).mark(); + } + + public static void recordNodeDataRefreshSuccess(DataStoreType dataStoreType, String metricId, long elapsed) { + metricRegistry.timer(MetricRegistry.name(PACKAGE_PREFIX, DATA_SOURCE, dataStoreType.name(), metricId, NODE_DATA_REFRESH, SUCCESS)) + .update(elapsed, MILLISECONDS); + } + + public static void recordNodeDataRefreshFailure(DataStoreType dataStoreType, String metricId, long elapsed) { + metricRegistry.timer(MetricRegistry.name(PACKAGE_PREFIX, DATA_SOURCE, dataStoreType.name(), metricId, NODE_DATA_REFRESH, FAILURE)) + .update(elapsed, MILLISECONDS); + } + + public static void recordStaleDataRetained(DataStoreType dataStoreType, String metricId) { + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_SOURCE, dataStoreType.name(), metricId, "staleDataRetained")).mark(); + } + + public static void recordNodeDataSinkUpdateStatus(DataStoreType dataStoreType, String metricId, String status) { + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_SOURCE, dataStoreType.name(), DATA_SOURCE, metricId, NODE_DATA_SINK, UPDATE, status)).mark(); + } + + public static void recordHealthcheckFailure(String metricId) { + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, HEALTHCHECK, metricId, FAILURE)).mark(); + } + + public static void recordHealthcheckStatus(String metricId, boolean healthy) { + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, HEALTHCHECK, metricId, healthy ? HEALTHY : UNHEALTHY)).mark(); + } + + public static void recordServicesFetchStatus(DataStoreType dataStoreType, String metricId, String success) { + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), + DATA_SOURCE, metricId, SERVICES_LIST, "fetch", success)).mark(); + } + + public static void recordRemoteCallStatusCode(DataStoreType dataStoreType, String metricId, String remoteCall, int statusCode) { + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), + DATA_SOURCE, metricId, HTTP_CALL, remoteCall, "responseStatus", Integer.toString(statusCode))).mark(); } - public static void recordZkConnection(boolean zkConnectionActive) { - metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX,"zkConnection", zkConnectionActive ? "active" : "inactive")).mark(); + public static void recordCacheUpdateOnDroveEvent(DataStoreType dataStoreType, String metricId, String eventName, String serviceName) { + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), + DATA_SOURCE, metricId, "cacheUpdateOnDroveEvent", eventName, SERVICE_NAME, serviceName)).mark(); } - public static void recordHttpUpstreamAvailability(String clientId, boolean httpUpstreamAvailable) { - metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX,"httpUpstream", clientId, httpUpstreamAvailable ? "active" : "inactive")).mark(); + public static void recordServicesParseFailure(DataStoreType dataStoreType, String metricId) { + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), + DATA_SOURCE, metricId, HTTP_CALL, SERVICES_LIST, RESPONSE_PARSE_FAILURE)).mark(); } + + public static void recordListNodesParseFailure(DataStoreType dataStoreType, String metricId, String serviceName) { + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), + DATA_SOURCE, metricId, HTTP_CALL, LIST_NODES, SERVICE_NAME, serviceName, RESPONSE_PARSE_FAILURE)).mark(); + } + + + public static void recordZookeeperReadUnknownFailure(DataStoreType dataStoreType, String metricId, + String operation, String exceptionName) { + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), + DATA_SOURCE, metricId, "zkRead", operation, UNKNOWN_FAILURE)).mark(); + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), + DATA_SOURCE, metricId, "zkRead", operation, UNKNOWN_FAILURE, exceptionName)).mark(); + } + + public static void recordRemoteCallUnknownFailure(DataStoreType dataStoreType, String metricId, + String httpMethodName, String exceptionName) { + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), + DATA_SOURCE, metricId, HTTP_CALL, httpMethodName, UNKNOWN_FAILURE)).mark(); + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), + DATA_SOURCE, metricId, HTTP_CALL, httpMethodName, UNKNOWN_FAILURE, exceptionName)).mark(); + + } + + public static void recordRemoteCallUnknownFailure(DataStoreType dataStoreType, String metricId, + String httpMethodName, String exceptionName, String serviceName) { + recordRemoteCallUnknownFailure(dataStoreType, metricId, httpMethodName, exceptionName); + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), + DATA_SOURCE, metricId, HTTP_CALL, httpMethodName, SERVICE_NAME, serviceName, + UNKNOWN_FAILURE)).mark(); + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), + DATA_SOURCE, metricId, HTTP_CALL, httpMethodName, SERVICE_NAME, serviceName, + UNKNOWN_FAILURE, exceptionName)).mark(); + + } + + public static void recordNullOrEmptyServicesListResponse(DataStoreType dataStoreType, String metricId) { + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), + DATA_SOURCE, metricId, HTTP_CALL, SERVICES_LIST, NULL_OR_EMPTY_RESPONSE)).mark(); + } + + public static void recordNullOrEmptyListNodeResponse(DataStoreType dataStoreType, String metricId, String serviceName) { + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), + DATA_SOURCE, metricId, HTTP_CALL, LIST_NODES, SERVICE_NAME, serviceName, NULL_OR_EMPTY_RESPONSE)).mark(); + } + + public static void recordNodeDataSinkUnknownFailure(DataStoreType dataStoreType, String metricId, String serviceName, String exceptionName) { + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), + DATA_SOURCE, metricId, NODE_DATA_SINK, SERVICE_NAME, serviceName, UNKNOWN_FAILURE)).mark(); + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), + DATA_SOURCE, metricId, NODE_DATA_SINK, SERVICE_NAME, serviceName, UNKNOWN_FAILURE, exceptionName)).mark(); + } + + public static void recordNodeDataSinkSerDeFailure(DataStoreType dataStoreType, String metricId, String serDe, String serviceName, String exceptionName) { + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), DATA_SOURCE, metricId, NODE_DATA_SINK, serDe, FAILURE)).mark(); + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), + DATA_SOURCE, metricId, NODE_DATA_SINK, serDe,SERVICE_NAME, serviceName, FAILURE)).mark(); + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), + DATA_SOURCE, metricId, NODE_DATA_SINK, serDe,SERVICE_NAME, serviceName, FAILURE, exceptionName)).mark(); + } + + public static void recordNullOrEmptyRegisterServiceResponse(DataStoreType dataStoreType, String metricId, String serviceName) { + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), + DATA_SOURCE, metricId, HTTP_CALL, REGISTER_SERVICE, NULL_OR_EMPTY_RESPONSE)).mark(); + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), + DATA_SOURCE, metricId, HTTP_CALL, REGISTER_SERVICE, SERVICE_NAME, serviceName, NULL_OR_EMPTY_RESPONSE)).mark(); + } } diff --git a/ranger-core/src/test/java/io/appform/ranger/core/finderhub/ServiceFinderHubTest.java b/ranger-core/src/test/java/io/appform/ranger/core/finderhub/ServiceFinderHubTest.java index 5c162910..d6cee0c0 100644 --- a/ranger-core/src/test/java/io/appform/ranger/core/finderhub/ServiceFinderHubTest.java +++ b/ranger-core/src/test/java/io/appform/ranger/core/finderhub/ServiceFinderHubTest.java @@ -167,7 +167,7 @@ public ServiceFinder> build( } @Override - protected NodeDataSource> dataSource(Service service) { + protected NodeDataSource> dataSource(String metricId, Service service) { return new TestNodeDataSource(); } @@ -186,6 +186,16 @@ public TestServiceFinderBuilder withSleepDuration(final int finderSleepDurationS } private static class TestNodeDataSource implements NodeDataSource> { + @Override + public String getMetricId() { + return "testNodeDataSource"; + } + + @Override + public DataStoreType getDataStoreType() { + return DataStoreType.HTTP; + } + @Override public Optional>> refresh(Deserializer deserializer) { val list = new ArrayList>(); diff --git a/ranger-core/src/test/java/io/appform/ranger/core/serviceprovider/ServiceProviderTest.java b/ranger-core/src/test/java/io/appform/ranger/core/serviceprovider/ServiceProviderTest.java index 1d26fcca..ec967352 100644 --- a/ranger-core/src/test/java/io/appform/ranger/core/serviceprovider/ServiceProviderTest.java +++ b/ranger-core/src/test/java/io/appform/ranger/core/serviceprovider/ServiceProviderTest.java @@ -93,7 +93,7 @@ public ServiceProvider> build() { } @Override - protected NodeDataSink> dataSink(Service service) { + protected NodeDataSink> dataSink(String metricId, Service service) { return new TestNodeDataSink<>(); } } diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/Constants.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/Constants.java index 018d15b8..1097eba2 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/Constants.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/Constants.java @@ -25,8 +25,6 @@ */ @UtilityClass public class Constants { - public static final String DEFAULT_NAMESPACE = "default"; - public static final String DEFAULT_HOST = "__DEFAULT_SERVICE_HOST"; public static final int DEFAULT_PORT = -1; public static final int DEFAULT_DW_CHECK_INTERVAL = 15; diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundle.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundle.java index d70a22fd..1c8286e2 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundle.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundle.java @@ -31,6 +31,7 @@ import io.appform.ranger.core.model.ServiceNode; import io.appform.ranger.core.model.ShardSelector; import io.appform.ranger.core.serviceprovider.ServiceProvider; +import io.appform.ranger.core.util.MetricRecorder; import io.appform.ranger.discovery.core.ServiceDiscoveryConfiguration; import io.appform.ranger.discovery.core.healthchecks.InitialDelayChecker; import io.appform.ranger.discovery.core.healthchecks.InternalHealthChecker; @@ -76,6 +77,7 @@ import java.util.stream.Collectors; import static io.appform.ranger.discovery.bundle.Constants.LOCAL_ADDRESSES; +import static io.appform.ranger.discovery.core.Constants.DEFAULT_DATA_SINK_ID; import static java.util.Objects.requireNonNull; @@ -108,8 +110,8 @@ protected ServiceDiscoveryBundle() { protected ServiceDiscoveryBundle(List globalIdConstraints) { this.globalIdConstraints = globalIdConstraints != null - ? globalIdConstraints - : Collections.emptyList(); + ? globalIdConstraints + : Collections.emptyList(); } @Override @@ -143,6 +145,9 @@ public void run(T configuration, portScheme); serviceDiscoveryClient = buildDiscoveryClient(environment, namespace, serviceName, initialCriteria, useInitialCriteria, shardSelector); + if (serviceDiscoveryConfiguration.isMetricsEnabled()){ + MetricRecorder.initialize(environment.metrics()); + } environment.lifecycle() .manage(new ServiceDiscoveryManager(serviceName)); environment.jersey() @@ -235,6 +240,7 @@ private RangerClient> buildDiscove boolean mergeWithInitialCriteria, ShardSelector> shardSelector) { return SimpleRangerZKClient.builder() + .metricId(DEFAULT_DATA_SINK_ID) .curatorFramework(curator) .namespace(namespace) .serviceName(serviceName) @@ -267,14 +273,15 @@ private ServiceProvider> buildService val nodeInfoResolver = createNodeInfoResolver(); val nodeInfo = nodeInfoResolver.resolve(serviceDiscoveryConfiguration); val initialDelayForMonitor = serviceDiscoveryConfiguration.getInitialDelaySeconds() > 1 - ? serviceDiscoveryConfiguration.getInitialDelaySeconds() - 1 - : 0; + ? serviceDiscoveryConfiguration.getInitialDelaySeconds() - 1 + : 0; val dwMonitoringInterval = serviceDiscoveryConfiguration.getDropwizardCheckInterval() == 0 - ? Constants.DEFAULT_DW_CHECK_INTERVAL - : serviceDiscoveryConfiguration.getDropwizardCheckInterval(); + ? Constants.DEFAULT_DW_CHECK_INTERVAL + : serviceDiscoveryConfiguration.getDropwizardCheckInterval(); val dwMonitoringStaleness = Math.max(serviceDiscoveryConfiguration.getDropwizardCheckStaleness(), dwMonitoringInterval + 1); val serviceProviderBuilder = ServiceProviderBuilders.shardedServiceProviderBuilder() + .withMetricId(DEFAULT_DATA_SINK_ID) .withCuratorFramework(curator) .withNamespace(namespace) .withServiceName(serviceName) diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleCustomHostPortTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleCustomHostPortTest.java index 754cc7ef..99ab3e86 100644 --- a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleCustomHostPortTest.java +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleCustomHostPortTest.java @@ -20,6 +20,7 @@ import com.codahale.metrics.health.HealthCheckRegistry; import com.fasterxml.jackson.databind.ObjectMapper; import io.appform.ranger.core.healthcheck.HealthcheckStatus; +import io.appform.ranger.discovery.core.Constants; import io.appform.ranger.discovery.core.ServiceDiscoveryConfiguration; import io.dropwizard.Configuration; import io.dropwizard.jersey.DropwizardResourceConfig; diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleDwMonitorTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleDwMonitorTest.java index 4c21f3be..7053bb65 100644 --- a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleDwMonitorTest.java +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleDwMonitorTest.java @@ -21,6 +21,7 @@ import com.codahale.metrics.health.HealthCheckRegistry; import com.fasterxml.jackson.databind.ObjectMapper; import io.appform.ranger.core.healthcheck.HealthcheckStatus; +import io.appform.ranger.discovery.core.Constants; import io.appform.ranger.discovery.core.ServiceDiscoveryConfiguration; import io.dropwizard.Configuration; import io.dropwizard.jersey.DropwizardResourceConfig; diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleDwStalenessMonitorTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleDwStalenessMonitorTest.java index beb7afda..62831615 100644 --- a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleDwStalenessMonitorTest.java +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleDwStalenessMonitorTest.java @@ -21,6 +21,7 @@ import com.codahale.metrics.health.HealthCheckRegistry; import com.fasterxml.jackson.databind.ObjectMapper; import io.appform.ranger.core.healthcheck.HealthcheckStatus; +import io.appform.ranger.discovery.core.Constants; import io.appform.ranger.discovery.core.ServiceDiscoveryConfiguration; import io.dropwizard.Configuration; import io.dropwizard.jersey.DropwizardResourceConfig; diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleHierarchicalSelectorTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleHierarchicalSelectorTest.java index 14c58c56..2f6e2018 100644 --- a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleHierarchicalSelectorTest.java +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleHierarchicalSelectorTest.java @@ -21,6 +21,7 @@ import com.codahale.metrics.health.HealthCheckRegistry; import com.fasterxml.jackson.databind.ObjectMapper; import io.appform.ranger.core.healthcheck.HealthcheckStatus; +import io.appform.ranger.discovery.core.Constants; import io.appform.ranger.discovery.core.ServiceDiscoveryConfiguration; import io.appform.ranger.discovery.core.util.ConfigurationUtils; import io.dropwizard.Configuration; diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleLocalHostPortTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleLocalHostPortTest.java index 3ecdfec2..c03d2c2e 100644 --- a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleLocalHostPortTest.java +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleLocalHostPortTest.java @@ -40,6 +40,7 @@ import java.util.UUID; import static io.appform.ranger.discovery.bundle.Constants.LOCAL_ADDRESSES; +import static io.appform.ranger.discovery.core.Constants.DEFAULT_DATA_SINK_ID; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; @@ -126,6 +127,7 @@ void shouldThrowExceptionForInvalidZkHost() { when(environment.admin()).thenReturn(adminEnvironment); serviceDiscoveryConfiguration = ServiceDiscoveryConfiguration.builder() + .zookeeper(String.format("%s:2181", UUID.randomUUID())) .namespace("test") .environment("testing") diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleRotationTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleRotationTest.java index 1d3a8663..0e9bb129 100644 --- a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleRotationTest.java +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleRotationTest.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import io.appform.ranger.discovery.bundle.rotationstatus.BIRTask; import io.appform.ranger.discovery.bundle.rotationstatus.OORTask; +import io.appform.ranger.discovery.core.Constants; import io.appform.ranger.discovery.core.ServiceDiscoveryConfiguration; import io.appform.ranger.discovery.core.rotationstatus.RotationStatus; import io.appform.ranger.discovery.core.util.ConfigurationUtils; diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleTest.java index 474e47a6..0011d63b 100644 --- a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleTest.java +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleTest.java @@ -21,6 +21,7 @@ import com.codahale.metrics.health.HealthCheckRegistry; import com.fasterxml.jackson.databind.ObjectMapper; import io.appform.ranger.core.healthcheck.HealthcheckStatus; +import io.appform.ranger.discovery.core.Constants; import io.appform.ranger.discovery.core.ServiceDiscoveryConfiguration; import io.appform.ranger.discovery.core.resolvers.DefaultNodeInfoResolver; import io.appform.ranger.discovery.core.resolvers.NodeInfoResolver; diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/resolvers/DefaultNodeInfoResolverTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/resolvers/DefaultNodeInfoResolverTest.java index cac313c4..fdf07928 100644 --- a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/resolvers/DefaultNodeInfoResolverTest.java +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/resolvers/DefaultNodeInfoResolverTest.java @@ -15,6 +15,7 @@ */ package io.appform.ranger.discovery.bundle.resolvers; +import io.appform.ranger.discovery.core.Constants; import io.appform.ranger.discovery.core.ServiceDiscoveryConfiguration; import io.appform.ranger.discovery.core.resolvers.DefaultNodeInfoResolver; import lombok.val; diff --git a/ranger-discovery-core/src/main/java/io/appform/ranger/discovery/core/Constants.java b/ranger-discovery-core/src/main/java/io/appform/ranger/discovery/core/Constants.java index 50b83eb3..36da39b5 100644 --- a/ranger-discovery-core/src/main/java/io/appform/ranger/discovery/core/Constants.java +++ b/ranger-discovery-core/src/main/java/io/appform/ranger/discovery/core/Constants.java @@ -24,7 +24,7 @@ @UtilityClass public class Constants { public static final String DEFAULT_NAMESPACE = "default"; - public static final String DEFAULT_ID = "RANGER_DISCOVERY"; + public static final String DEFAULT_DATA_SINK_ID = "RANGER_DISCOVERY_ZOOKEEPER"; public static final String DEFAULT_HOST = "__DEFAULT_SERVICE_HOST"; public static final int DEFAULT_PORT = -1; diff --git a/ranger-discovery-core/src/main/java/io/appform/ranger/discovery/core/ServiceDiscoveryConfiguration.java b/ranger-discovery-core/src/main/java/io/appform/ranger/discovery/core/ServiceDiscoveryConfiguration.java index eb503a05..bd25c8fe 100644 --- a/ranger-discovery-core/src/main/java/io/appform/ranger/discovery/core/ServiceDiscoveryConfiguration.java +++ b/ranger-discovery-core/src/main/java/io/appform/ranger/discovery/core/ServiceDiscoveryConfiguration.java @@ -39,9 +39,6 @@ @NoArgsConstructor public class ServiceDiscoveryConfiguration { - @NotBlank - private String id = Constants.DEFAULT_ID; - @NotNull @NotEmpty private String namespace = Constants.DEFAULT_NAMESPACE; @@ -83,6 +80,8 @@ public class ServiceDiscoveryConfiguration { private Set tags; + private boolean metricsEnabled = true; + @Builder public ServiceDiscoveryConfiguration(String namespace, String environment, @@ -96,7 +95,8 @@ public ServiceDiscoveryConfiguration(String namespace, boolean initialRotationStatus, int dropwizardCheckInterval, int dropwizardCheckStaleness, - Set tags) { + Set tags, + boolean metricsEnabled) { this.namespace = Strings.isNullOrEmpty(namespace) ? Constants.DEFAULT_NAMESPACE : namespace; @@ -120,5 +120,6 @@ public ServiceDiscoveryConfiguration(String namespace, : dropwizardCheckInterval; this.dropwizardCheckStaleness = dropwizardCheckStaleness; this.tags = tags; + this.metricsEnabled = metricsEnabled; } } diff --git a/ranger-discovery-dw5-bundle/src/main/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundle.java b/ranger-discovery-dw5-bundle/src/main/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundle.java index 7d84c57c..59224073 100644 --- a/ranger-discovery-dw5-bundle/src/main/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundle.java +++ b/ranger-discovery-dw5-bundle/src/main/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundle.java @@ -32,6 +32,7 @@ import io.appform.ranger.core.model.ServiceNode; import io.appform.ranger.core.model.ShardSelector; import io.appform.ranger.core.serviceprovider.ServiceProvider; +import io.appform.ranger.core.util.MetricRecorder; import io.appform.ranger.discovery.core.ServiceDiscoveryConfiguration; import io.appform.ranger.discovery.core.healthchecks.InitialDelayChecker; import io.appform.ranger.discovery.core.healthchecks.InternalHealthChecker; @@ -76,6 +77,7 @@ import java.util.stream.Collectors; import static io.appform.ranger.discovery.bundle.Constants.LOCAL_ADDRESSES; +import static io.appform.ranger.discovery.core.Constants.DEFAULT_DATA_SINK_ID; /** @@ -107,8 +109,8 @@ protected ServiceDiscoveryBundle() { protected ServiceDiscoveryBundle(List globalIdConstraints) { this.globalIdConstraints = globalIdConstraints != null - ? globalIdConstraints - : Collections.emptyList(); + ? globalIdConstraints + : Collections.emptyList(); } @Override @@ -142,6 +144,9 @@ public void run(T configuration, portScheme); serviceDiscoveryClient = buildDiscoveryClient(environment, namespace, serviceName, initialCriteria, useInitialCriteria, shardSelector); + if (serviceDiscoveryConfiguration.isMetricsEnabled()){ + MetricRecorder.initialize(environment.metrics()); + } environment.lifecycle() .manage(new ServiceDiscoveryManager(serviceName)); environment.jersey() @@ -234,6 +239,7 @@ private RangerClient> buildDiscove boolean mergeWithInitialCriteria, ShardSelector> shardSelector) { return SimpleRangerZKClient.builder() + .metricId(DEFAULT_DATA_SINK_ID) .curatorFramework(curator) .namespace(namespace) .serviceName(serviceName) @@ -274,6 +280,7 @@ private ServiceProvider> buildService val dwMonitoringStaleness = Math.max(serviceDiscoveryConfiguration.getDropwizardCheckStaleness(), dwMonitoringInterval + 1); val serviceProviderBuilder = ServiceProviderBuilders.shardedServiceProviderBuilder() + .withMetricId(DEFAULT_DATA_SINK_ID) .withCuratorFramework(curator) .withNamespace(namespace) .withServiceName(serviceName) diff --git a/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleCustomHostPortTest.java b/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleCustomHostPortTest.java index f41e92df..f1327239 100644 --- a/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleCustomHostPortTest.java +++ b/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleCustomHostPortTest.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Lists; import io.appform.ranger.core.healthcheck.HealthcheckStatus; +import io.appform.ranger.discovery.core.Constants; import io.appform.ranger.discovery.core.ServiceDiscoveryConfiguration; import io.dropwizard.core.Configuration; import io.dropwizard.core.server.DefaultServerFactory; @@ -104,6 +105,7 @@ void setup() throws Exception { testingCluster.start(); serviceDiscoveryConfiguration = ServiceDiscoveryConfiguration.builder() + .zookeeper(testingCluster.getConnectString()) .namespace("test") .environment("testing") diff --git a/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleDwMonitorTest.java b/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleDwMonitorTest.java index 5cfbc20c..581d7acf 100644 --- a/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleDwMonitorTest.java +++ b/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleDwMonitorTest.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Lists; import io.appform.ranger.core.healthcheck.HealthcheckStatus; +import io.appform.ranger.discovery.core.Constants; import io.appform.ranger.discovery.core.ServiceDiscoveryConfiguration; import io.dropwizard.core.Configuration; import io.dropwizard.core.server.DefaultServerFactory; @@ -120,6 +121,7 @@ protected Result check() { testingCluster.start(); serviceDiscoveryConfiguration = ServiceDiscoveryConfiguration.builder() + .zookeeper(testingCluster.getConnectString()) .namespace("test") .environment("testing") diff --git a/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleDwStalenessMonitorTest.java b/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleDwStalenessMonitorTest.java index 593fefff..9cfb0c52 100644 --- a/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleDwStalenessMonitorTest.java +++ b/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleDwStalenessMonitorTest.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Lists; import io.appform.ranger.core.healthcheck.HealthcheckStatus; +import io.appform.ranger.discovery.core.Constants; import io.appform.ranger.discovery.core.ServiceDiscoveryConfiguration; import io.dropwizard.core.Configuration; import io.dropwizard.core.server.DefaultServerFactory; @@ -119,6 +120,7 @@ protected Result check() { testingCluster.start(); serviceDiscoveryConfiguration = ServiceDiscoveryConfiguration.builder() + .zookeeper(testingCluster.getConnectString()) .namespace("test") .environment("testing") diff --git a/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleHierarchicalSelectorTest.java b/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleHierarchicalSelectorTest.java index 0a2c8757..f61a10c8 100644 --- a/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleHierarchicalSelectorTest.java +++ b/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleHierarchicalSelectorTest.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Lists; import io.appform.ranger.core.healthcheck.HealthcheckStatus; +import io.appform.ranger.discovery.core.Constants; import io.appform.ranger.discovery.core.ServiceDiscoveryConfiguration; import io.appform.ranger.discovery.core.util.ConfigurationUtils; import io.dropwizard.core.Configuration; @@ -98,6 +99,7 @@ void setup() throws Exception { DnsCacheManipulator.setDnsCache("TestHost", "127.0.0.1"); serviceDiscoveryConfiguration = ServiceDiscoveryConfiguration.builder() + .zookeeper(testingCluster.getConnectString()) .namespace("test") .environment("x.y") diff --git a/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleLocalHostPortTest.java b/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleLocalHostPortTest.java index 62d97c13..0b360334 100644 --- a/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleLocalHostPortTest.java +++ b/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleLocalHostPortTest.java @@ -40,6 +40,7 @@ import java.util.UUID; import static io.appform.ranger.discovery.bundle.Constants.LOCAL_ADDRESSES; +import static io.appform.ranger.discovery.core.Constants.*; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; @@ -98,6 +99,7 @@ void shouldFailLocalhostPublish() { when(environment.admin()).thenReturn(adminEnvironment); serviceDiscoveryConfiguration = ServiceDiscoveryConfiguration.builder() + .zookeeper("myzookeeper:2181,myfavzookeeper:2181") .namespace("test") .environment("testing") @@ -126,6 +128,7 @@ void shouldThrowExceptionForInvalidZkHost() { when(environment.admin()).thenReturn(adminEnvironment); serviceDiscoveryConfiguration = ServiceDiscoveryConfiguration.builder() + .zookeeper(String.format("%s:2181", UUID.randomUUID())) .namespace("test") .environment("testing") @@ -162,6 +165,7 @@ void testPublishWithEmptyZkHost() throws UnknownHostException { when(environment.admin()).thenReturn(adminEnvironment); serviceDiscoveryConfiguration = ServiceDiscoveryConfiguration.builder() + .zookeeper("myzookeeper:2181,myfavzookeeper:2181") .namespace("test") .environment("testing") @@ -198,6 +202,7 @@ void testPublishWithNullZkHost() throws UnknownHostException { when(environment.admin()).thenReturn(adminEnvironment); serviceDiscoveryConfiguration = ServiceDiscoveryConfiguration.builder() + .zookeeper("myzookeeper:2181,myfavzookeeper:2181") .namespace("test") .environment("testing") @@ -233,6 +238,7 @@ void shouldPublishingToLocalZk() { when(environment.admin()).thenReturn(adminEnvironment); serviceDiscoveryConfiguration = ServiceDiscoveryConfiguration.builder() + .zookeeper("localhost:2181,myfavzookeeper:2181") .namespace("test") .environment("testing") @@ -263,6 +269,7 @@ void shouldPublishToRemoteZk() { when(environment.admin()).thenReturn(adminEnvironment); serviceDiscoveryConfiguration = ServiceDiscoveryConfiguration.builder() + .zookeeper("myfavzookeeper:2181") .namespace("test") .environment("testing") diff --git a/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleRotationTest.java b/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleRotationTest.java index b52b92e4..8e4a62ff 100644 --- a/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleRotationTest.java +++ b/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleRotationTest.java @@ -23,6 +23,7 @@ import com.google.common.collect.Lists; import io.appform.ranger.discovery.bundle.rotationstatus.BIRTask; import io.appform.ranger.discovery.bundle.rotationstatus.OORTask; +import io.appform.ranger.discovery.core.Constants; import io.appform.ranger.discovery.core.ServiceDiscoveryConfiguration; import io.appform.ranger.discovery.core.rotationstatus.RotationStatus; import io.appform.ranger.discovery.core.util.ConfigurationUtils; @@ -100,6 +101,7 @@ void setup() throws Exception { testingCluster.start(); serviceDiscoveryConfiguration = ServiceDiscoveryConfiguration.builder() + .zookeeper(testingCluster.getConnectString()) .namespace("test") .environment("testing") diff --git a/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleTest.java b/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleTest.java index 67ce0212..3587594e 100644 --- a/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleTest.java +++ b/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleTest.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Lists; import io.appform.ranger.core.healthcheck.HealthcheckStatus; +import io.appform.ranger.discovery.core.Constants; import io.appform.ranger.discovery.core.ServiceDiscoveryConfiguration; import io.appform.ranger.discovery.core.resolvers.DefaultNodeInfoResolver; import io.appform.ranger.discovery.core.resolvers.NodeInfoResolver; @@ -103,6 +104,7 @@ void setup() throws Exception { testingCluster.start(); serviceDiscoveryConfiguration = ServiceDiscoveryConfiguration.builder() + .zookeeper(testingCluster.getConnectString()) .namespace("test") .environment("testing") diff --git a/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/resolvers/DefaultNodeInfoResolverTest.java b/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/resolvers/DefaultNodeInfoResolverTest.java index cac313c4..fd577bae 100644 --- a/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/resolvers/DefaultNodeInfoResolverTest.java +++ b/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/resolvers/DefaultNodeInfoResolverTest.java @@ -15,6 +15,7 @@ */ package io.appform.ranger.discovery.bundle.resolvers; +import io.appform.ranger.discovery.core.Constants; import io.appform.ranger.discovery.core.ServiceDiscoveryConfiguration; import io.appform.ranger.discovery.core.resolvers.DefaultNodeInfoResolver; import lombok.val; @@ -26,6 +27,7 @@ class DefaultNodeInfoResolverTest { @Test void testNodeInfoResolver() { val configuration = ServiceDiscoveryConfiguration.builder() + .zookeeper("connectionString") .namespace("test") .environment("testing") diff --git a/ranger-drove-client/src/main/java/io/appform/ranger/client/drove/AbstractRangerDroveHubClient.java b/ranger-drove-client/src/main/java/io/appform/ranger/client/drove/AbstractRangerDroveHubClient.java index b205f9d8..fdf14d27 100644 --- a/ranger-drove-client/src/main/java/io/appform/ranger/client/drove/AbstractRangerDroveHubClient.java +++ b/ranger-drove-client/src/main/java/io/appform/ranger/client/drove/AbstractRangerDroveHubClient.java @@ -45,8 +45,8 @@ public abstract class AbstractRangerDroveHubClient nodeSelector = new RandomServiceNodeSelector<>(); @Override - protected ServiceDataSource getDefaultDataSource() { - return new DroveServiceDataSource<>(clientConfig, getMapper(), getNamespace(), droveCommunicator); + protected ServiceDataSource getDefaultDataSource(String metricId) { + return new DroveServiceDataSource<>(metricId, clientConfig, getMapper(), getNamespace(), droveCommunicator); } @Override diff --git a/ranger-drove-client/src/main/java/io/appform/ranger/client/drove/SimpleRangerDroveClient.java b/ranger-drove-client/src/main/java/io/appform/ranger/client/drove/SimpleRangerDroveClient.java index 87b7caf5..9aea8f0e 100644 --- a/ranger-drove-client/src/main/java/io/appform/ranger/client/drove/SimpleRangerDroveClient.java +++ b/ranger-drove-client/src/main/java/io/appform/ranger/client/drove/SimpleRangerDroveClient.java @@ -55,6 +55,7 @@ public void start() { requireNonNull(deserializer, "deserializer can't be null"); this.serviceFinder = DroveServiceFinderBuilders.droveUnshardedServiceFinderBuilider() + .withMetricId(clientConfig.getId()) .withClientConfig(clientConfig) .withServiceName(serviceName) .withNamespace(namespace) diff --git a/ranger-drove/src/main/java/io/appform/ranger/drove/common/DroveApiCommunicator.java b/ranger-drove/src/main/java/io/appform/ranger/drove/common/DroveApiCommunicator.java index c1da48b0..11b5310e 100644 --- a/ranger-drove/src/main/java/io/appform/ranger/drove/common/DroveApiCommunicator.java +++ b/ranger-drove/src/main/java/io/appform/ranger/drove/common/DroveApiCommunicator.java @@ -16,6 +16,7 @@ package io.appform.ranger.drove.common; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Strings; @@ -25,16 +26,15 @@ import com.phonepe.drove.models.api.AppSummary; import com.phonepe.drove.models.api.ExposedAppInfo; import com.phonepe.drove.models.application.ApplicationState; +import io.appform.ranger.core.model.DataStoreType; import io.appform.ranger.core.model.Service; +import io.appform.ranger.core.util.MetricRecorder; import io.appform.ranger.drove.config.DroveUpstreamConfig; import lombok.extern.slf4j.Slf4j; import lombok.val; import org.apache.http.HttpStatus; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; +import java.util.*; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -43,11 +43,15 @@ import java.util.stream.Collectors; import java.util.stream.StreamSupport; +import static io.appform.ranger.core.util.MetricRecorder.LIST_NODES; +import static io.appform.ranger.core.util.MetricRecorder.SERVICES_LIST; + /** * */ @Slf4j public class DroveApiCommunicator implements DroveCommunicator { + private final String metricId; private final String namespace; private final DroveUpstreamConfig config; private final DroveClient droveClient; @@ -63,6 +67,7 @@ public DroveApiCommunicator( this.config = config; this.droveClient = droveClient; this.mapper = mapper; + this.metricId = config.getId(); resetter.scheduleWithFixedDelay(() -> upstreamAvailable.set(true), 0, 60, TimeUnit.SECONDS); } @@ -98,13 +103,11 @@ public List defaultValue() { @Override public List handle(DroveClient.Response response) throws Exception { + MetricRecorder.recordRemoteCallStatusCode(DataStoreType.DROVE, metricId, SERVICES_LIST, response.statusCode()); if (response.statusCode() != HttpStatus.SC_OK) { throw new DroveCommunicationException("Error communicating to drove: " + response); } - val apiResponse = mapper.readValue( - response.body(), - new TypeReference>>() { - }); + final var apiResponse = parseServicesResponse(response); if (!apiResponse.getStatus().equals(ApiErrorCode.SUCCESS)) { log.error("Error calling drove: " + apiResponse.getMessage()); throwDroveCommError(response); @@ -125,6 +128,27 @@ public List handle(DroveClient.Response response) throws Exception { })); } + private ApiResponse> parseServicesResponse(DroveClient.Response response) { + try { + val apiResponse = mapper.readValue( + response.body(), + new TypeReference>>() { + }); + + if(apiResponse == null || apiResponse.getData() == null || apiResponse.getData().isEmpty()) { + MetricRecorder.recordNullOrEmptyServicesListResponse(DataStoreType.DROVE, metricId); + log.warn("Received empty services list from drove. Response body: {}", response.body()); + } + + return apiResponse; + } catch (JsonProcessingException e) { + MetricRecorder.recordServicesParseFailure(DataStoreType.DROVE, metricId); + log.error("Error parsing response from drove: {}. Response body: {}", e.getMessage(), response.body(), e); + throw new DroveCommunicationException("Error parsing response from drove: " + e.getMessage()); + } + } + + @Override @SuppressWarnings("java:S1168") public Map> listNodes(Iterable services) { @@ -144,13 +168,12 @@ public Map> defaultValue() { } @Override - public Map> handle(DroveClient.Response response) throws Exception { + public Map> handle(DroveClient.Response response) { + MetricRecorder.recordRemoteCallStatusCode(DataStoreType.DROVE, metricId, LIST_NODES, response.statusCode()); if (response.statusCode() != HttpStatus.SC_OK) { throwDroveCommError(response); } - val apiResponse = mapper.readValue(response.body(), - new TypeReference>>() { - }); + final var apiResponse = parseListNodesResponse(services, response); if (!apiResponse.getStatus().equals(ApiErrorCode.SUCCESS)) { throwDroveCommError(response); } @@ -165,6 +188,25 @@ public Map> handle(DroveClient.Response response) })); } + private ApiResponse> parseListNodesResponse(Iterable services, DroveClient.Response response) { + try { + val apiResponse = mapper.readValue(response.body(), + new TypeReference>>() { + }); + + if(apiResponse == null || apiResponse.getData() == null || apiResponse.getData().isEmpty()) { + services.forEach(service -> MetricRecorder.recordNullOrEmptyListNodeResponse(DataStoreType.DROVE, metricId, service.getServiceName())); + log.warn("Received empty services list from drove. Response body: {}", response.body()); + } + + return apiResponse; + } catch (JsonProcessingException e) { + services.forEach(service -> MetricRecorder.recordListNodesParseFailure(DataStoreType.DROVE, metricId, service.getServiceName())); + log.error("Error parsing response from drove: {}. Response body: {}", e.getMessage(), response.body(), e); + throw new DroveCommunicationException("Error parsing response from drove: " + e.getMessage()); + } + } + private T executeRemoteCall(Supplier executor) { upstreamAvailable.set(true); try { diff --git a/ranger-drove/src/main/java/io/appform/ranger/drove/common/DroveCachingCommunicator.java b/ranger-drove/src/main/java/io/appform/ranger/drove/common/DroveCachingCommunicator.java index 892a39ec..a67f6207 100644 --- a/ranger-drove/src/main/java/io/appform/ranger/drove/common/DroveCachingCommunicator.java +++ b/ranger-drove/src/main/java/io/appform/ranger/drove/common/DroveCachingCommunicator.java @@ -32,7 +32,9 @@ import com.phonepe.drove.models.events.events.DroveInstanceStateChangeEvent; import com.phonepe.drove.models.events.events.datatags.AppEventDataTag; import com.phonepe.drove.models.events.events.datatags.AppInstanceEventDataTag; +import io.appform.ranger.core.model.DataStoreType; import io.appform.ranger.core.model.Service; +import io.appform.ranger.core.util.MetricRecorder; import io.appform.ranger.drove.config.DroveUpstreamConfig; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; @@ -48,6 +50,8 @@ */ @Slf4j public class DroveCachingCommunicator implements DroveCommunicator { + + private final String metricId; private final DroveCommunicator root; private final DroveRemoteEventListener listener; //Zombie check is 60 secs .. so this provides about 10 secs @@ -60,6 +64,7 @@ public DroveCachingCommunicator( DroveUpstreamConfig config, DroveClient droveClient, ObjectMapper mapper) { + this.metricId = config.getId(); this.root = root; val offsetStore = new DroveEventPollingOffsetInMemoryStore(); offsetStore.setLastOffset(System.currentTimeMillis()); //Only interested in new events @@ -136,14 +141,18 @@ private void handleEvents(String namespace, List events, EnumSet implements NodeDataStoreConnector { + protected final String metricId; protected final DroveUpstreamConfig config; protected final ObjectMapper mapper; protected final DroveCommunicator droveClient; @@ -35,6 +38,7 @@ public DroveNodeDataStoreConnector( final DroveUpstreamConfig config, final ObjectMapper mapper, final DroveCommunicator droveClient) { + this.metricId = config.getId(); this.config = config; this.mapper = mapper; this.droveClient = droveClient; @@ -51,7 +55,6 @@ public void start() { public void ensureConnected() { do { Thread.sleep(1_000); - } while (droveClient.leader().orElse(null) == null); } @@ -62,7 +65,9 @@ public void stop() { @Override public boolean isActive() { - return droveClient.healthy(); + var healthy = droveClient.healthy(); + MetricRecorder.recordNoteDataSourceStatus(DataStoreType.DROVE, metricId, healthy); + return healthy; } } diff --git a/ranger-drove/src/main/java/io/appform/ranger/drove/common/DroveOkHttpTransport.java b/ranger-drove/src/main/java/io/appform/ranger/drove/common/DroveOkHttpTransport.java index 928a91a1..8f857a90 100644 --- a/ranger-drove/src/main/java/io/appform/ranger/drove/common/DroveOkHttpTransport.java +++ b/ranger-drove/src/main/java/io/appform/ranger/drove/common/DroveOkHttpTransport.java @@ -18,12 +18,16 @@ import com.phonepe.drove.client.DroveClient; import com.phonepe.drove.client.DroveHttpTransport; +import io.appform.ranger.core.model.DataStoreType; +import io.appform.ranger.core.util.MetricRecorder; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import lombok.val; import okhttp3.Headers; import okhttp3.OkHttpClient; import okhttp3.Request; +import okhttp3.internal.http.HttpMethod; +import org.apache.http.client.methods.HttpGet; import java.net.URI; import java.util.List; @@ -34,9 +38,12 @@ */ @Slf4j public class DroveOkHttpTransport implements DroveHttpTransport { + + private final String metricId; private final OkHttpClient httpClient; - public DroveOkHttpTransport(final OkHttpClient httpClient) { + public DroveOkHttpTransport(String metricId, final OkHttpClient httpClient) { + this.metricId = metricId; this.httpClient = httpClient; log.info("Okhttp based transport initialized"); } @@ -68,6 +75,7 @@ public T get( return responseHandler.handle(droveResponse); } catch (Exception e) { + MetricRecorder.recordRemoteCallUnknownFailure(DataStoreType.DROVE, metricId, HttpGet.METHOD_NAME, e.getClass().getSimpleName()); log.error("Error calling drove: {}. Error: {}", e.getMessage(), e.getClass().getSimpleName()); throw new DroveCommunicationException(e.getMessage()); } diff --git a/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinder/DroveNodeDataSource.java b/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinder/DroveNodeDataSource.java index bc3e448c..a56c7fd7 100644 --- a/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinder/DroveNodeDataSource.java +++ b/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinder/DroveNodeDataSource.java @@ -16,9 +16,11 @@ package io.appform.ranger.drove.servicefinder; import com.fasterxml.jackson.databind.ObjectMapper; +import io.appform.ranger.core.model.DataStoreType; import io.appform.ranger.core.model.NodeDataSource; import io.appform.ranger.core.model.Service; import io.appform.ranger.core.model.ServiceNode; +import io.appform.ranger.core.util.MetricRecorder; import io.appform.ranger.drove.common.DroveCommunicationException; import io.appform.ranger.drove.common.DroveCommunicator; import io.appform.ranger.drove.common.DroveNodeDataStoreConnector; @@ -39,17 +41,30 @@ @Slf4j public class DroveNodeDataSource> extends DroveNodeDataStoreConnector implements NodeDataSource { + private final String metricId; private final Service service; public DroveNodeDataSource( + String metricId, Service service, final DroveUpstreamConfig config, ObjectMapper mapper, DroveCommunicator droveClient) { super(config, mapper, droveClient); + this.metricId = metricId; this.service = service; } + @Override + public String getMetricId() { + return metricId; + } + + @Override + public DataStoreType getDataStoreType() { + return DataStoreType.DROVE; + } + @Override public Optional>> refresh(D deserializer) { requireNonNull(config, "client config has not been set for node data"); @@ -59,16 +74,17 @@ public Optional>> refresh(D deserializer) { val nodes = deserializer.deserialize( Objects.requireNonNull(exposedAppInfos, "Unexpected empty response from server")); return Optional.of(nodes); - } - catch (DroveCommunicationException e) { - log.error("Drove communication error", e); + } catch (DroveCommunicationException e) { + log.error("Drove communication error while refreshing data for service : {}", service.getServiceName(), e); return Optional.empty(); //In case of refresh failure, maintain old list } } @Override public boolean isActive() { - return droveClient.healthy(); + var healthy = droveClient.healthy(); + MetricRecorder.recordNoteDataSourceStatus(DataStoreType.DROVE, metricId, healthy); + return healthy; } } diff --git a/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinder/DroveShardedServiceFinderBuilder.java b/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinder/DroveShardedServiceFinderBuilder.java index fb6071fb..96741e7d 100644 --- a/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinder/DroveShardedServiceFinderBuilder.java +++ b/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinder/DroveShardedServiceFinderBuilder.java @@ -59,11 +59,11 @@ public SimpleShardedServiceFinder build() { } @Override - protected NodeDataSource> dataSource(Service service) { - return new DroveNodeDataSource<>(service, clientConfig, mapper, - Objects.requireNonNullElseGet(droveClient, - () -> RangerDroveUtils.buildDroveClient( - namespace, clientConfig, mapper))); + protected NodeDataSource> dataSource(String metricId, Service service) { + return new DroveNodeDataSource<>(metricId, service, clientConfig, + mapper, Objects.requireNonNullElseGet(droveClient, + () -> RangerDroveUtils.buildDroveClient( + namespace, clientConfig, mapper))); } } diff --git a/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinder/DroveUnshardedServiceFinderBuilider.java b/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinder/DroveUnshardedServiceFinderBuilider.java index 87cff3c4..2cb63459 100644 --- a/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinder/DroveUnshardedServiceFinderBuilider.java +++ b/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinder/DroveUnshardedServiceFinderBuilider.java @@ -56,12 +56,11 @@ public SimpleUnshardedServiceFinder build() { } @Override - protected NodeDataSource> dataSource(Service service) { - return new DroveNodeDataSource<>( + protected NodeDataSource> dataSource(String metricId, Service service) { + return new DroveNodeDataSource<>(metricId, service, clientConfig, - mapper, - Objects.requireNonNullElseGet(droveCommunicator, + mapper, Objects.requireNonNullElseGet(droveCommunicator, () -> RangerDroveUtils.buildDroveClient(namespace, clientConfig, mapper))); } diff --git a/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinderhub/DroveServiceDataSource.java b/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinderhub/DroveServiceDataSource.java index 5d839305..2c0bf0dd 100644 --- a/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinderhub/DroveServiceDataSource.java +++ b/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinderhub/DroveServiceDataSource.java @@ -16,8 +16,11 @@ package io.appform.ranger.drove.servicefinderhub; import com.fasterxml.jackson.databind.ObjectMapper; +import io.appform.functionmetrics.MonitoredFunction; import io.appform.ranger.core.finderhub.ServiceDataSource; import io.appform.ranger.core.model.Service; +import io.appform.ranger.core.model.DataStoreType; +import io.appform.ranger.core.util.MetricRecorder; import io.appform.ranger.drove.common.DroveNodeDataStoreConnector; import io.appform.ranger.drove.config.DroveUpstreamConfig; import io.appform.ranger.drove.common.DroveCommunicator; @@ -25,28 +28,42 @@ import java.util.Collection; +import static io.appform.ranger.core.util.MetricRecorder.FAILURE; +import static io.appform.ranger.core.util.MetricRecorder.SUCCESS; import static java.util.Objects.requireNonNull; @Slf4j public class DroveServiceDataSource extends DroveNodeDataStoreConnector implements ServiceDataSource { + private final String metricId; private final String namespace; public DroveServiceDataSource( + final String metricId, final DroveUpstreamConfig config, final ObjectMapper mapper, final String namespace, final DroveCommunicator droveClient) { super(config, mapper, droveClient); + this.metricId = metricId; this.namespace = namespace; } @Override + @MonitoredFunction public Collection services() { requireNonNull(config, "client config has not been set for node data"); requireNonNull(mapper, "mapper has not been set for node data"); - return droveClient.services() - .stream() - .map(serviceName -> new Service(namespace, serviceName)) - .toList(); + try { + var result = droveClient.services() + .stream() + .map(serviceName -> new Service(namespace, serviceName)) + .toList(); + MetricRecorder.recordServicesFetchStatus(DataStoreType.DROVE, metricId, SUCCESS); + return result; + } catch (Exception e) { + log.error("Error fetching services from drove data source id: {}, namespace: {}", metricId, namespace, e); + MetricRecorder.recordServicesFetchStatus(DataStoreType.DROVE, metricId, FAILURE); + throw e; + } } } diff --git a/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinderhub/DroveShardedServiceFinderFactory.java b/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinderhub/DroveShardedServiceFinderFactory.java index cb83841e..0f8520b5 100644 --- a/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinderhub/DroveShardedServiceFinderFactory.java +++ b/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinderhub/DroveShardedServiceFinderFactory.java @@ -61,6 +61,7 @@ public DroveShardedServiceFinderFactory( @Override public ServiceFinder> buildFinder(Service service) { val serviceFinder = new DroveShardedServiceFinderBuilder() + .withMetricId(clientConfig.getId()) .withClientConfig(clientConfig) .withDroveClient(droveClient) .withObjectMapper(mapper) diff --git a/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinderhub/DroveUnshardedServiceFinderFactory.java b/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinderhub/DroveUnshardedServiceFinderFactory.java index 3bf4c505..cafd4370 100644 --- a/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinderhub/DroveUnshardedServiceFinderFactory.java +++ b/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinderhub/DroveUnshardedServiceFinderFactory.java @@ -61,6 +61,7 @@ public DroveUnshardedServiceFinderFactory( @Override public ServiceFinder> buildFinder(Service service) { val serviceFinder = new DroveUnshardedServiceFinderBuilider() + .withMetricId(clientConfig.getId()) .withClientConfig(clientConfig) .withDroveCommunicator(droveCommunicator) .withObjectMapper(mapper) diff --git a/ranger-drove/src/main/java/io/appform/ranger/drove/utils/RangerDroveUtils.java b/ranger-drove/src/main/java/io/appform/ranger/drove/utils/RangerDroveUtils.java index 96f6143f..67c313d1 100644 --- a/ranger-drove/src/main/java/io/appform/ranger/drove/utils/RangerDroveUtils.java +++ b/ranger-drove/src/main/java/io/appform/ranger/drove/utils/RangerDroveUtils.java @@ -99,9 +99,7 @@ public static DroveCommunicator buildDroveClient( List.of(new BasicAuthDecorator(config.getUsername(), config.getPassword()), new AuthHeaderDecorator(config.getAuthHeader())), - new DroveOkHttpTransport(createOkHttpClient(config))); -// new DroveHttpComponentsTransport(droveConfig, -// createHttpClient(config))); + new DroveOkHttpTransport(config.getId(), createOkHttpClient(config))); val apiCommunicator = new DroveApiCommunicator(namespace, config, droveClient, mapper); return config.isSkipCaching() ? apiCommunicator diff --git a/ranger-drove/src/test/java/io/appform/ranger/drove/servicefinder/DroveNodeDataSourceTest.java b/ranger-drove/src/test/java/io/appform/ranger/drove/servicefinder/DroveNodeDataSourceTest.java index 3a7b90b7..aec348a7 100644 --- a/ranger-drove/src/test/java/io/appform/ranger/drove/servicefinder/DroveNodeDataSourceTest.java +++ b/ranger-drove/src/test/java/io/appform/ranger/drove/servicefinder/DroveNodeDataSourceTest.java @@ -75,11 +75,10 @@ void testSuccess() { Map.of(), List.of(new ExposedAppInfo.ExposedHost("h1", 100, TCP), new ExposedAppInfo.ExposedHost("h2", 100, TCP))))); - val ds = new DroveNodeDataSource( + val ds = new DroveNodeDataSource("testDroveSource", service, config, - MAPPER, - droveClient); + MAPPER, droveClient); ds.start(); val res = ds.refresh(new DNodeDataDeserializer()).orElse(null); assertNotNull(res); @@ -101,11 +100,10 @@ void testUpstreamFailure() { when(droveClient.listNodes(any(Service.class))) .thenThrow(new DroveCommunicationException("test")); - val ds = new DroveNodeDataSource( + val ds = new DroveNodeDataSource("testDroveSource", service, config, - MAPPER, - droveClient); + MAPPER, droveClient); ds.start(); val res = ds.refresh(new DNodeDataDeserializer()).orElse(null); assertNull(res); diff --git a/ranger-drove/src/test/java/io/appform/ranger/drove/servicefinderhub/DroveServiceDataSourceTest.java b/ranger-drove/src/test/java/io/appform/ranger/drove/servicefinderhub/DroveServiceDataSourceTest.java index fe5fac64..0f025fa6 100644 --- a/ranger-drove/src/test/java/io/appform/ranger/drove/servicefinderhub/DroveServiceDataSourceTest.java +++ b/ranger-drove/src/test/java/io/appform/ranger/drove/servicefinderhub/DroveServiceDataSourceTest.java @@ -100,7 +100,7 @@ void testServiceDataSource(WireMockRuntimeInfo wireMockRuntimeInfo) { val namespace = "test"; try(val droveClient = RangerDroveUtils.buildDroveClient(namespace, clientConfig, MAPPER)) { val finder = new DroveServiceDataSource( - clientConfig, + null, clientConfig, MAPPER, namespace, droveClient); diff --git a/ranger-http-client/src/main/java/io/appform/ranger/client/http/AbstractRangerHttpHubClient.java b/ranger-http-client/src/main/java/io/appform/ranger/client/http/AbstractRangerHttpHubClient.java index 5dd6910b..f078e287 100644 --- a/ranger-http-client/src/main/java/io/appform/ranger/client/http/AbstractRangerHttpHubClient.java +++ b/ranger-http-client/src/main/java/io/appform/ranger/client/http/AbstractRangerHttpHubClient.java @@ -49,8 +49,8 @@ public abstract class AbstractRangerHttpHubClient nodeSelector = new RandomServiceNodeSelector<>(); @Override - protected ServiceDataSource getDefaultDataSource() { - return new HttpServiceDataSource<>(clientConfig, + protected ServiceDataSource getDefaultDataSource(String metricId) { + return new HttpServiceDataSource<>(metricId, clientConfig, Objects.requireNonNullElseGet(getHttpClient(), () -> RangerHttpUtils.httpClient( clientConfig, diff --git a/ranger-http-client/src/main/java/io/appform/ranger/client/http/SimpleRangerHttpClient.java b/ranger-http-client/src/main/java/io/appform/ranger/client/http/SimpleRangerHttpClient.java index 6eebe5c5..8c839d76 100644 --- a/ranger-http-client/src/main/java/io/appform/ranger/client/http/SimpleRangerHttpClient.java +++ b/ranger-http-client/src/main/java/io/appform/ranger/client/http/SimpleRangerHttpClient.java @@ -57,6 +57,7 @@ public void start() { requireNonNull(deserializer, "deserializer can't be null"); this.serviceFinder = HttpServiceFinderBuilders.httpUnshardedServiceFinderBuilider() + .withMetricId(clientConfig.getId()) .withClientConfig(clientConfig) .withServiceName(serviceName) .withNamespace(namespace) diff --git a/ranger-http/pom.xml b/ranger-http/pom.xml index 959ca844..be9f550c 100644 --- a/ranger-http/pom.xml +++ b/ranger-http/pom.xml @@ -28,6 +28,10 @@ ranger-http HTTP and Hub based Discovery For Ranger + + 2.0.1.Final + + io.appform.ranger @@ -55,6 +59,11 @@ ${wiremock.version} test + + javax.validation + validation-api + ${javax.validation.version} + io.appform.ranger ranger-core diff --git a/ranger-http/src/main/java/io/appform/ranger/http/common/HttpNodeDataStoreConnector.java b/ranger-http/src/main/java/io/appform/ranger/http/common/HttpNodeDataStoreConnector.java index 210a6e02..1f945c93 100644 --- a/ranger-http/src/main/java/io/appform/ranger/http/common/HttpNodeDataStoreConnector.java +++ b/ranger-http/src/main/java/io/appform/ranger/http/common/HttpNodeDataStoreConnector.java @@ -15,7 +15,9 @@ */ package io.appform.ranger.http.common; +import io.appform.ranger.core.model.DataStoreType; import io.appform.ranger.core.model.NodeDataStoreConnector; +import io.appform.ranger.core.util.MetricRecorder; import io.appform.ranger.http.config.HttpClientConfig; import io.appform.ranger.http.servicefinder.HttpCommunicator; import lombok.extern.slf4j.Slf4j; @@ -60,6 +62,7 @@ protected int defaultPort() { @Override public boolean isActive() { + MetricRecorder.recordNoteDataSourceStatus(DataStoreType.HTTP, config.getId(), true); return true; } } diff --git a/ranger-http/src/main/java/io/appform/ranger/http/config/HttpClientConfig.java b/ranger-http/src/main/java/io/appform/ranger/http/config/HttpClientConfig.java index f7397342..65d41c87 100644 --- a/ranger-http/src/main/java/io/appform/ranger/http/config/HttpClientConfig.java +++ b/ranger-http/src/main/java/io/appform/ranger/http/config/HttpClientConfig.java @@ -20,6 +20,8 @@ import lombok.Value; import lombok.extern.jackson.Jacksonized; +import javax.validation.constraints.NotBlank; + /** * */ @@ -28,6 +30,8 @@ @Jacksonized @AllArgsConstructor public class HttpClientConfig { + + @NotBlank String id; String host; int port; diff --git a/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpApiCommunicator.java b/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpApiCommunicator.java index 3dc180de..f6ab5b1c 100644 --- a/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpApiCommunicator.java +++ b/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpApiCommunicator.java @@ -17,8 +17,10 @@ package io.appform.ranger.http.servicefinder; import com.fasterxml.jackson.databind.ObjectMapper; +import io.appform.ranger.core.model.DataStoreType; import io.appform.ranger.core.model.Service; import io.appform.ranger.core.model.ServiceNode; +import io.appform.ranger.core.util.MetricRecorder; import io.appform.ranger.http.config.HttpClientConfig; import io.appform.ranger.http.model.ServiceDataSourceResponse; import io.appform.ranger.http.serde.HTTPResponseDataDeserializer; @@ -38,6 +40,8 @@ import okhttp3.Request; import okhttp3.Response; +import static io.appform.ranger.core.util.MetricRecorder.*; + /** * Direct api based communication */ @@ -46,6 +50,7 @@ public class HttpApiCommunicator implements HttpCommunicator { private final AtomicBoolean upstreamAvailable = new AtomicBoolean(true); private final ScheduledExecutorService resetter = Executors.newSingleThreadScheduledExecutor(); + private final String metricId; @Getter private final OkHttpClient httpClient; private final HttpClientConfig config; @@ -53,6 +58,7 @@ public class HttpApiCommunicator implements HttpCommunicator { public HttpApiCommunicator(OkHttpClient httpClient, HttpClientConfig config, ObjectMapper mapper) { Objects.requireNonNull(mapper, "mapper has not been set for node data"); + this.metricId = config.getId(); this.httpClient = httpClient; this.config = config; this.mapper = mapper; @@ -80,6 +86,7 @@ public Set services() { .build(); try (val response = httpClient.newCall(request).execute()) { + MetricRecorder.recordRemoteCallStatusCode(DataStoreType.HTTP, metricId, SERVICES_LIST, response.code()); if (response.isSuccessful()) { return parseServices(response, httpUrl); } @@ -89,6 +96,7 @@ public Set services() { } } catch (Exception e) { + MetricRecorder.recordRemoteCallUnknownFailure(DataStoreType.HTTP, metricId, SERVICES_LIST, e.getClass().getSimpleName()); throw new HttpCommunicationException( "Error parsing the response from server for url: " + httpUrl + " with exception " + e.getClass().getSimpleName() + ": " + e.getMessage()); @@ -117,14 +125,16 @@ public List> listNodes( .build(); try (val response = httpClient.newCall(request).execute()) { + MetricRecorder.recordRemoteCallStatusCode(DataStoreType.HTTP, metricId, LIST_NODES, response.code()); if (response.isSuccessful()) { - return parseNodeList(deserializer, response, httpUrl); + return parseNodeList(service, deserializer, response, httpUrl); } else { throw new HttpCommunicationException("HTTP call failed. url: " + httpUrl + " status: " + response.code()); } } catch (Exception e) { + MetricRecorder.recordRemoteCallUnknownFailure(DataStoreType.HTTP, metricId, LIST_NODES, e.getClass().getSimpleName(), service.getServiceName()); throw new HttpCommunicationException("Error getting node data from the http endpoint: " + httpUrl + ". Error: " + e.getMessage()); } @@ -161,6 +171,12 @@ private Set parseServices(Response response, HttpUrl httpUrl) { else { val bytes = body.bytes(); val serviceDataSourceResponse = mapper.readValue(bytes, ServiceDataSourceResponse.class); + + if(serviceDataSourceResponse == null || serviceDataSourceResponse.getData() == null || serviceDataSourceResponse.getData().isEmpty()) { + MetricRecorder.recordNullOrEmptyServicesListResponse(DataStoreType.HTTP, metricId); + log.warn("Received empty services list from http. Response body: {}", response.body()); + } + if (serviceDataSourceResponse.valid()) { return serviceDataSourceResponse.getData(); } @@ -169,25 +185,35 @@ private Set parseServices(Response response, HttpUrl httpUrl) { "Http call to returned a failure response. Url:" + httpUrl + " data: " + serviceDataSourceResponse); } } - } - catch (Exception e) { + } catch (HttpCommunicationException httpCommunicationException){ + throw httpCommunicationException; + } catch (Exception e) { + MetricRecorder.recordServicesParseFailure(DataStoreType.HTTP, metricId); throw new HttpCommunicationException( "Error reading data from server. Url: " + httpUrl + "Error: " + e.getMessage()); } } - private static List> parseNodeList( + private List> parseNodeList( + Service service, HTTPResponseDataDeserializer deserializer, Response response, HttpUrl httpUrl) { try (val body = response.body()) { if (null == body) { log.warn("HTTP call to {} returned empty body", httpUrl); + MetricRecorder.recordNullOrEmptyListNodeResponse(DataStoreType.HTTP, metricId, service.getServiceName()); throw new HttpCommunicationException("Empty response received for call to " + httpUrl); } else { val bytes = body.bytes(); val serviceNodesResponse = deserializer.deserialize(bytes); + + if(serviceNodesResponse == null || serviceNodesResponse.getData() == null || serviceNodesResponse.getData().isEmpty()) { + MetricRecorder.recordNullOrEmptyListNodeResponse(DataStoreType.HTTP, metricId, service.getServiceName()); + log.warn("Received empty node list from http. Response body: {}", response.body()); + } + if (serviceNodesResponse.valid()) { return serviceNodesResponse.getData(); } @@ -196,8 +222,10 @@ private static List> parseNodeList( "Http call returned null nodes for url: " + httpUrl + " response: " + serviceNodesResponse); } } - } - catch (Exception e) { + } catch (HttpCommunicationException httpCommunicationException){ + throw httpCommunicationException; + } catch (Exception e) { + MetricRecorder.recordListNodesParseFailure(DataStoreType.HTTP, metricId, service.getServiceName()); throw new HttpCommunicationException( "Error parsing node data from server. Url: " + httpUrl + "Error: " + e.getMessage()); } diff --git a/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpNodeDataSource.java b/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpNodeDataSource.java index 980dc54f..900fa63f 100644 --- a/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpNodeDataSource.java +++ b/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpNodeDataSource.java @@ -16,6 +16,7 @@ package io.appform.ranger.http.servicefinder; import io.appform.functionmetrics.MonitoredFunction; +import io.appform.ranger.core.model.DataStoreType; import io.appform.ranger.core.model.NodeDataSource; import io.appform.ranger.core.model.Service; import io.appform.ranger.core.model.ServiceNode; @@ -39,21 +40,33 @@ @Slf4j public class HttpNodeDataSource> extends HttpNodeDataStoreConnector implements NodeDataSource { + private final String metricId; private final Service service; private final AtomicBoolean upstreamAvailable = new AtomicBoolean(true); private final ScheduledExecutorService resetter = Executors.newSingleThreadScheduledExecutor(); public HttpNodeDataSource( + final String metricId, final Service service, final HttpClientConfig config, final HttpCommunicator httpCommunicator) { super(config, httpCommunicator); Objects.requireNonNull(config, "client config has not been set for node data"); Objects.requireNonNull(httpCommunicator, "http communicator has not been set for node data"); + this.metricId = metricId; this.service = service; resetter.scheduleWithFixedDelay(() -> upstreamAvailable.set(true), 0, 60, TimeUnit.SECONDS); } + public String getMetricId() { + return metricId; + } + + @Override + public DataStoreType getDataStoreType() { + return DataStoreType.HTTP; + } + @Override @MonitoredFunction public Optional>> refresh(D deserializer) { @@ -63,7 +76,7 @@ public Optional>> refresh(D deserializer) { @Override public boolean isActive() { var httpUpstreamAvailable = upstreamAvailable.get(); - MetricRecorder.recordHttpUpstreamAvailability(this.config.getId(), httpUpstreamAvailable); + MetricRecorder.recordNoteDataSourceStatus(DataStoreType.HTTP, metricId, httpUpstreamAvailable); return httpUpstreamAvailable; } } diff --git a/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpShardedServiceFinderBuilder.java b/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpShardedServiceFinderBuilder.java index 44d3cf6e..70f995f5 100644 --- a/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpShardedServiceFinderBuilder.java +++ b/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpShardedServiceFinderBuilder.java @@ -56,8 +56,8 @@ public SimpleShardedServiceFinder build() { } @Override - protected NodeDataSource> dataSource(Service service) { - return new HttpNodeDataSource<>(service, clientConfig, + protected NodeDataSource> dataSource(String metricId, Service service) { + return new HttpNodeDataSource<>(metricId, service, clientConfig, Objects.requireNonNullElseGet(httpCommunicator, () -> RangerHttpUtils.httpClient(clientConfig, mapper))); } diff --git a/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpUnshardedServiceFinderBuilider.java b/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpUnshardedServiceFinderBuilider.java index fae76ea2..3954c5a0 100644 --- a/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpUnshardedServiceFinderBuilider.java +++ b/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpUnshardedServiceFinderBuilider.java @@ -54,8 +54,8 @@ public SimpleUnshardedServiceFinder build() { } @Override - protected NodeDataSource> dataSource(Service service) { - return new HttpNodeDataSource<>(service, clientConfig, + protected NodeDataSource> dataSource(String metricId, Service service) { + return new HttpNodeDataSource<>(metricId, service, clientConfig, Objects.requireNonNullElseGet(httpClient, () -> RangerHttpUtils.httpClient(clientConfig, mapper))); } diff --git a/ranger-http/src/main/java/io/appform/ranger/http/servicefinderhub/HttpServiceDataSource.java b/ranger-http/src/main/java/io/appform/ranger/http/servicefinderhub/HttpServiceDataSource.java index c5bf0805..4107556f 100644 --- a/ranger-http/src/main/java/io/appform/ranger/http/servicefinderhub/HttpServiceDataSource.java +++ b/ranger-http/src/main/java/io/appform/ranger/http/servicefinderhub/HttpServiceDataSource.java @@ -18,25 +18,42 @@ import io.appform.functionmetrics.MonitoredFunction; import io.appform.ranger.core.finderhub.ServiceDataSource; import io.appform.ranger.core.model.Service; +import io.appform.ranger.core.model.DataStoreType; +import io.appform.ranger.core.util.MetricRecorder; import io.appform.ranger.http.common.HttpNodeDataStoreConnector; import io.appform.ranger.http.config.HttpClientConfig; import io.appform.ranger.http.servicefinder.HttpCommunicator; import lombok.extern.slf4j.Slf4j; +import lombok.val; import java.util.Collection; import java.util.Objects; +import static io.appform.ranger.core.util.MetricRecorder.FAILURE; +import static io.appform.ranger.core.util.MetricRecorder.SUCCESS; + @Slf4j public class HttpServiceDataSource extends HttpNodeDataStoreConnector implements ServiceDataSource { - public HttpServiceDataSource(HttpClientConfig config, HttpCommunicator httpClient) { + private final String metricId; + + public HttpServiceDataSource(String metricId, HttpClientConfig config, HttpCommunicator httpClient) { super(config, httpClient); + this.metricId = metricId; } @Override @MonitoredFunction public Collection services() { Objects.requireNonNull(config, "client config has not been set for node data"); - return httpCommunicator.services(); + try { + val result = httpCommunicator.services(); + MetricRecorder.recordServicesFetchStatus(DataStoreType.HTTP, metricId, SUCCESS); + return result; + } + catch (Exception e) { + MetricRecorder.recordServicesFetchStatus(DataStoreType.HTTP, metricId, FAILURE); + throw e; + } } } diff --git a/ranger-http/src/main/java/io/appform/ranger/http/servicefinderhub/HttpShardedServiceFinderFactory.java b/ranger-http/src/main/java/io/appform/ranger/http/servicefinderhub/HttpShardedServiceFinderFactory.java index b1b1f508..3e1c042d 100644 --- a/ranger-http/src/main/java/io/appform/ranger/http/servicefinderhub/HttpShardedServiceFinderFactory.java +++ b/ranger-http/src/main/java/io/appform/ranger/http/servicefinderhub/HttpShardedServiceFinderFactory.java @@ -61,6 +61,7 @@ public HttpShardedServiceFinderFactory( @Override public ServiceFinder> buildFinder(Service service) { val serviceFinder = new HttpShardedServiceFinderBuilder() + .withMetricId(clientConfig.getId()) .withClientConfig(clientConfig) .withObjectMapper(mapper) .withHttpCommunicator(httpClient) diff --git a/ranger-http/src/main/java/io/appform/ranger/http/servicefinderhub/HttpUnshardedServiceFinderFactory.java b/ranger-http/src/main/java/io/appform/ranger/http/servicefinderhub/HttpUnshardedServiceFinderFactory.java index 51c8cb02..3bdd5a73 100644 --- a/ranger-http/src/main/java/io/appform/ranger/http/servicefinderhub/HttpUnshardedServiceFinderFactory.java +++ b/ranger-http/src/main/java/io/appform/ranger/http/servicefinderhub/HttpUnshardedServiceFinderFactory.java @@ -60,6 +60,7 @@ public HttpUnshardedServiceFinderFactory( @Override public ServiceFinder> buildFinder(Service service) { val serviceFinder = new HttpUnshardedServiceFinderBuilider() + .withMetricId(clientConfig.getId()) .withClientConfig(clientConfig) .withObjectMapper(mapper) .withHttpClient(httpClient) diff --git a/ranger-http/src/main/java/io/appform/ranger/http/serviceprovider/HttpNodeDataSink.java b/ranger-http/src/main/java/io/appform/ranger/http/serviceprovider/HttpNodeDataSink.java index 96964be4..d295efcb 100644 --- a/ranger-http/src/main/java/io/appform/ranger/http/serviceprovider/HttpNodeDataSink.java +++ b/ranger-http/src/main/java/io/appform/ranger/http/serviceprovider/HttpNodeDataSink.java @@ -18,10 +18,12 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import io.appform.functionmetrics.MonitoredFunction; +import io.appform.ranger.core.model.DataStoreType; import io.appform.ranger.core.model.NodeDataSink; import io.appform.ranger.core.model.Service; import io.appform.ranger.core.model.ServiceNode; import io.appform.ranger.core.util.Exceptions; +import io.appform.ranger.core.util.MetricRecorder; import io.appform.ranger.http.common.HttpNodeDataStoreConnector; import io.appform.ranger.http.config.HttpClientConfig; import io.appform.ranger.http.model.ServiceRegistrationResponse; @@ -32,20 +34,24 @@ import okhttp3.HttpUrl; import okhttp3.Request; import okhttp3.RequestBody; +import okhttp3.ResponseBody; import java.io.IOException; import java.util.Optional; +import static io.appform.ranger.core.util.MetricRecorder.*; import static java.util.Objects.requireNonNull; @Slf4j public class HttpNodeDataSink> extends HttpNodeDataStoreConnector implements NodeDataSink { + private final String metricId; private final Service service; private final ObjectMapper mapper; - public HttpNodeDataSink(Service service, HttpClientConfig config, ObjectMapper mapper, HttpCommunicator httpClient) { + public HttpNodeDataSink(String metricId, Service service, HttpClientConfig config, ObjectMapper mapper, HttpCommunicator httpClient) { super(config, httpClient); + this.metricId = metricId; this.service = service; this.mapper = mapper; } @@ -69,28 +75,42 @@ public void updateState(S serializer, ServiceNode serviceNode) { : config.getPort()) .encodedPath(url) .build(); - val requestBody = RequestBody.create(serializer.serialize(serviceNode)); - val serviceRegistrationResponse = registerService(httpUrl, requestBody).orElse(null); + val serializedData = getSerializedData(service.getServiceName(), serializer, serviceNode); + val requestBody = RequestBody.create(serializedData); + val serviceRegistrationResponse = registerService(service.getServiceName(), httpUrl, requestBody).orElse(null); if(null == serviceRegistrationResponse || !serviceRegistrationResponse.valid()){ log.warn("Http call to {} returned a failure response {}", httpUrl, serviceRegistrationResponse); + MetricRecorder.recordNullOrEmptyRegisterServiceResponse(DataStoreType.HTTP, metricId, service.getServiceName()); + MetricRecorder.recordNodeDataSinkUpdateStatus(DataStoreType.HTTP, metricId, FAILURE); Exceptions.illegalState("Error updating state on the server for node data: " + httpUrl); } + MetricRecorder.recordNodeDataSinkUpdateStatus(DataStoreType.HTTP, metricId, SUCCESS); } - private Optional> registerService(HttpUrl httpUrl, RequestBody requestBody){ + private > byte[] getSerializedData(String serviceName, S serializer, ServiceNode serviceNode) { + try { + return serializer.serialize(serviceNode); + } catch (Exception e) { + MetricRecorder.recordNodeDataSinkSerDeFailure(DataStoreType.HTTP, metricId, MetricRecorder.SERIALIZAION, serviceName, e.getClass().getSimpleName()); + log.error("Error serializing data for service {} with node {} with exception", serviceName, serviceNode, e); + throw e; + } + } + + private Optional> registerService(String serviceName, HttpUrl httpUrl, RequestBody requestBody){ val request = new Request.Builder() .url(httpUrl) .post(requestBody) .build(); try (val response = httpCommunicator.getHttpClient().newCall(request).execute()) { + MetricRecorder.recordRemoteCallStatusCode(DataStoreType.HTTP, metricId, REGISTER_SERVICE, response.code()); if (response.isSuccessful()) { try (val body = response.body()) { if (null == body) { log.warn("HTTP call to {} returned empty body", httpUrl); } else { - return Optional.of(mapper.readValue(body.bytes(), - new TypeReference>() {})); + return Optional.of(parseRegisterServiceResponse(serviceName, body)); } } } @@ -99,8 +119,21 @@ private Optional> registerService(HttpUrl httpUrl } } catch (IOException e) { + MetricRecorder.recordNodeDataSinkUnknownFailure(DataStoreType.HTTP, metricId, serviceName, e.getClass().getSimpleName()); log.error("Error updating state on the server with httpUrl {} with exception {} ", httpUrl, e); } return Optional.empty(); } + + private ServiceRegistrationResponse parseRegisterServiceResponse(String serviceName, ResponseBody body) throws IOException { + try { + return mapper.readValue(body.bytes(), + new TypeReference>() { + }); + } + catch (IOException e) { + MetricRecorder.recordNodeDataSinkSerDeFailure(DataStoreType.HTTP, metricId, DESERIALIZATION, serviceName, e.getClass().getSimpleName()); + throw e; + } + } } diff --git a/ranger-http/src/main/java/io/appform/ranger/http/serviceprovider/HttpShardedServiceProviderBuilder.java b/ranger-http/src/main/java/io/appform/ranger/http/serviceprovider/HttpShardedServiceProviderBuilder.java index 4caec43d..504434d3 100644 --- a/ranger-http/src/main/java/io/appform/ranger/http/serviceprovider/HttpShardedServiceProviderBuilder.java +++ b/ranger-http/src/main/java/io/appform/ranger/http/serviceprovider/HttpShardedServiceProviderBuilder.java @@ -56,8 +56,8 @@ public ServiceProvider> build() { } @Override - protected NodeDataSink> dataSink(Service service) { - return new HttpNodeDataSink<>(service, clientConfig, mapper, + protected NodeDataSink> dataSink(String metricId, Service service) { + return new HttpNodeDataSink<>(metricId, service, clientConfig, mapper, Objects.requireNonNullElseGet(httpClient, () -> RangerHttpUtils.httpClient(clientConfig, mapper))); } diff --git a/ranger-http/src/test/java/io/appform/ranger/http/servicefinderhub/HttpServiceDataSourceTest.java b/ranger-http/src/test/java/io/appform/ranger/http/servicefinderhub/HttpServiceDataSourceTest.java index 9a967926..362f2944 100644 --- a/ranger-http/src/test/java/io/appform/ranger/http/servicefinderhub/HttpServiceDataSourceTest.java +++ b/ranger-http/src/test/java/io/appform/ranger/http/servicefinderhub/HttpServiceDataSourceTest.java @@ -65,7 +65,7 @@ void testServiceDataSource(WireMockRuntimeInfo wireMockRuntimeInfo) throws IOExc .connectionTimeoutMs(30_000) .operationTimeoutMs(30_000) .build(); - val httpServiceDataSource = new HttpServiceDataSource<>(clientConfig, + val httpServiceDataSource = new HttpServiceDataSource<>(null, clientConfig, RangerHttpUtils.httpClient(clientConfig, MAPPER)); val services = httpServiceDataSource.services(); Assertions.assertNotNull(services); @@ -86,7 +86,7 @@ void testServiceDataSource(WireMockRuntimeInfo wireMockRuntimeInfo) throws IOExc .operationTimeoutMs(30_000) .replicationSource(true) .build(); - val httpServiceDataSource = new HttpServiceDataSource<>(clientConfig, + val httpServiceDataSource = new HttpServiceDataSource<>(null, clientConfig, RangerHttpUtils.httpClient(clientConfig, MAPPER)); val services = httpServiceDataSource.services(); Assertions.assertNotNull(services); diff --git a/ranger-hub-server-bundle/src/main/java/io/appform/ranger/hub/server/bundle/RangerHubServerBundle.java b/ranger-hub-server-bundle/src/main/java/io/appform/ranger/hub/server/bundle/RangerHubServerBundle.java index 50196088..5808600b 100644 --- a/ranger-hub-server-bundle/src/main/java/io/appform/ranger/hub/server/bundle/RangerHubServerBundle.java +++ b/ranger-hub-server-bundle/src/main/java/io/appform/ranger/hub/server/bundle/RangerHubServerBundle.java @@ -100,6 +100,7 @@ private RangerHubClient> addCurat .build(); curatorFrameworks.add(curatorFramework); return UnshardedRangerZKHubClient.builder() + .metricId(zkConfiguration.getId()) .namespace(namespace) .connectionString(zookeeper) .curatorFramework(curatorFramework) @@ -125,6 +126,7 @@ private RangerHubClient> addCurat private RangerHubClient> getHttpHubClient( HttpClientConfig httpClientConfig, RangerHttpUpstreamConfiguration httpConfiguration) { return UnshardedRangerHttpHubClient.builder() + .metricId(httpClientConfig.getId()) .namespace(namespace) .mapper(getMapper()) .clientConfig(httpClientConfig) @@ -153,6 +155,7 @@ private RangerHubClient> getDrove DroveUpstreamConfig.DEFAULT_REGION_TAG_NAME); val droveCommunicator = RangerDroveUtils.buildDroveClient(namespace, droveConfig, getMapper()); return UnshardedRangerDroveHubClient.builder() + .metricId(droveConfig.getId()) .namespace(namespace) .mapper(getMapper()) .clientConfig(droveConfig) diff --git a/ranger-server-bundle/src/main/java/io/appform/ranger/server/bundle/RangerServerBundle.java b/ranger-server-bundle/src/main/java/io/appform/ranger/server/bundle/RangerServerBundle.java index 59b2e451..db4fcfa0 100644 --- a/ranger-server-bundle/src/main/java/io/appform/ranger/server/bundle/RangerServerBundle.java +++ b/ranger-server-bundle/src/main/java/io/appform/ranger/server/bundle/RangerServerBundle.java @@ -20,6 +20,7 @@ import io.appform.ranger.client.RangerHubClient; import io.appform.ranger.core.model.ServiceRegistry; import io.appform.ranger.core.signals.Signal; +import io.appform.ranger.core.util.MetricRecorder; import io.appform.ranger.server.bundle.resources.RangerResource; import io.appform.ranger.server.bundle.rotation.BirTask; import io.appform.ranger.server.bundle.rotation.OorTask; @@ -82,11 +83,14 @@ protected List> withLifecycleSignals(U configuration){ protected abstract List withHealthChecks(U configuration); + protected boolean withMetricsEnabled(U configuration){ + return true; + } + + @Override public void initialize(Bootstrap bootstrap) { - /* - Nothing to init here! - */ + } @@ -101,6 +105,10 @@ public void run(U configuration, Environment environment) { val lifecycleSignals = withLifecycleSignals(configuration); val healthChecks = withHealthChecks(configuration); + if(withMetricsEnabled(configuration)){ + MetricRecorder.initialize(environment.metrics()); + } + environment.admin() .addTask(new OorTask(rotationStatus)); environment.admin() diff --git a/ranger-server-dw5-bundle/src/main/java/io/appform/ranger/server/bundle/RangerServerBundle.java b/ranger-server-dw5-bundle/src/main/java/io/appform/ranger/server/bundle/RangerServerBundle.java index 92d17285..c6cbb026 100644 --- a/ranger-server-dw5-bundle/src/main/java/io/appform/ranger/server/bundle/RangerServerBundle.java +++ b/ranger-server-dw5-bundle/src/main/java/io/appform/ranger/server/bundle/RangerServerBundle.java @@ -20,6 +20,7 @@ import io.appform.ranger.client.RangerHubClient; import io.appform.ranger.core.model.ServiceRegistry; import io.appform.ranger.core.signals.Signal; +import io.appform.ranger.core.util.MetricRecorder; import io.appform.ranger.server.bundle.resources.RangerResource; import io.appform.ranger.server.bundle.rotation.BirTask; import io.appform.ranger.server.bundle.rotation.OorTask; @@ -81,15 +82,17 @@ protected List> withLifecycleSignals(U configuration) { return List.of(); } + protected boolean withMetricsEnabled(U configuration) { + return true; + } + protected abstract List> withHubs(U configuration); protected abstract List withHealthChecks(U configuration); @Override public void initialize(Bootstrap bootstrap) { - /* - Nothing to init here! - */ + } @@ -118,6 +121,10 @@ protected void configure() { val lifecycleSignals = withLifecycleSignals(configuration); val healthChecks = withHealthChecks(configuration); + if(withMetricsEnabled(configuration)){ + MetricRecorder.initialize(environment.metrics()); + } + environment.admin() .addTask(new OorTask(rotationStatus)); environment.admin() diff --git a/ranger-server/src/main/java/io/appform/ranger/server/App.java b/ranger-server/src/main/java/io/appform/ranger/server/App.java index 061f02ad..a7a42092 100644 --- a/ranger-server/src/main/java/io/appform/ranger/server/App.java +++ b/ranger-server/src/main/java/io/appform/ranger/server/App.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.databind.MapperFeature; import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.module.paramnames.ParameterNamesModule; +import io.appform.ranger.core.util.MetricRecorder; import io.appform.ranger.hub.server.bundle.RangerHubServerBundle; import io.appform.ranger.hub.server.bundle.configuration.RangerServerConfiguration; import io.dropwizard.core.Application; @@ -36,6 +37,7 @@ public class App extends Application { @Override public void initialize(Bootstrap bootstrap) { + MetricRecorder.initialize(bootstrap.getMetricRegistry()); bootstrap.addBundle(new RangerHubServerBundle<>() { @Override protected RangerServerConfiguration getRangerConfiguration(AppConfig configuration) { diff --git a/ranger-zk-client/src/main/java/io/appform/ranger/client/zk/AbstractRangerZKHubClient.java b/ranger-zk-client/src/main/java/io/appform/ranger/client/zk/AbstractRangerZKHubClient.java index 41872e14..68364789 100644 --- a/ranger-zk-client/src/main/java/io/appform/ranger/client/zk/AbstractRangerZKHubClient.java +++ b/ranger-zk-client/src/main/java/io/appform/ranger/client/zk/AbstractRangerZKHubClient.java @@ -53,8 +53,8 @@ protected ServiceFinderHub buildHub() { } @Override - protected ServiceDataSource getDefaultDataSource() { - return new ZkServiceDataSource(getNamespace(), connectionString, curatorFramework); + protected ServiceDataSource getDefaultDataSource(String metricId) { + return new ZkServiceDataSource(metricId, getNamespace(), connectionString, curatorFramework); } } diff --git a/ranger-zk-client/src/main/java/io/appform/ranger/client/zk/SimpleRangerZKClient.java b/ranger-zk-client/src/main/java/io/appform/ranger/client/zk/SimpleRangerZKClient.java index 1ba4a7fd..a549ab65 100644 --- a/ranger-zk-client/src/main/java/io/appform/ranger/client/zk/SimpleRangerZKClient.java +++ b/ranger-zk-client/src/main/java/io/appform/ranger/client/zk/SimpleRangerZKClient.java @@ -39,6 +39,7 @@ @SuperBuilder public class SimpleRangerZKClient extends AbstractRangerClient> { + private String metricId; private final String serviceName; private final String namespace; private final ObjectMapper mapper; @@ -55,6 +56,7 @@ public class SimpleRangerZKClient extends AbstractRangerClientshardedFinderBuilder() + .withMetricId(metricId) .withCuratorFramework(curatorFramework) .withNamespace(namespace) .withServiceName(serviceName) diff --git a/ranger-zk-client/src/test/java/io/appform/ranger/client/zk/ShardedZKRangerClientTest.java b/ranger-zk-client/src/test/java/io/appform/ranger/client/zk/ShardedZKRangerClientTest.java index 48278988..c016d782 100644 --- a/ranger-zk-client/src/test/java/io/appform/ranger/client/zk/ShardedZKRangerClientTest.java +++ b/ranger-zk-client/src/test/java/io/appform/ranger/client/zk/ShardedZKRangerClientTest.java @@ -28,6 +28,7 @@ class ShardedZKRangerClientTest extends BaseRangerZKClientTest { @Test void testShardedHub(){ val zkHubClient = ShardedRangerZKHubClient.builder() + .metricId("testzk") .namespace("test-n") .connectionString(getTestingCluster().getConnectString()) .curatorFramework(getCuratorFramework()) diff --git a/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/common/ZkNodeDataStoreConnector.java b/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/common/ZkNodeDataStoreConnector.java index 4208fbd2..db5f7fd4 100644 --- a/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/common/ZkNodeDataStoreConnector.java +++ b/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/common/ZkNodeDataStoreConnector.java @@ -18,6 +18,7 @@ import dev.failsafe.Failsafe; import dev.failsafe.Fallback; import dev.failsafe.RetryPolicy; +import io.appform.ranger.core.model.DataStoreType; import io.appform.ranger.core.model.NodeDataStoreConnector; import io.appform.ranger.core.model.Service; import io.appform.ranger.core.util.Exceptions; @@ -38,6 +39,7 @@ @Slf4j public class ZkNodeDataStoreConnector implements NodeDataStoreConnector { + protected final String metricId; @Getter(AccessLevel.PROTECTED) protected final Service service; @Getter(AccessLevel.PROTECTED) @@ -62,9 +64,10 @@ public class ZkNodeDataStoreConnector implements NodeDataStoreConnector { .build(); protected ZkNodeDataStoreConnector( - final Service service, + String metricId, final Service service, final CuratorFramework curatorFramework, final ZkStoreType storeType) { + this.metricId = metricId; this.service = service; this.curatorFramework = curatorFramework; this.storeType = storeType; @@ -165,7 +168,7 @@ public void stop() { public boolean isActive() { var zkConnectionActive = curatorFramework != null && curatorFramework.getZookeeperClient() != null && curatorFramework.getZookeeperClient().isConnected(); - MetricRecorder.recordZkConnection(zkConnectionActive); + MetricRecorder.recordNoteDataSourceStatus(DataStoreType.ZK, metricId, zkConnectionActive); return zkConnectionActive; } diff --git a/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinder/ZkNodeDataSource.java b/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinder/ZkNodeDataSource.java index 34e53857..b50b590f 100644 --- a/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinder/ZkNodeDataSource.java +++ b/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinder/ZkNodeDataSource.java @@ -15,7 +15,9 @@ */ package io.appform.ranger.zookeeper.servicefinder; +import com.fasterxml.jackson.core.JsonProcessingException; import io.appform.functionmetrics.MonitoredFunction; +import io.appform.ranger.core.model.DataStoreType; import io.appform.ranger.core.model.NodeDataSource; import io.appform.ranger.core.model.Service; import io.appform.ranger.core.model.ServiceNode; @@ -35,6 +37,7 @@ import java.util.List; import java.util.Optional; +import static io.appform.ranger.core.util.MetricRecorder.LIST_NODES; import static java.util.Objects.requireNonNull; /** @@ -44,9 +47,19 @@ public class ZkNodeDataSource> extends ZkNodeDataStoreConnector implements NodeDataSource { public ZkNodeDataSource( - Service service, + String metricId, Service service, CuratorFramework curatorFramework) { - super(service, curatorFramework, ZkStoreType.SOURCE); + super(metricId, service, curatorFramework, ZkStoreType.SOURCE); + } + + @Override + public String getMetricId() { + return metricId; + } + + @Override + public DataStoreType getDataStoreType() { + return DataStoreType.ZK; } @Override @@ -67,8 +80,8 @@ private Optional>> checkForUpdateOnZookeeper(D deserializer) return Optional.empty(); } requireNonNull(deserializer, "Deserializer has not been set for node data"); + val serviceName = service.getServiceName(); try { - val serviceName = service.getServiceName(); if (!isActive()) { log.warn("ZK connection is not active. Ignoring refresh request for service: {}", service.getServiceName()); @@ -79,42 +92,60 @@ private Optional>> checkForUpdateOnZookeeper(D deserializer) val children = curatorFramework.getChildren().forPath(parentPath); List> nodes = new ArrayList<>(children.size()); log.debug("Found {} nodes for [{}]", children.size(), serviceName); + if(children.isEmpty()){ + MetricRecorder.recordNullOrEmptyListNodeResponse(DataStoreType.ZK, metricId, serviceName); + } for (val child : children) { - byte[] data = readChild(parentPath, child).orElse(null); + byte[] data = readChild(serviceName, parentPath, child).orElse(null); if (data == null || data.length == 0) { continue; } - val node = deserializer.deserialize(data); + final var node = parseServiceNodeData(serviceName, deserializer, data); nodes.add(node); } return Optional.of(nodes); } catch (NoNodeException e) { + MetricRecorder.recordNullOrEmptyListNodeResponse(DataStoreType.ZK, metricId, serviceName); log.error( "No ZK container node found for service: {}. Will return empty list for now. Please doublecheck service name", service.getServiceName()); return Optional.of(Collections.emptyList()); } catch (Exception e) { + MetricRecorder.recordZookeeperReadUnknownFailure(DataStoreType.ZK, metricId, LIST_NODES, e.getClass().getSimpleName()); log.error("Error getting node data from zookeeper: ", e); throw new ZkCommunicationException("Error getting node data from zookeeper: exception %s , message: %s" .formatted(e.getClass().getSimpleName(), e.getMessage())); } } - private Optional readChild(String parentPath, String child) throws Exception { + private > ServiceNode parseServiceNodeData(String serviceName, D deserializer, byte[] data) { + try { + return deserializer.deserialize(data); + } catch (Exception e) { + MetricRecorder.recordListNodesParseFailure(DataStoreType.ZK, metricId, serviceName); + log.error("Error deserializing node data : {} for service name: {} ", new String(data), serviceName, e); + throw e; + } + } + + private Optional readChild(String serviceName, String parentPath, String child) throws Exception { final String path = String.format("%s/%s", parentPath, child); try { return Optional.ofNullable(curatorFramework.getData().forPath(path)); } catch (KeeperException.NoNodeException e) { + MetricRecorder.recordNullOrEmptyListNodeResponse(DataStoreType.ZK, metricId, serviceName); log.warn("Node not found for path {}", path); return Optional.empty(); } catch (KeeperException e) { + MetricRecorder.recordNullOrEmptyListNodeResponse(DataStoreType.ZK, metricId, serviceName); log.error("Could not get data for node: {}", path, e); return Optional.empty(); } catch (Exception e){ + MetricRecorder.recordZookeeperReadUnknownFailure(DataStoreType.ZK, metricId, LIST_NODES, e.getClass().getSimpleName()); log.error("Could not read child for node: {}", path, e); throw e; } diff --git a/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinder/ZkSimpleShardedServiceFinderBuilder.java b/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinder/ZkSimpleShardedServiceFinderBuilder.java index cac3c738..293fcb5d 100644 --- a/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinder/ZkSimpleShardedServiceFinderBuilder.java +++ b/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinder/ZkSimpleShardedServiceFinderBuilder.java @@ -38,9 +38,11 @@ */ @Slf4j public class ZkSimpleShardedServiceFinderBuilder extends SimpleShardedServiceFinderBuilder, ZkNodeDataDeserializer> { + protected CuratorFramework curatorFramework; protected String connectionString; + public ZkSimpleShardedServiceFinderBuilder withCuratorFramework(CuratorFramework curatorFramework) { this.curatorFramework = curatorFramework; return this; @@ -67,8 +69,8 @@ public SimpleShardedServiceFinder build() { } @Override - protected NodeDataSource> dataSource(Service service) { - return new ZkNodeDataSource<>(service, curatorFramework); + protected NodeDataSource> dataSource(String metricId, Service service) { + return new ZkNodeDataSource<>(metricId, service, curatorFramework); } @Override diff --git a/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinder/ZkSimpleUnshardedServiceFinderBuilder.java b/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinder/ZkSimpleUnshardedServiceFinderBuilder.java index cd3cf6b5..e4abc7c9 100644 --- a/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinder/ZkSimpleUnshardedServiceFinderBuilder.java +++ b/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinder/ZkSimpleUnshardedServiceFinderBuilder.java @@ -69,8 +69,8 @@ public SimpleUnshardedServiceFinder build() { @Override protected NodeDataSource> dataSource( - Service service) { - return new ZkNodeDataSource<>(service, curatorFramework); + String metricId, Service service) { + return new ZkNodeDataSource<>(metricId, service, curatorFramework); } diff --git a/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinderhub/ZkServiceDataSource.java b/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinderhub/ZkServiceDataSource.java index 5437d2f9..f35f4e15 100644 --- a/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinderhub/ZkServiceDataSource.java +++ b/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinderhub/ZkServiceDataSource.java @@ -18,6 +18,8 @@ import io.appform.functionmetrics.MonitoredFunction; import io.appform.ranger.core.finderhub.ServiceDataSource; import io.appform.ranger.core.model.Service; +import io.appform.ranger.core.model.DataStoreType; +import io.appform.ranger.core.util.MetricRecorder; import io.appform.ranger.zookeeper.util.PathBuilder; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; @@ -30,6 +32,7 @@ import java.util.Collections; import java.util.stream.Collectors; +import static io.appform.ranger.core.util.MetricRecorder.*; import static java.util.Objects.requireNonNull; /** @@ -38,14 +41,17 @@ @Slf4j public class ZkServiceDataSource implements ServiceDataSource { + private final String metricId; private final String namespace; private final String connectionString; private CuratorFramework curatorFramework; private boolean curatorProvided; - public ZkServiceDataSource(String namespace, + public ZkServiceDataSource(String metricId, + String namespace, String connectionString, CuratorFramework curatorFramework){ + this.metricId = metricId; this.namespace = namespace; this.connectionString = connectionString; this.curatorFramework = curatorFramework; @@ -55,12 +61,27 @@ public ZkServiceDataSource(String namespace, @SneakyThrows @MonitoredFunction public Collection services() { - val children = curatorFramework.getChildren() - .forPath(PathBuilder.REGISTERED_SERVICES_PATH); - return null == children ? Collections.emptySet() : - children.stream() - .map(child -> Service.builder().namespace(namespace).serviceName(child).build()) - .collect(Collectors.toSet()); + try { + val children = curatorFramework.getChildren() + .forPath(PathBuilder.REGISTERED_SERVICES_PATH); + + if(children == null || children.isEmpty()) { + MetricRecorder.recordNullOrEmptyServicesListResponse(DataStoreType.ZK, metricId); + log.warn("No services found for namespace: {} in zk data source with metric id: {}", namespace, metricId); + } + val result = null == children + ? Collections.emptySet() + : children.stream() + .map(child -> Service.builder().namespace(namespace).serviceName(child).build()) + .collect(Collectors.toSet()); + MetricRecorder.recordServicesFetchStatus(DataStoreType.ZK, metricId, SUCCESS); + return result; + } + catch (Exception e) { + MetricRecorder.recordZookeeperReadUnknownFailure(DataStoreType.ZK, metricId, SERVICES_LIST, e.getClass().getSimpleName()); + MetricRecorder.recordServicesFetchStatus(DataStoreType.ZK, metricId, FAILURE); + throw e; + } } @Override diff --git a/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/serviceprovider/ZkNodeDataSink.java b/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/serviceprovider/ZkNodeDataSink.java index 8b982271..39abf3c3 100644 --- a/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/serviceprovider/ZkNodeDataSink.java +++ b/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/serviceprovider/ZkNodeDataSink.java @@ -16,10 +16,12 @@ package io.appform.ranger.zookeeper.serviceprovider; import io.appform.functionmetrics.MonitoredFunction; +import io.appform.ranger.core.model.DataStoreType; import io.appform.ranger.core.model.NodeDataSink; import io.appform.ranger.core.model.Service; import io.appform.ranger.core.model.ServiceNode; import io.appform.ranger.core.util.Exceptions; +import io.appform.ranger.core.util.MetricRecorder; import io.appform.ranger.zookeeper.common.ZkNodeDataStoreConnector; import io.appform.ranger.zookeeper.common.ZkStoreType; import io.appform.ranger.zookeeper.serde.ZkNodeDataSerializer; @@ -30,6 +32,8 @@ import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; +import static io.appform.ranger.core.util.MetricRecorder.FAILURE; +import static io.appform.ranger.core.util.MetricRecorder.SUCCESS; import static java.util.Objects.requireNonNull; /** @@ -38,9 +42,9 @@ @Slf4j public class ZkNodeDataSink> extends ZkNodeDataStoreConnector implements NodeDataSink { public ZkNodeDataSink( - Service service, + String metricId, Service service, CuratorFramework curatorFramework) { - super(service, curatorFramework, ZkStoreType.SINK); + super(metricId, service, curatorFramework, ZkStoreType.SINK); } @Override @@ -56,20 +60,32 @@ public void updateState(S serializer, ServiceNode serviceNode) { try { if (null == curatorFramework.checkExists().forPath(path)) { log.info("No node exists for path: {}. Will create now.", path); - createPath(serviceNode, serializer); + createPath(service.getServiceName(), serviceNode, serializer); } else { - curatorFramework.setData().forPath(path, serializer.serialize(serviceNode)); + val serviceData = getSerializedData(service.getServiceName(), serializer, serviceNode); + curatorFramework.setData().forPath(path, serviceData); } + MetricRecorder.recordNodeDataSinkUpdateStatus(DataStoreType.ZK, metricId, SUCCESS); } catch (Exception e) { log.error("Error updating node data at path " + path, e); + MetricRecorder.recordNodeDataSinkUpdateStatus(DataStoreType.ZK, metricId, FAILURE); Exceptions.illegalState(e); } } + private > byte[] getSerializedData(String serviceName, S serializer, ServiceNode serviceNode) { + try { + return serializer.serialize(serviceNode); + } catch (Exception e) { + MetricRecorder.recordNodeDataSinkSerDeFailure(DataStoreType.ZK, metricId, MetricRecorder.SERIALIZAION, serviceName, e.getClass().getSimpleName()); + throw e; + } + } + private synchronized void createPath( - ServiceNode serviceNode, + String serviceName, ServiceNode serviceNode, S serializer) { val instancePath = PathBuilder.instancePath(service, serviceNode); try { @@ -77,7 +93,7 @@ private synchronized void createPath( curatorFramework.create() .creatingParentContainersIfNeeded() .withMode(CreateMode.EPHEMERAL) - .forPath(instancePath, serializer.serialize(serviceNode)); + .forPath(instancePath, getSerializedData(serviceName, serializer, serviceNode)); log.info("Created instance path: {}", instancePath); } } @@ -85,6 +101,7 @@ private synchronized void createPath( log.warn("Node already exists.. Race condition?", e); } catch (Exception e) { + MetricRecorder.recordNodeDataSinkUnknownFailure(DataStoreType.ZK, metricId, service.getServiceName(), e.getClass().getSimpleName()); val message = String.format( "Could not create node for %s after 60 retries (1 min). " + "This service will not be discoverable. Retry after some time.", service.getServiceName()); diff --git a/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/serviceprovider/ZkServiceProviderBuilder.java b/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/serviceprovider/ZkServiceProviderBuilder.java index 16824194..cbbfaa25 100644 --- a/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/serviceprovider/ZkServiceProviderBuilder.java +++ b/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/serviceprovider/ZkServiceProviderBuilder.java @@ -61,7 +61,7 @@ public ServiceProvider> build() { } @Override - protected NodeDataSink> dataSink(Service service) { - return new ZkNodeDataSink<>(service, curatorFramework); + protected NodeDataSink> dataSink(String metricId, Service service) { + return new ZkNodeDataSink<>(metricId, service, curatorFramework); } } diff --git a/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/servicehub/ServiceHubTest.java b/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/servicehub/ServiceHubTest.java index ffc26164..843f09a2 100644 --- a/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/servicehub/ServiceHubTest.java +++ b/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/servicehub/ServiceHubTest.java @@ -106,7 +106,7 @@ void testHub() { .withCuratorFramework(curatorFramework) .withNamespace("test") .withRefreshFrequencyMs(1000) - .withServiceDataSource(new ZkServiceDataSource("test", testingCluster.getConnectString(), curatorFramework)) + .withServiceDataSource(new ZkServiceDataSource(null, "test", testingCluster.getConnectString(), curatorFramework)) .withServiceFinderFactory(ZkShardedServiceFinderFactory.builder() .curatorFramework(curatorFramework) .deserializer(this::read) From 7991b116b1920296a4dedcb732138084525a87d7 Mon Sep 17 00:00:00 2001 From: Jitendra Dhawan Date: Sun, 31 May 2026 18:31:35 +0530 Subject: [PATCH 08/24] Fix the unit tests --- .../stubs/TestServiceFinderFactory.java | 1 + .../client/utils/RangerHubTestUtils.java | 2 ++ ranger-core/pom.xml | 7 ++++- .../BaseServiceProviderBuilder.java | 1 + .../core/finderhub/ServiceFinderHubTest.java | 29 ++++++++++++++++- .../serviceprovider/ServiceProviderTest.java | 2 ++ .../bundle/ServiceDiscoveryBundle.java | 31 ++++++++++--------- ...viceDiscoveryBundleCustomHostPortTest.java | 1 + .../ServiceDiscoveryBundleDwMonitorTest.java | 1 + ...DiscoveryBundleDwStalenessMonitorTest.java | 1 + ...scoveryBundleHierarchicalSelectorTest.java | 1 + ...rviceDiscoveryBundleLocalHostPortTest.java | 6 ++++ .../ServiceDiscoveryBundleRotationTest.java | 1 + .../bundle/ServiceDiscoveryBundleTest.java | 1 + .../bundle/ServiceDiscoveryBundle.java | 17 +++++++++- ...viceDiscoveryBundleCustomHostPortTest.java | 1 + .../ServiceDiscoveryBundleDwMonitorTest.java | 1 + ...DiscoveryBundleDwStalenessMonitorTest.java | 1 + ...scoveryBundleHierarchicalSelectorTest.java | 1 + ...rviceDiscoveryBundleLocalHostPortTest.java | 6 ++++ .../ServiceDiscoveryBundleRotationTest.java | 1 + .../bundle/ServiceDiscoveryBundleTest.java | 1 + .../drove/BaseRangerDroveClientTest.java | 1 + .../drove/ShardedRangerDroveClientTest.java | 1 + .../drove/UnshardedRangerDroveClientTest.java | 1 + .../DroveShardedServiceFinderBuilderTest.java | 1 + .../client/http/BaseRangerHttpClientTest.java | 1 + .../http/ShardedRangerHttpClientTest.java | 1 + .../http/UnshardedRangerHttpClientTest.java | 1 + .../HttpShardedServiceFinderBuilderTest.java | 1 + ...HttpShardedServiceProviderBuilderTest.java | 1 + .../server/bundle/RangerHubServerBundle.java | 3 ++ .../bundle/RangerHubServerBundleTest.java | 3 ++ .../server/bundle/RangerHubServerBundle.java | 3 ++ .../bundle/RangerHubServerBundleTest.java | 13 +++++--- ...r.id.IdGeneratorPerfTest.testGenerate.json | 2 +- ...dGeneratorPerfTest.testGenerateBase36.json | 2 +- .../client/zk/ShardedRangerZKHubClient.java | 1 + .../client/zk/UnshardedRangerZKHubClient.java | 1 + .../client/zk/BaseRangerZKClientTest.java | 1 + .../client/zk/SimpleRangerZKClientTest.java | 1 + .../zk/UnshardedZKRangerClientTest.java | 1 + .../ZKUnshardedServiceFinderFactory.java | 6 +++- .../ZkShardedServiceFinderFactory.java | 6 +++- .../ServiceProviderIntegrationTest.java | 2 ++ .../model/CustomShardSelectorTest.java | 2 ++ .../model/ServiceNoProviderTest.java | 2 ++ .../model/ServiceProviderExtCuratorTest.java | 2 ++ .../model/ServiceProviderHealthcheckTest.java | 2 ++ .../zookeeper/model/ServiceProviderTest.java | 4 +++ .../model/SimpleServiceProviderTest.java | 2 ++ .../zookeeper/servicehub/ServiceHubTest.java | 2 ++ .../BaseServiceProviderBuilderTest.java | 2 ++ 53 files changed, 160 insertions(+), 26 deletions(-) diff --git a/ranger-client/src/test/java/io/appform/ranger/client/stubs/TestServiceFinderFactory.java b/ranger-client/src/test/java/io/appform/ranger/client/stubs/TestServiceFinderFactory.java index 24e302c5..0d65a53e 100644 --- a/ranger-client/src/test/java/io/appform/ranger/client/stubs/TestServiceFinderFactory.java +++ b/ranger-client/src/test/java/io/appform/ranger/client/stubs/TestServiceFinderFactory.java @@ -28,6 +28,7 @@ public class TestServiceFinderFactory implements ServiceFinderFactory> buildFinder(Service service) { val finder = new TestSimpleUnshardedServiceFinder() + .withMetricId("test-metric") .withNamespace(service.getNamespace()) .withServiceName(service.getServiceName()) .withDeserializer(new Deserializer<>() { diff --git a/ranger-client/src/test/java/io/appform/ranger/client/utils/RangerHubTestUtils.java b/ranger-client/src/test/java/io/appform/ranger/client/utils/RangerHubTestUtils.java index 6c6dcb0f..2f037a82 100644 --- a/ranger-client/src/test/java/io/appform/ranger/client/utils/RangerHubTestUtils.java +++ b/ranger-client/src/test/java/io/appform/ranger/client/utils/RangerHubTestUtils.java @@ -39,6 +39,7 @@ public static RangerTestHub getTestHub(){ .nodeRefreshTimeMs(1000) .initialCriteria(new TestCriteria()) .deserializer(new TestDeserializer<>()) + .metricId("test-metric") .build(); } @@ -51,6 +52,7 @@ public static RangerTestHub getTestHubWithDataSource(){ .useDefaultDataSource(false) .serviceDataSource(new StaticDataSource(Set.of(RangerHubTestUtils.service))) .deserializer(new TestDeserializer<>()) + .metricId("test-metric") .build(); } diff --git a/ranger-core/pom.xml b/ranger-core/pom.xml index af87a831..2b66747b 100644 --- a/ranger-core/pom.xml +++ b/ranger-core/pom.xml @@ -51,12 +51,17 @@ io.dropwizard dropwizard-metrics ${dropwizard.version} - provided io.appform.functionmetrics function-metrics + + org.mockito + mockito-core + ${mockito.version} + test + diff --git a/ranger-core/src/main/java/io/appform/ranger/core/serviceprovider/BaseServiceProviderBuilder.java b/ranger-core/src/main/java/io/appform/ranger/core/serviceprovider/BaseServiceProviderBuilder.java index 9ab4abc2..90d9861f 100644 --- a/ranger-core/src/main/java/io/appform/ranger/core/serviceprovider/BaseServiceProviderBuilder.java +++ b/ranger-core/src/main/java/io/appform/ranger/core/serviceprovider/BaseServiceProviderBuilder.java @@ -18,6 +18,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; import io.appform.ranger.core.healthcheck.HealthChecker; import io.appform.ranger.core.healthcheck.Healthcheck; import io.appform.ranger.core.healthcheck.HealthcheckResult; diff --git a/ranger-core/src/test/java/io/appform/ranger/core/finderhub/ServiceFinderHubTest.java b/ranger-core/src/test/java/io/appform/ranger/core/finderhub/ServiceFinderHubTest.java index f80e3b24..29fa77c7 100644 --- a/ranger-core/src/test/java/io/appform/ranger/core/finderhub/ServiceFinderHubTest.java +++ b/ranger-core/src/test/java/io/appform/ranger/core/finderhub/ServiceFinderHubTest.java @@ -47,6 +47,7 @@ class ServiceFinderHubTest { new DynamicDataSource(Lists.newArrayList(new Service("NS", "PRE_REGISTERED_SERVICE"))), service -> new TestServiceFinderBuilder() + .withMetricId("test-metric") .withNamespace(service.getNamespace()) .withServiceName(service.getServiceName()) .withDeserializer(new Deserializer() { @@ -93,6 +94,7 @@ void testTimeoutOnHubStartup() { void testDelayedServiceAddition() { val delayedHub = new ServiceFinderHub<>(new DynamicDataSource(Lists.newArrayList(new Service("NS", "SERVICE"))), service -> new TestServiceFinderBuilder() + .withMetricId("test-metric") .withNamespace(service.getNamespace()) .withServiceName(service.getServiceName()) .withDeserializer(new Deserializer() {}) @@ -101,6 +103,7 @@ void testDelayedServiceAddition() { Assertions.assertThrows(IllegalStateException.class, delayedHub::start); val serviceFinderHub = new ServiceFinderHub<>(new DynamicDataSource(Lists.newArrayList(new Service("NS", "SERVICE"))), service -> new TestServiceFinderBuilder() + .withMetricId("test-metric") .withNamespace(service.getNamespace()) .withServiceName(service.getServiceName()) .withDeserializer(new Deserializer() {}) @@ -114,6 +117,7 @@ void testDelayedServiceAddition() { @Test void testDynamicServiceAdditionWithNonDynamicDataSource() { val serviceFinderHub = new ServiceFinderHub<>(new StaticDataSource(new HashSet<>()), service -> new TestServiceFinderBuilder() + .withMetricId("test-metric") .withNamespace(service.getNamespace()) .withServiceName(service.getServiceName()) .withDeserializer(new Deserializer() { @@ -135,6 +139,7 @@ void testWeightedNodeSelectionWithVaryingWeights() { new DynamicDataSource(Lists.newArrayList(new Service("NS", "PRE_REGISTERED_SERVICE"))), service -> new TestServiceFinderBuilder() + .withMetricId("test-metric") .withNamespace(service.getNamespace()) .withServiceName(service.getServiceName()) .withNodeSelector(new WeightedRandomServiceNodeSelector<>( @@ -146,6 +151,16 @@ void testWeightedNodeSelectionWithVaryingWeights() { .withDeserializer(new Deserializer() { }) .withDataSource(new NodeDataSource<>() { + @Override + public String getMetricId() { + return "testVaryingWeights"; + } + + @Override + public DataStoreType getDataStoreType() { + return DataStoreType.HTTP; + } + @Override public Optional>> refresh( final Deserializer deserializer) @@ -229,6 +244,7 @@ void testWeightedNodeSelectionWithVaryingNodeAge() { new DynamicDataSource(Lists.newArrayList(new Service("NS", "PRE_REGISTERED_SERVICE"))), service -> new TestServiceFinderBuilder() + .withMetricId("test-metric") .withNamespace(service.getNamespace()) .withServiceName(service.getServiceName()) .withNodeSelector(new WeightedRandomServiceNodeSelector<>( @@ -240,6 +256,16 @@ void testWeightedNodeSelectionWithVaryingNodeAge() { .withDeserializer(new Deserializer() { }) .withDataSource(new NodeDataSource<>() { + @Override + public String getMetricId() { + return "testVaryingNodeAge"; + } + + @Override + public DataStoreType getDataStoreType() { + return DataStoreType.HTTP; + } + @Override public Optional>> refresh( final Deserializer deserializer) @@ -323,6 +349,7 @@ public class TestServiceFinderFactory implements ServiceFinderFactory> buildFinder(Service service) { val finder = new TestServiceFinderBuilder() + .withMetricId("test-metric") .withNamespace(service.getNamespace()) .withServiceName(service.getServiceName()) .withDeserializer(new Deserializer() {}) @@ -359,7 +386,7 @@ public ServiceFinder> build( } @Override - protected NodeDataSource> dataSource(Service service) { + protected NodeDataSource> dataSource(String metricId, Service service) { return testNodeDataSource; } diff --git a/ranger-core/src/test/java/io/appform/ranger/core/serviceprovider/ServiceProviderTest.java b/ranger-core/src/test/java/io/appform/ranger/core/serviceprovider/ServiceProviderTest.java index 8d675ed8..b0c1cae6 100644 --- a/ranger-core/src/test/java/io/appform/ranger/core/serviceprovider/ServiceProviderTest.java +++ b/ranger-core/src/test/java/io/appform/ranger/core/serviceprovider/ServiceProviderTest.java @@ -116,6 +116,7 @@ void testInvalidServiceProviderNoHealthCheck() { final HealthUpdateHandler healthUpdateHandler = new LastUpdatedHandler() .setNext(new HealthStatusHandler()); Assertions.assertThrowsExactly(IllegalArgumentException.class, () -> new TestServiceProviderBuilder<>() + .withMetricId("test-metric") .withServiceName("test-service") .withNamespace("test") .withHostname("localhost-1") @@ -142,6 +143,7 @@ void testBuildServiceProvider() { final HealthUpdateHandler healthUpdateHandler = new LastUpdatedHandler() .setNext(new HealthStatusHandler()); val testProvider = new TestServiceProviderBuilder<>() + .withMetricId("test-metric") .withServiceName("test-service") .withNamespace("test") .withHostname("localhost-1") diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundle.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundle.java index 59c7fda7..a4c627b8 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundle.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundle.java @@ -39,24 +39,26 @@ import io.appform.ranger.core.model.ServiceNodeSelector; import io.appform.ranger.core.model.ShardSelector; import io.appform.ranger.core.serviceprovider.ServiceProvider; -import io.appform.ranger.discovery.bundle.healthchecks.InitialDelayChecker; -import io.appform.ranger.discovery.bundle.healthchecks.InternalHealthChecker; -import io.appform.ranger.discovery.bundle.healthchecks.RotationCheck; -import io.appform.ranger.discovery.bundle.id.IdGenerator; -import io.appform.ranger.discovery.bundle.id.NodeIdManager; -import io.appform.ranger.discovery.bundle.id.constraints.IdValidationConstraint; -import io.appform.ranger.discovery.bundle.monitors.DropwizardHealthMonitor; +import io.appform.ranger.core.util.MetricRecorder; +import io.appform.ranger.discovery.core.ServiceDiscoveryConfiguration; +import io.appform.ranger.discovery.core.healthchecks.InitialDelayChecker; +import io.appform.ranger.discovery.core.healthchecks.InternalHealthChecker; +import io.appform.ranger.discovery.core.healthchecks.RotationCheck; +import io.appform.ranger.id.IdGenerator; +import io.appform.ranger.id.NodeIdManager; +import io.appform.ranger.id.constraints.IdValidationConstraint; +import io.appform.ranger.discovery.core.monitors.DropwizardHealthMonitor; import io.appform.ranger.discovery.bundle.monitors.DropwizardServerStartupCheck; -import io.appform.ranger.discovery.bundle.resolvers.DefaultNodeInfoResolver; +import io.appform.ranger.discovery.core.resolvers.DefaultNodeInfoResolver; import io.appform.ranger.discovery.bundle.resolvers.DefaultPortSchemeResolver; -import io.appform.ranger.discovery.bundle.resolvers.NodeInfoResolver; +import io.appform.ranger.discovery.core.resolvers.NodeInfoResolver; import io.appform.ranger.discovery.bundle.resolvers.PortSchemeResolver; import io.appform.ranger.discovery.bundle.rotationstatus.BIRTask; -import io.appform.ranger.discovery.bundle.rotationstatus.DropwizardServerStatus; +import io.appform.ranger.discovery.core.rotationstatus.DropwizardServerStatus; import io.appform.ranger.discovery.bundle.rotationstatus.OORTask; -import io.appform.ranger.discovery.bundle.rotationstatus.RotationStatus; -import io.appform.ranger.discovery.bundle.selectors.HierarchicalEnvironmentAwareShardSelector; -import io.appform.ranger.discovery.bundle.util.ConfigurationUtils; +import io.appform.ranger.discovery.core.rotationstatus.RotationStatus; +import io.appform.ranger.discovery.core.selectors.HierarchicalEnvironmentAwareShardSelector; +import io.appform.ranger.discovery.core.util.ConfigurationUtils; import io.appform.ranger.zookeeper.ServiceProviderBuilders; import io.appform.ranger.zookeeper.serde.ZkNodeDataSerializer; import io.dropwizard.Configuration; @@ -258,6 +260,7 @@ private RangerClient> buildDiscove ShardSelector> shardSelector, final ServiceNodeSelector nodeSelector) { return SimpleRangerZKClient.builder() + .metricId(DEFAULT_DATA_SINK_ID) .curatorFramework(curator) .namespace(namespace) .serviceName(serviceName) @@ -325,7 +328,7 @@ private ServiceProvider> buildService .withHealthcheck(new DropwizardServerStartupCheck(environment, serverStatus)) .withIsolatedHealthMonitor(new DropwizardHealthMonitor( new TimeEntity(initialDelayForMonitor, dwMonitoringInterval, TimeUnit.SECONDS), - dwMonitoringStaleness * 1_000L, environment)) + dwMonitoringStaleness * 1_000L, environment.healthChecks())) .withHealthUpdateIntervalMs(serviceDiscoveryConfiguration.getRefreshTimeMs()) .withStaleUpdateThresholdMs(10000) .healthUpdateHandler(shardInfoHealthUpdateHandler); diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleCustomHostPortTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleCustomHostPortTest.java index 99ab3e86..c942313f 100644 --- a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleCustomHostPortTest.java +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleCustomHostPortTest.java @@ -98,6 +98,7 @@ void setup() throws Exception { when(environment.lifecycle()).thenReturn(lifecycleEnvironment); when(environment.healthChecks()).thenReturn(healthChecks); when(environment.getObjectMapper()).thenReturn(new ObjectMapper()); + when(environment.metrics()).thenReturn(metricRegistry); AdminEnvironment adminEnvironment = mock(AdminEnvironment.class); doNothing().when(adminEnvironment) .addTask(any()); diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleDwMonitorTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleDwMonitorTest.java index 7053bb65..7e377b97 100644 --- a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleDwMonitorTest.java +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleDwMonitorTest.java @@ -113,6 +113,7 @@ protected Result check() { when(environment.lifecycle()).thenReturn(lifecycleEnvironment); when(environment.healthChecks()).thenReturn(healthChecks); when(environment.getObjectMapper()).thenReturn(new ObjectMapper()); + when(environment.metrics()).thenReturn(metricRegistry); AdminEnvironment adminEnvironment = mock(AdminEnvironment.class); doNothing().when(adminEnvironment) .addTask(any()); diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleDwStalenessMonitorTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleDwStalenessMonitorTest.java index 62831615..4ce15359 100644 --- a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleDwStalenessMonitorTest.java +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleDwStalenessMonitorTest.java @@ -112,6 +112,7 @@ protected Result check() { when(environment.lifecycle()).thenReturn(lifecycleEnvironment); when(environment.healthChecks()).thenReturn(healthChecks); when(environment.getObjectMapper()).thenReturn(new ObjectMapper()); + when(environment.metrics()).thenReturn(metricRegistry); AdminEnvironment adminEnvironment = mock(AdminEnvironment.class); doNothing().when(adminEnvironment) .addTask(any()); diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleHierarchicalSelectorTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleHierarchicalSelectorTest.java index 2f6e2018..c5fb7a4e 100644 --- a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleHierarchicalSelectorTest.java +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleHierarchicalSelectorTest.java @@ -90,6 +90,7 @@ void setup() throws Exception { when(environment.lifecycle()).thenReturn(lifecycleEnvironment); when(environment.healthChecks()).thenReturn(healthChecks); when(environment.getObjectMapper()).thenReturn(new ObjectMapper()); + when(environment.metrics()).thenReturn(metricRegistry); AdminEnvironment adminEnvironment = mock(AdminEnvironment.class); doNothing().when(adminEnvironment) .addTask(any()); diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleLocalHostPortTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleLocalHostPortTest.java index c03d2c2e..ea9466f4 100644 --- a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleLocalHostPortTest.java +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleLocalHostPortTest.java @@ -93,6 +93,7 @@ void shouldFailLocalhostPublish() { when(environment.lifecycle()).thenReturn(lifecycleEnvironment); when(environment.healthChecks()).thenReturn(healthChecks); when(environment.getObjectMapper()).thenReturn(new ObjectMapper()); + when(environment.metrics()).thenReturn(metricRegistry); AdminEnvironment adminEnvironment = mock(AdminEnvironment.class); doNothing().when(adminEnvironment) .addTask(any()); @@ -121,6 +122,7 @@ void shouldThrowExceptionForInvalidZkHost() { when(environment.lifecycle()).thenReturn(lifecycleEnvironment); when(environment.healthChecks()).thenReturn(healthChecks); when(environment.getObjectMapper()).thenReturn(new ObjectMapper()); + when(environment.metrics()).thenReturn(metricRegistry); AdminEnvironment adminEnvironment = mock(AdminEnvironment.class); doNothing().when(adminEnvironment) .addTask(any()); @@ -158,6 +160,7 @@ void testPublishWithEmptyZkHost() throws UnknownHostException { when(environment.lifecycle()).thenReturn(lifecycleEnvironment); when(environment.healthChecks()).thenReturn(healthChecks); when(environment.getObjectMapper()).thenReturn(new ObjectMapper()); + when(environment.metrics()).thenReturn(metricRegistry); AdminEnvironment adminEnvironment = mock(AdminEnvironment.class); doNothing().when(adminEnvironment) .addTask(any()); @@ -194,6 +197,7 @@ void testPublishWithNullZkHost() throws UnknownHostException { when(environment.lifecycle()).thenReturn(lifecycleEnvironment); when(environment.healthChecks()).thenReturn(healthChecks); when(environment.getObjectMapper()).thenReturn(new ObjectMapper()); + when(environment.metrics()).thenReturn(metricRegistry); AdminEnvironment adminEnvironment = mock(AdminEnvironment.class); doNothing().when(adminEnvironment) .addTask(any()); @@ -229,6 +233,7 @@ void shouldPublishingToLocalZk() { when(environment.lifecycle()).thenReturn(lifecycleEnvironment); when(environment.healthChecks()).thenReturn(healthChecks); when(environment.getObjectMapper()).thenReturn(new ObjectMapper()); + when(environment.metrics()).thenReturn(metricRegistry); AdminEnvironment adminEnvironment = mock(AdminEnvironment.class); doNothing().when(adminEnvironment) .addTask(any()); @@ -259,6 +264,7 @@ void shouldPublishToRemoteZk() { when(environment.lifecycle()).thenReturn(lifecycleEnvironment); when(environment.healthChecks()).thenReturn(healthChecks); when(environment.getObjectMapper()).thenReturn(new ObjectMapper()); + when(environment.metrics()).thenReturn(metricRegistry); AdminEnvironment adminEnvironment = mock(AdminEnvironment.class); doNothing().when(adminEnvironment) .addTask(any()); diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleRotationTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleRotationTest.java index 0e9bb129..eb34c944 100644 --- a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleRotationTest.java +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleRotationTest.java @@ -93,6 +93,7 @@ void setup() throws Exception { when(environment.lifecycle()).thenReturn(lifecycleEnvironment); when(environment.healthChecks()).thenReturn(healthChecks); when(environment.getObjectMapper()).thenReturn(new ObjectMapper()); + when(environment.metrics()).thenReturn(metricRegistry); AdminEnvironment adminEnvironment = mock(AdminEnvironment.class); doNothing().when(adminEnvironment) .addTask(any()); diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleTest.java index 0011d63b..edabd5c6 100644 --- a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleTest.java +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleTest.java @@ -96,6 +96,7 @@ void setup() throws Exception { when(environment.lifecycle()).thenReturn(lifecycleEnvironment); when(environment.healthChecks()).thenReturn(healthChecks); when(environment.getObjectMapper()).thenReturn(new ObjectMapper()); + when(environment.metrics()).thenReturn(metricRegistry); AdminEnvironment adminEnvironment = mock(AdminEnvironment.class); doNothing().when(adminEnvironment) .addTask(any()); diff --git a/ranger-discovery-dw5-bundle/src/main/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundle.java b/ranger-discovery-dw5-bundle/src/main/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundle.java index 59224073..74a82782 100644 --- a/ranger-discovery-dw5-bundle/src/main/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundle.java +++ b/ranger-discovery-dw5-bundle/src/main/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundle.java @@ -27,6 +27,11 @@ import io.appform.ranger.core.finder.serviceregistry.MapBasedServiceRegistry; import io.appform.ranger.core.healthcheck.Healthcheck; import io.appform.ranger.core.healthcheck.HealthcheckStatus; +import io.appform.ranger.core.healthcheck.updater.HealthStatusHandler; +import io.appform.ranger.core.healthcheck.updater.HealthUpdateHandler; +import io.appform.ranger.core.healthcheck.updater.LastUpdatedHandler; +import io.appform.ranger.core.healthcheck.updater.RoutingWeightHandler; +import io.appform.ranger.core.healthcheck.updater.StartupTimeHandler; import io.appform.ranger.core.healthservice.TimeEntity; import io.appform.ranger.core.healthservice.monitor.IsolatedHealthMonitor; import io.appform.ranger.core.model.ServiceNode; @@ -74,6 +79,7 @@ import java.util.List; import java.util.concurrent.TimeUnit; import java.util.function.Predicate; +import java.util.function.Supplier; import java.util.stream.Collectors; import static io.appform.ranger.discovery.bundle.Constants.LOCAL_ADDRESSES; @@ -188,6 +194,10 @@ protected List> getHealthMonitors() { return Collections.emptyList(); } + protected Supplier getWeightSupplier() { + return () -> 1.0; + } + @SuppressWarnings("unused") protected int getPort(T configuration) { Preconditions.checkArgument(Constants.DEFAULT_PORT != serviceDiscoveryConfiguration.getPublishedPort() @@ -279,6 +289,10 @@ private ServiceProvider> buildService : serviceDiscoveryConfiguration.getDropwizardCheckInterval(); val dwMonitoringStaleness = Math.max(serviceDiscoveryConfiguration.getDropwizardCheckStaleness(), dwMonitoringInterval + 1); + final HealthUpdateHandler shardInfoHealthUpdateHandler = new LastUpdatedHandler() + .setNext(new HealthStatusHandler<>()) + .setNext(new RoutingWeightHandler<>(getWeightSupplier().get())) + .setNext(new StartupTimeHandler<>()); val serviceProviderBuilder = ServiceProviderBuilders.shardedServiceProviderBuilder() .withMetricId(DEFAULT_DATA_SINK_ID) .withCuratorFramework(curator) @@ -304,7 +318,8 @@ private ServiceProvider> buildService new TimeEntity(initialDelayForMonitor, dwMonitoringInterval, TimeUnit.SECONDS), dwMonitoringStaleness * 1_000L, environment.healthChecks())) .withHealthUpdateIntervalMs(serviceDiscoveryConfiguration.getRefreshTimeMs()) - .withStaleUpdateThresholdMs(10000); + .withStaleUpdateThresholdMs(10000) + .healthUpdateHandler(shardInfoHealthUpdateHandler); val healthMonitors = getHealthMonitors(); if (healthMonitors != null && !healthMonitors.isEmpty()) { diff --git a/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleCustomHostPortTest.java b/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleCustomHostPortTest.java index f1327239..127c165d 100644 --- a/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleCustomHostPortTest.java +++ b/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleCustomHostPortTest.java @@ -97,6 +97,7 @@ void setup() throws Exception { when(environment.lifecycle()).thenReturn(lifecycleEnvironment); when(environment.healthChecks()).thenReturn(healthChecks); when(environment.getObjectMapper()).thenReturn(new ObjectMapper()); + when(environment.metrics()).thenReturn(metricRegistry); AdminEnvironment adminEnvironment = mock(AdminEnvironment.class); doNothing().when(adminEnvironment) .addTask(any()); diff --git a/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleDwMonitorTest.java b/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleDwMonitorTest.java index 581d7acf..ab2abd9c 100644 --- a/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleDwMonitorTest.java +++ b/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleDwMonitorTest.java @@ -113,6 +113,7 @@ protected Result check() { when(environment.lifecycle()).thenReturn(lifecycleEnvironment); when(environment.healthChecks()).thenReturn(healthChecks); when(environment.getObjectMapper()).thenReturn(new ObjectMapper()); + when(environment.metrics()).thenReturn(metricRegistry); AdminEnvironment adminEnvironment = mock(AdminEnvironment.class); doNothing().when(adminEnvironment) .addTask(any()); diff --git a/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleDwStalenessMonitorTest.java b/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleDwStalenessMonitorTest.java index 9cfb0c52..11f73934 100644 --- a/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleDwStalenessMonitorTest.java +++ b/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleDwStalenessMonitorTest.java @@ -112,6 +112,7 @@ protected Result check() { when(environment.lifecycle()).thenReturn(lifecycleEnvironment); when(environment.healthChecks()).thenReturn(healthChecks); when(environment.getObjectMapper()).thenReturn(new ObjectMapper()); + when(environment.metrics()).thenReturn(metricRegistry); AdminEnvironment adminEnvironment = mock(AdminEnvironment.class); doNothing().when(adminEnvironment) .addTask(any()); diff --git a/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleHierarchicalSelectorTest.java b/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleHierarchicalSelectorTest.java index f61a10c8..4f8aa00c 100644 --- a/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleHierarchicalSelectorTest.java +++ b/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleHierarchicalSelectorTest.java @@ -89,6 +89,7 @@ void setup() throws Exception { when(environment.lifecycle()).thenReturn(lifecycleEnvironment); when(environment.healthChecks()).thenReturn(healthChecks); when(environment.getObjectMapper()).thenReturn(new ObjectMapper()); + when(environment.metrics()).thenReturn(metricRegistry); AdminEnvironment adminEnvironment = mock(AdminEnvironment.class); doNothing().when(adminEnvironment) .addTask(any()); diff --git a/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleLocalHostPortTest.java b/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleLocalHostPortTest.java index 0b360334..9d630408 100644 --- a/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleLocalHostPortTest.java +++ b/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleLocalHostPortTest.java @@ -93,6 +93,7 @@ void shouldFailLocalhostPublish() { when(environment.lifecycle()).thenReturn(lifecycleEnvironment); when(environment.healthChecks()).thenReturn(healthChecks); when(environment.getObjectMapper()).thenReturn(new ObjectMapper()); + when(environment.metrics()).thenReturn(metricRegistry); AdminEnvironment adminEnvironment = mock(AdminEnvironment.class); doNothing().when(adminEnvironment) .addTask(any()); @@ -122,6 +123,7 @@ void shouldThrowExceptionForInvalidZkHost() { when(environment.lifecycle()).thenReturn(lifecycleEnvironment); when(environment.healthChecks()).thenReturn(healthChecks); when(environment.getObjectMapper()).thenReturn(new ObjectMapper()); + when(environment.metrics()).thenReturn(metricRegistry); AdminEnvironment adminEnvironment = mock(AdminEnvironment.class); doNothing().when(adminEnvironment) .addTask(any()); @@ -159,6 +161,7 @@ void testPublishWithEmptyZkHost() throws UnknownHostException { when(environment.lifecycle()).thenReturn(lifecycleEnvironment); when(environment.healthChecks()).thenReturn(healthChecks); when(environment.getObjectMapper()).thenReturn(new ObjectMapper()); + when(environment.metrics()).thenReturn(metricRegistry); AdminEnvironment adminEnvironment = mock(AdminEnvironment.class); doNothing().when(adminEnvironment) .addTask(any()); @@ -196,6 +199,7 @@ void testPublishWithNullZkHost() throws UnknownHostException { when(environment.lifecycle()).thenReturn(lifecycleEnvironment); when(environment.healthChecks()).thenReturn(healthChecks); when(environment.getObjectMapper()).thenReturn(new ObjectMapper()); + when(environment.metrics()).thenReturn(metricRegistry); AdminEnvironment adminEnvironment = mock(AdminEnvironment.class); doNothing().when(adminEnvironment) .addTask(any()); @@ -232,6 +236,7 @@ void shouldPublishingToLocalZk() { when(environment.lifecycle()).thenReturn(lifecycleEnvironment); when(environment.healthChecks()).thenReturn(healthChecks); when(environment.getObjectMapper()).thenReturn(new ObjectMapper()); + when(environment.metrics()).thenReturn(metricRegistry); AdminEnvironment adminEnvironment = mock(AdminEnvironment.class); doNothing().when(adminEnvironment) .addTask(any()); @@ -263,6 +268,7 @@ void shouldPublishToRemoteZk() { when(environment.lifecycle()).thenReturn(lifecycleEnvironment); when(environment.healthChecks()).thenReturn(healthChecks); when(environment.getObjectMapper()).thenReturn(new ObjectMapper()); + when(environment.metrics()).thenReturn(metricRegistry); AdminEnvironment adminEnvironment = mock(AdminEnvironment.class); doNothing().when(adminEnvironment) .addTask(any()); diff --git a/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleRotationTest.java b/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleRotationTest.java index 8e4a62ff..5e8b5a4a 100644 --- a/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleRotationTest.java +++ b/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleRotationTest.java @@ -93,6 +93,7 @@ void setup() throws Exception { when(environment.lifecycle()).thenReturn(lifecycleEnvironment); when(environment.healthChecks()).thenReturn(healthChecks); when(environment.getObjectMapper()).thenReturn(new ObjectMapper()); + when(environment.metrics()).thenReturn(metricRegistry); AdminEnvironment adminEnvironment = mock(AdminEnvironment.class); doNothing().when(adminEnvironment) .addTask(any()); diff --git a/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleTest.java b/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleTest.java index 3587594e..9a848b46 100644 --- a/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleTest.java +++ b/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleTest.java @@ -96,6 +96,7 @@ void setup() throws Exception { when(environment.lifecycle()).thenReturn(lifecycleEnvironment); when(environment.healthChecks()).thenReturn(healthChecks); when(environment.getObjectMapper()).thenReturn(new ObjectMapper()); + when(environment.metrics()).thenReturn(metricRegistry); AdminEnvironment adminEnvironment = mock(AdminEnvironment.class); doNothing().when(adminEnvironment) .addTask(any()); diff --git a/ranger-drove-client/src/test/java/io/appform/ranger/client/drove/BaseRangerDroveClientTest.java b/ranger-drove-client/src/test/java/io/appform/ranger/client/drove/BaseRangerDroveClientTest.java index 1b655966..42701615 100644 --- a/ranger-drove-client/src/test/java/io/appform/ranger/client/drove/BaseRangerDroveClientTest.java +++ b/ranger-drove-client/src/test/java/io/appform/ranger/client/drove/BaseRangerDroveClientTest.java @@ -114,6 +114,7 @@ public void prepareHttpMocks() throws Exception { response)))); clientConfig = DroveUpstreamConfig.builder() + .id("test-metric") .endpoints(List.of("http://localhost:" + wireMockExtension.getPort())) .build(); log.debug("Started http subsystem. Wiremock port: {}", wireMockExtension.getPort()); diff --git a/ranger-drove-client/src/test/java/io/appform/ranger/client/drove/ShardedRangerDroveClientTest.java b/ranger-drove-client/src/test/java/io/appform/ranger/client/drove/ShardedRangerDroveClientTest.java index f458c268..faba7ad5 100644 --- a/ranger-drove-client/src/test/java/io/appform/ranger/client/drove/ShardedRangerDroveClientTest.java +++ b/ranger-drove-client/src/test/java/io/appform/ranger/client/drove/ShardedRangerDroveClientTest.java @@ -43,6 +43,7 @@ public TestNodeData translate(ExposedAppInfo appInfo, ExposedAppInfo.ExposedHost .namespace(namespace) .mapper(getObjectMapper()) .nodeRefreshTimeMs(1000) + .metricId("test-metric") .build(); client.start(); val service = RangerTestUtils.getService(namespace, "TEST_APP"); diff --git a/ranger-drove-client/src/test/java/io/appform/ranger/client/drove/UnshardedRangerDroveClientTest.java b/ranger-drove-client/src/test/java/io/appform/ranger/client/drove/UnshardedRangerDroveClientTest.java index 64019c98..a30949cd 100644 --- a/ranger-drove-client/src/test/java/io/appform/ranger/client/drove/UnshardedRangerDroveClientTest.java +++ b/ranger-drove-client/src/test/java/io/appform/ranger/client/drove/UnshardedRangerDroveClientTest.java @@ -42,6 +42,7 @@ public TestNodeData translate(ExposedAppInfo appInfo, ExposedAppInfo.ExposedHost }) .mapper(getObjectMapper()) .nodeRefreshTimeMs(1000) + .metricId("test-metric") .build(); client.start(); val service = RangerTestUtils.getService(namespace, "TEST_APP"); diff --git a/ranger-drove/src/test/java/io/appform/ranger/drove/servicefinder/DroveShardedServiceFinderBuilderTest.java b/ranger-drove/src/test/java/io/appform/ranger/drove/servicefinder/DroveShardedServiceFinderBuilderTest.java index 05d7f579..cd5cadb7 100644 --- a/ranger-drove/src/test/java/io/appform/ranger/drove/servicefinder/DroveShardedServiceFinderBuilderTest.java +++ b/ranger-drove/src/test/java/io/appform/ranger/drove/servicefinder/DroveShardedServiceFinderBuilderTest.java @@ -89,6 +89,7 @@ public NodeData translate(ExposedAppInfo appInfo, ExposedAppInfo.ExposedHost hos .withShardSelector((criteria, registry) -> registry.nodeList()) .withNodeRefreshIntervalMs(1000) .withDroveClient(droveClient) + .withMetricId("test-metric") .build(); finder.start(); await().atMost(Duration.ofSeconds(30)) diff --git a/ranger-http-client/src/test/java/io/appform/ranger/client/http/BaseRangerHttpClientTest.java b/ranger-http-client/src/test/java/io/appform/ranger/client/http/BaseRangerHttpClientTest.java index cd748f2f..f086c00f 100644 --- a/ranger-http-client/src/test/java/io/appform/ranger/client/http/BaseRangerHttpClientTest.java +++ b/ranger-http-client/src/test/java/io/appform/ranger/client/http/BaseRangerHttpClientTest.java @@ -76,6 +76,7 @@ public void prepareHttpMocks() throws Exception { .withStatus(200))); httpClientConfig = HttpClientConfig.builder() + .id("test-metric") .host("127.0.0.1") .port(wireMockExtension.getPort()) .connectionTimeoutMs(30_000) diff --git a/ranger-http-client/src/test/java/io/appform/ranger/client/http/ShardedRangerHttpClientTest.java b/ranger-http-client/src/test/java/io/appform/ranger/client/http/ShardedRangerHttpClientTest.java index 956c5ff6..26b981df 100644 --- a/ranger-http-client/src/test/java/io/appform/ranger/client/http/ShardedRangerHttpClientTest.java +++ b/ranger-http-client/src/test/java/io/appform/ranger/client/http/ShardedRangerHttpClientTest.java @@ -34,6 +34,7 @@ void testShardedHttpHubClient(){ .deserializer(this::read) .mapper(getObjectMapper()) .nodeRefreshTimeMs(1000) + .metricId("test-metric") .build(); client.start(); val service = RangerTestUtils.getService("test-n", "test-s"); diff --git a/ranger-http-client/src/test/java/io/appform/ranger/client/http/UnshardedRangerHttpClientTest.java b/ranger-http-client/src/test/java/io/appform/ranger/client/http/UnshardedRangerHttpClientTest.java index b4b64325..0b734f0e 100644 --- a/ranger-http-client/src/test/java/io/appform/ranger/client/http/UnshardedRangerHttpClientTest.java +++ b/ranger-http-client/src/test/java/io/appform/ranger/client/http/UnshardedRangerHttpClientTest.java @@ -34,6 +34,7 @@ void testUnshardedRangerHubClient(){ .deserializer(this::read) .mapper(getObjectMapper()) .nodeRefreshTimeMs(1000) + .metricId("test-metric") .build(); client.start(); val service = RangerTestUtils.getService("test-n", "test-s"); diff --git a/ranger-http/src/test/java/io/appform/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java b/ranger-http/src/test/java/io/appform/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java index 88c1b6cb..6f882785 100644 --- a/ranger-http/src/test/java/io/appform/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java +++ b/ranger-http/src/test/java/io/appform/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java @@ -88,6 +88,7 @@ void testFinder(WireMockRuntimeInfo wireMockRuntimeInfo) throws Exception { }) .withShardSelector((criteria, registry) -> registry.nodeList()) .withNodeRefreshIntervalMs(1000) + .withMetricId("test-metric") .build(); finder.start(); RangerTestUtils.sleepUntilFinderStarts(finder); diff --git a/ranger-http/src/test/java/io/appform/ranger/http/serviceprovider/HttpShardedServiceProviderBuilderTest.java b/ranger-http/src/test/java/io/appform/ranger/http/serviceprovider/HttpShardedServiceProviderBuilderTest.java index d60e3794..a89b45a8 100644 --- a/ranger-http/src/test/java/io/appform/ranger/http/serviceprovider/HttpShardedServiceProviderBuilderTest.java +++ b/ranger-http/src/test/java/io/appform/ranger/http/serviceprovider/HttpShardedServiceProviderBuilderTest.java @@ -89,6 +89,7 @@ void testProvider(WireMockRuntimeInfo wireMockRuntimeInfo) throws Exception { .withNodeData(farmNodeData) .withSerializer(node -> requestBytes) .healthUpdateHandler(healthUpdateHandler) + .withMetricId("test-metric") .build(); serviceProvider.start(); Assertions.assertNotNull(serviceProvider); diff --git a/ranger-hub-server-bundle-core/ranger-hub-server-bundle/src/main/java/io/appform/ranger/hub/server/bundle/RangerHubServerBundle.java b/ranger-hub-server-bundle-core/ranger-hub-server-bundle/src/main/java/io/appform/ranger/hub/server/bundle/RangerHubServerBundle.java index 50196088..5808600b 100644 --- a/ranger-hub-server-bundle-core/ranger-hub-server-bundle/src/main/java/io/appform/ranger/hub/server/bundle/RangerHubServerBundle.java +++ b/ranger-hub-server-bundle-core/ranger-hub-server-bundle/src/main/java/io/appform/ranger/hub/server/bundle/RangerHubServerBundle.java @@ -100,6 +100,7 @@ private RangerHubClient> addCurat .build(); curatorFrameworks.add(curatorFramework); return UnshardedRangerZKHubClient.builder() + .metricId(zkConfiguration.getId()) .namespace(namespace) .connectionString(zookeeper) .curatorFramework(curatorFramework) @@ -125,6 +126,7 @@ private RangerHubClient> addCurat private RangerHubClient> getHttpHubClient( HttpClientConfig httpClientConfig, RangerHttpUpstreamConfiguration httpConfiguration) { return UnshardedRangerHttpHubClient.builder() + .metricId(httpClientConfig.getId()) .namespace(namespace) .mapper(getMapper()) .clientConfig(httpClientConfig) @@ -153,6 +155,7 @@ private RangerHubClient> getDrove DroveUpstreamConfig.DEFAULT_REGION_TAG_NAME); val droveCommunicator = RangerDroveUtils.buildDroveClient(namespace, droveConfig, getMapper()); return UnshardedRangerDroveHubClient.builder() + .metricId(droveConfig.getId()) .namespace(namespace) .mapper(getMapper()) .clientConfig(droveConfig) diff --git a/ranger-hub-server-bundle/src/test/java/io/appform/ranger/hub/server/bundle/RangerHubServerBundleTest.java b/ranger-hub-server-bundle/src/test/java/io/appform/ranger/hub/server/bundle/RangerHubServerBundleTest.java index 677f394c..a228cf66 100644 --- a/ranger-hub-server-bundle/src/test/java/io/appform/ranger/hub/server/bundle/RangerHubServerBundleTest.java +++ b/ranger-hub-server-bundle/src/test/java/io/appform/ranger/hub/server/bundle/RangerHubServerBundleTest.java @@ -16,6 +16,7 @@ package io.appform.ranger.hub.server.bundle; +import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.health.HealthCheckRegistry; import com.fasterxml.jackson.databind.ObjectMapper; import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo; @@ -67,6 +68,7 @@ class TestConfig extends Configuration { .upstreams(List.of( new RangerHttpUpstreamConfiguration() .setHttpClientConfigs(List.of(HttpClientConfig.builder() + .id("test-metric") .host("localhost") .port(wm.getHttpPort()) .build())) @@ -90,6 +92,7 @@ class TestConfig extends Configuration { when(environment.healthChecks()).thenReturn(healthChecks); when(environment.admin()).thenReturn(adminEnvironment); when(environment.getObjectMapper()).thenReturn(mapper); + when(environment.metrics()).thenReturn(new MetricRegistry()); when(bootstrap.getHealthCheckRegistry()).thenReturn(mock(HealthCheckRegistry.class)); val bundle = new RangerHubServerBundle() { diff --git a/ranger-hub-server-dw5-bundle/src/main/java/io/appform/ranger/hub/server/bundle/RangerHubServerBundle.java b/ranger-hub-server-dw5-bundle/src/main/java/io/appform/ranger/hub/server/bundle/RangerHubServerBundle.java index e6d240bf..a6adb695 100644 --- a/ranger-hub-server-dw5-bundle/src/main/java/io/appform/ranger/hub/server/bundle/RangerHubServerBundle.java +++ b/ranger-hub-server-dw5-bundle/src/main/java/io/appform/ranger/hub/server/bundle/RangerHubServerBundle.java @@ -100,6 +100,7 @@ private RangerHubClient> addCurat .build(); curatorFrameworks.add(curatorFramework); return UnshardedRangerZKHubClient.builder() + .metricId(zkConfiguration.getId()) .namespace(namespace) .connectionString(zookeeper) .curatorFramework(curatorFramework) @@ -125,6 +126,7 @@ private RangerHubClient> addCurat private RangerHubClient> getHttpHubClient( HttpClientConfig httpClientConfig, RangerHttpUpstreamConfiguration httpConfiguration) { return UnshardedRangerHttpHubClient.builder() + .metricId(httpClientConfig.getId()) .namespace(namespace) .mapper(getMapper()) .clientConfig(httpClientConfig) @@ -153,6 +155,7 @@ private RangerHubClient> getDrove DroveUpstreamConfig.DEFAULT_REGION_TAG_NAME); val droveCommunicator = RangerDroveUtils.buildDroveClient(namespace, droveConfig, getMapper()); return UnshardedRangerDroveHubClient.builder() + .metricId(droveConfig.getId()) .namespace(namespace) .mapper(getMapper()) .clientConfig(droveConfig) diff --git a/ranger-hub-server-dw5-bundle/src/test/java/io/appform/ranger/hub/server/bundle/RangerHubServerBundleTest.java b/ranger-hub-server-dw5-bundle/src/test/java/io/appform/ranger/hub/server/bundle/RangerHubServerBundleTest.java index a5d8e7c3..6b44167b 100644 --- a/ranger-hub-server-dw5-bundle/src/test/java/io/appform/ranger/hub/server/bundle/RangerHubServerBundleTest.java +++ b/ranger-hub-server-dw5-bundle/src/test/java/io/appform/ranger/hub/server/bundle/RangerHubServerBundleTest.java @@ -16,6 +16,7 @@ package io.appform.ranger.hub.server.bundle; +import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.health.HealthCheckRegistry; import com.fasterxml.jackson.databind.ObjectMapper; import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo; @@ -68,11 +69,12 @@ class TestConfig extends Configuration { private final RangerServerConfiguration upstreams = RangerServerConfiguration.builder() .namespace("test") .upstreams(ImmutableList.builder() - .add(new RangerHttpUpstreamConfiguration() - .setHttpClientConfigs(List.of(HttpClientConfig.builder() - .host("localhost") - .port(wm.getHttpPort()) - .build()))) + .add(new RangerHttpUpstreamConfiguration() + .setHttpClientConfigs(List.of(HttpClientConfig.builder() + .id("test-metric") + .host("localhost") + .port(wm.getHttpPort()) + .build()))) .build()) .build(); @@ -92,6 +94,7 @@ class TestConfig extends Configuration { when(environment.healthChecks()).thenReturn(healthChecks); when(environment.admin()).thenReturn(adminEnvironment); when(environment.getObjectMapper()).thenReturn(mapper); + when(environment.metrics()).thenReturn(new MetricRegistry()); when(bootstrap.getHealthCheckRegistry()).thenReturn(mock(HealthCheckRegistry.class)); val bundle = new RangerHubServerBundle() { diff --git a/ranger-id/perf/results/io.appform.ranger.id.IdGeneratorPerfTest.testGenerate.json b/ranger-id/perf/results/io.appform.ranger.id.IdGeneratorPerfTest.testGenerate.json index 4d90b3ee..709b8f13 100644 --- a/ranger-id/perf/results/io.appform.ranger.id.IdGeneratorPerfTest.testGenerate.json +++ b/ranger-id/perf/results/io.appform.ranger.id.IdGeneratorPerfTest.testGenerate.json @@ -4,5 +4,5 @@ "iterations" : 4, "threads" : 1, "forks" : 3, - "mean_ops" : 333419.8831864388 + "mean_ops" : 665263.0845729186 } \ No newline at end of file diff --git a/ranger-id/perf/results/io.appform.ranger.id.IdGeneratorPerfTest.testGenerateBase36.json b/ranger-id/perf/results/io.appform.ranger.id.IdGeneratorPerfTest.testGenerateBase36.json index 78e157bb..efbfb290 100644 --- a/ranger-id/perf/results/io.appform.ranger.id.IdGeneratorPerfTest.testGenerateBase36.json +++ b/ranger-id/perf/results/io.appform.ranger.id.IdGeneratorPerfTest.testGenerateBase36.json @@ -4,5 +4,5 @@ "iterations" : 4, "threads" : 1, "forks" : 3, - "mean_ops" : 243301.4533266693 + "mean_ops" : 506179.5641038627 } \ No newline at end of file diff --git a/ranger-zk-client/src/main/java/io/appform/ranger/client/zk/ShardedRangerZKHubClient.java b/ranger-zk-client/src/main/java/io/appform/ranger/client/zk/ShardedRangerZKHubClient.java index 55af8445..aaa88288 100644 --- a/ranger-zk-client/src/main/java/io/appform/ranger/client/zk/ShardedRangerZKHubClient.java +++ b/ranger-zk-client/src/main/java/io/appform/ranger/client/zk/ShardedRangerZKHubClient.java @@ -41,6 +41,7 @@ public class ShardedRangerZKHubClient @Override protected ServiceFinderFactory> getFinderFactory() { return ZkShardedServiceFinderFactory.builder() + .metricId(getMetricId()) .curatorFramework(getCuratorFramework()) .connectionString(getConnectionString()) .nodeRefreshIntervalMs(getNodeRefreshTimeMs()) diff --git a/ranger-zk-client/src/main/java/io/appform/ranger/client/zk/UnshardedRangerZKHubClient.java b/ranger-zk-client/src/main/java/io/appform/ranger/client/zk/UnshardedRangerZKHubClient.java index 2b3eea83..7e34a320 100644 --- a/ranger-zk-client/src/main/java/io/appform/ranger/client/zk/UnshardedRangerZKHubClient.java +++ b/ranger-zk-client/src/main/java/io/appform/ranger/client/zk/UnshardedRangerZKHubClient.java @@ -41,6 +41,7 @@ public class UnshardedRangerZKHubClient @Override protected ServiceFinderFactory> getFinderFactory() { return ZKUnshardedServiceFinderFactory.builder() + .metricId(getMetricId()) .curatorFramework(getCuratorFramework()) .connectionString(getConnectionString()) .nodeRefreshIntervalMs(getNodeRefreshTimeMs()) diff --git a/ranger-zk-client/src/test/java/io/appform/ranger/client/zk/BaseRangerZKClientTest.java b/ranger-zk-client/src/test/java/io/appform/ranger/client/zk/BaseRangerZKClientTest.java index 67459647..c81d0b0c 100644 --- a/ranger-zk-client/src/test/java/io/appform/ranger/client/zk/BaseRangerZKClientTest.java +++ b/ranger-zk-client/src/test/java/io/appform/ranger/client/zk/BaseRangerZKClientTest.java @@ -110,6 +110,7 @@ protected void initilizeProvider(){ final HealthUpdateHandler healthUpdateHandler = new LastUpdatedHandler() .setNext(new HealthStatusHandler()); provider = ServiceProviderBuilders.shardedServiceProviderBuilder() + .withMetricId("test-metric") .withHostname("localhost") .withPort(1080) .withNamespace("test-n") diff --git a/ranger-zk-client/src/test/java/io/appform/ranger/client/zk/SimpleRangerZKClientTest.java b/ranger-zk-client/src/test/java/io/appform/ranger/client/zk/SimpleRangerZKClientTest.java index c0603c50..40bcf201 100644 --- a/ranger-zk-client/src/test/java/io/appform/ranger/client/zk/SimpleRangerZKClientTest.java +++ b/ranger-zk-client/src/test/java/io/appform/ranger/client/zk/SimpleRangerZKClientTest.java @@ -31,6 +31,7 @@ void testBaseClient(){ .serviceName("s1") .disableWatchers(true) .mapper(getObjectMapper()) + .metricId("test-metric") .build(); client.start(); Assertions.assertNotNull( client.getNode().orElse(null)); diff --git a/ranger-zk-client/src/test/java/io/appform/ranger/client/zk/UnshardedZKRangerClientTest.java b/ranger-zk-client/src/test/java/io/appform/ranger/client/zk/UnshardedZKRangerClientTest.java index a7a479c0..ad78e8b6 100644 --- a/ranger-zk-client/src/test/java/io/appform/ranger/client/zk/UnshardedZKRangerClientTest.java +++ b/ranger-zk-client/src/test/java/io/appform/ranger/client/zk/UnshardedZKRangerClientTest.java @@ -35,6 +35,7 @@ void testShardedHub(){ .mapper(getObjectMapper()) .deserializer(this::read) .nodeRefreshTimeMs(1000) + .metricId("test-metric") .build(); zkHubClient.start(); val service = RangerTestUtils.getService("test-n", "s1"); diff --git a/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinderhub/ZKUnshardedServiceFinderFactory.java b/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinderhub/ZKUnshardedServiceFinderFactory.java index 4a470522..d46f9994 100644 --- a/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinderhub/ZKUnshardedServiceFinderFactory.java +++ b/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinderhub/ZKUnshardedServiceFinderFactory.java @@ -37,6 +37,7 @@ public class ZKUnshardedServiceFinderFactory implements ServiceFinderFactory< private final ZkNodeDataDeserializer deserializer; private final ShardSelector> shardSelector; private final ServiceNodeSelector nodeSelector; + private final String metricId; @Builder public ZKUnshardedServiceFinderFactory( @@ -46,7 +47,8 @@ public ZKUnshardedServiceFinderFactory( boolean disablePushUpdaters, ZkNodeDataDeserializer deserializer, ShardSelector> shardSelector, - ServiceNodeSelector nodeSelector) { + ServiceNodeSelector nodeSelector, + String metricId) { this.curatorFramework = curatorFramework; this.connectionString = connectionString; this.nodeRefreshIntervalMs = nodeRefreshIntervalMs; @@ -54,11 +56,13 @@ public ZKUnshardedServiceFinderFactory( this.deserializer = deserializer; this.shardSelector = shardSelector; this.nodeSelector = nodeSelector; + this.metricId = metricId; } @Override public SimpleUnshardedServiceFinder buildFinder(Service service) { val finder = new ZkSimpleUnshardedServiceFinderBuilder() + .withMetricId(metricId) .withDeserializer(deserializer) .withNamespace(service.getNamespace()) .withServiceName(service.getServiceName()) diff --git a/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinderhub/ZkShardedServiceFinderFactory.java b/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinderhub/ZkShardedServiceFinderFactory.java index 5689cab3..e05faed8 100644 --- a/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinderhub/ZkShardedServiceFinderFactory.java +++ b/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinderhub/ZkShardedServiceFinderFactory.java @@ -38,6 +38,7 @@ public class ZkShardedServiceFinderFactory implements ServiceFinderFactory deserializer; private final ShardSelector> shardSelector; private final ServiceNodeSelector nodeSelector; + private final String metricId; @Builder public ZkShardedServiceFinderFactory( @@ -47,7 +48,8 @@ public ZkShardedServiceFinderFactory( boolean disablePushUpdaters, ZkNodeDataDeserializer deserializer, ShardSelector> shardSelector, - ServiceNodeSelector nodeSelector) { + ServiceNodeSelector nodeSelector, + String metricId) { this.curatorFramework = curatorFramework; this.connectionString = connectionString; this.nodeRefreshIntervalMs = nodeRefreshIntervalMs; @@ -55,11 +57,13 @@ public ZkShardedServiceFinderFactory( this.deserializer = deserializer; this.shardSelector = shardSelector; this.nodeSelector = nodeSelector; + this.metricId = metricId; } @Override public SimpleShardedServiceFinder buildFinder(Service service) { val finder = new ZkSimpleShardedServiceFinderBuilder() + .withMetricId(metricId) .withDeserializer(deserializer) .withNamespace(service.getNamespace()) .withServiceName(service.getServiceName()) diff --git a/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/healthservice/ServiceProviderIntegrationTest.java b/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/healthservice/ServiceProviderIntegrationTest.java index 7c83dbf9..ea5b6990 100644 --- a/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/healthservice/ServiceProviderIntegrationTest.java +++ b/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/healthservice/ServiceProviderIntegrationTest.java @@ -70,6 +70,7 @@ public void startTestCluster() throws Exception { registerService("localhost-4", 9003, 2, anotherFile); serviceFinder = ServiceFinderBuilders.shardedFinderBuilder() + .withMetricId("test-metric") .withConnectionString(testingCluster.getConnectString()) .withNamespace("test") .withServiceName("test-service") @@ -135,6 +136,7 @@ private void registerService(String host, int port, int shardId, File file) { final HealthUpdateHandler healthUpdateHandler = new LastUpdatedHandler() .setNext(new HealthStatusHandler()); val serviceProvider = ServiceProviderBuilders.shardedServiceProviderBuilder() + .withMetricId("test-metric") .withConnectionString(testingCluster.getConnectString()) .withNamespace("test") .withServiceName("test-service") diff --git a/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/model/CustomShardSelectorTest.java b/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/model/CustomShardSelectorTest.java index 3fff1b67..3e0c5c08 100644 --- a/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/model/CustomShardSelectorTest.java +++ b/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/model/CustomShardSelectorTest.java @@ -100,6 +100,7 @@ public List> nodes(Predicate criteria, @Test void testBasicDiscovery() { val serviceFinder = ServiceFinderBuilders.shardedFinderBuilder() + .withMetricId("test-metric") .withConnectionString(testingCluster.getConnectString()) .withNamespace("test") .withServiceName("test-service") @@ -135,6 +136,7 @@ private void registerService(String host, int port, int a, int b) { final HealthUpdateHandler healthUpdateHandler = new LastUpdatedHandler() .setNext(new HealthStatusHandler()); val serviceProvider = ServiceProviderBuilders.shardedServiceProviderBuilder() + .withMetricId("test-metric") .withConnectionString(testingCluster.getConnectString()) .withNamespace("test") .withServiceName("test-service") diff --git a/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/model/ServiceNoProviderTest.java b/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/model/ServiceNoProviderTest.java index d1947e9c..3fd94327 100644 --- a/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/model/ServiceNoProviderTest.java +++ b/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/model/ServiceNoProviderTest.java @@ -54,6 +54,7 @@ public void stopTestCluster() throws Exception { @Test void testBasicDiscovery() { val serviceFinder = ServiceFinderBuilders.shardedFinderBuilder() + .withMetricId("test-metric") .withConnectionString(testingCluster.getConnectString()) .withNamespace("test") .withServiceName("test-service") @@ -79,6 +80,7 @@ void testBasicDiscovery() { @Test void testBasicDiscoveryRR() { val serviceFinder = ServiceFinderBuilders.shardedFinderBuilder() + .withMetricId("test-metric") .withConnectionString(testingCluster.getConnectString()) .withNamespace("test") .withServiceName("test-service") diff --git a/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/model/ServiceProviderExtCuratorTest.java b/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/model/ServiceProviderExtCuratorTest.java index 1dd16e2b..4ac38bd4 100644 --- a/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/model/ServiceProviderExtCuratorTest.java +++ b/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/model/ServiceProviderExtCuratorTest.java @@ -80,6 +80,7 @@ public void stopTestCluster() throws Exception { @Test void testBasicDiscovery() { val serviceFinder = ServiceFinderBuilders.shardedFinderBuilder() + .withMetricId("test-metric") .withCuratorFramework(curatorFramework) .withNamespace("test") .withServiceName("test-service") @@ -122,6 +123,7 @@ private void registerService(String host, int port, int shardId) { final HealthUpdateHandler healthUpdateHandler = new LastUpdatedHandler() .setNext(new HealthStatusHandler()); val serviceProvider = ServiceProviderBuilders.shardedServiceProviderBuilder() + .withMetricId("test-metric") .withCuratorFramework(curatorFramework) .withNamespace("test") .withServiceName("test-service") diff --git a/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/model/ServiceProviderHealthcheckTest.java b/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/model/ServiceProviderHealthcheckTest.java index 506b76ba..9384f9cd 100644 --- a/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/model/ServiceProviderHealthcheckTest.java +++ b/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/model/ServiceProviderHealthcheckTest.java @@ -66,6 +66,7 @@ public void stopTestCluster() throws Exception { @Test void testBasicDiscovery() { val serviceFinder = ServiceFinderBuilders.shardedFinderBuilder() + .withMetricId("test-metric") .withConnectionString(testingCluster.getConnectString()) .withNamespace("test") .withServiceName("test-service") @@ -141,6 +142,7 @@ public void start() { final HealthUpdateHandler healthUpdateHandler = new LastUpdatedHandler() .setNext(new HealthStatusHandler()); val serviceProvider = ServiceProviderBuilders.shardedServiceProviderBuilder() + .withMetricId("test-metric") .withConnectionString(connectionString) .withNamespace("test") .withServiceName("test-service") diff --git a/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/model/ServiceProviderTest.java b/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/model/ServiceProviderTest.java index f8105a49..b23e1de7 100644 --- a/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/model/ServiceProviderTest.java +++ b/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/model/ServiceProviderTest.java @@ -73,6 +73,7 @@ public void stopTestCluster() throws Exception { @Test void testBasicDiscovery() { val serviceFinder = ServiceFinderBuilders.shardedFinderBuilder() + .withMetricId("test-metric") .withConnectionString(testingCluster.getConnectString()) .withNamespace("test") .withServiceName("test-service") @@ -114,6 +115,7 @@ void testBasicDiscovery() { void testBasicDiscoveryRR() { val serviceFinder = ServiceFinderBuilders.shardedFinderBuilder() + .withMetricId("test-metric") .withConnectionString(testingCluster.getConnectString()) .withNamespace("test") .withServiceName("test-service") @@ -158,6 +160,7 @@ void testBasicDiscoveryRR() { void testVisibility() { val serviceFinder = ServiceFinderBuilders. shardedFinderBuilder() + .withMetricId("test-metric") .withConnectionString(testingCluster.getConnectString()) .withNamespace("test") .withServiceName("test-service") @@ -186,6 +189,7 @@ private void registerService(String host, int port, int shardId) { final HealthUpdateHandler healthUpdateHandler = new LastUpdatedHandler() .setNext(new HealthStatusHandler<>()); val serviceProvider = ServiceProviderBuilders.shardedServiceProviderBuilder() + .withMetricId("test-metric") .withConnectionString(testingCluster.getConnectString()) .withNamespace("test") .withServiceName("test-service") diff --git a/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/model/SimpleServiceProviderTest.java b/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/model/SimpleServiceProviderTest.java index 7c9c7ce5..27fcddc3 100644 --- a/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/model/SimpleServiceProviderTest.java +++ b/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/model/SimpleServiceProviderTest.java @@ -75,6 +75,7 @@ public boolean equals(Object obj) { @Test void testBasicDiscovery() { val serviceFinder = ServiceFinderBuilders.unshardedFinderBuilder() + .withMetricId("test-metric") .withConnectionString(testingCluster.getConnectString()) .withNamespace("test") .withServiceName("test-service") @@ -110,6 +111,7 @@ private void registerService(String host, int port) { final HealthUpdateHandler healthUpdateHandler = new LastUpdatedHandler() .setNext(new HealthStatusHandler()); val serviceProvider = ServiceProviderBuilders.unshardedServiceProviderBuilder() + .withMetricId("test-metric") .withConnectionString(testingCluster.getConnectString()) .withNamespace("test") .withServiceName("test-service") diff --git a/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/servicehub/ServiceHubTest.java b/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/servicehub/ServiceHubTest.java index eaa661ba..6511f5aa 100644 --- a/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/servicehub/ServiceHubTest.java +++ b/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/servicehub/ServiceHubTest.java @@ -93,6 +93,7 @@ void testHub() { final HealthUpdateHandler healthUpdateHandler = new LastUpdatedHandler() .setNext(new HealthStatusHandler<>()); val provider1 = ServiceProviderBuilders.shardedServiceProviderBuilder() + .withMetricId("test-metric") .withHostname("localhost") .withPort(1080) .withNamespace(NAMESPACE) @@ -114,6 +115,7 @@ void testHub() { .withRefreshFrequencyMs(1000) .withServiceDataSource(new ZkServiceDataSource(null, "test", testingCluster.getConnectString(), curatorFramework)) .withServiceFinderFactory(ZkShardedServiceFinderFactory.builder() + .metricId("test-metric") .curatorFramework(curatorFramework) .deserializer(this::read) .build()) diff --git a/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/serviceprovider/BaseServiceProviderBuilderTest.java b/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/serviceprovider/BaseServiceProviderBuilderTest.java index e120afc0..eaa17865 100644 --- a/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/serviceprovider/BaseServiceProviderBuilderTest.java +++ b/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/serviceprovider/BaseServiceProviderBuilderTest.java @@ -63,6 +63,7 @@ void testServiceProviderBuilder() { .setNext(new HealthStatusHandler<>()); try { val serviceProvider = ServiceProviderBuilders.unshardedServiceProviderBuilder() + .withMetricId("test-metric") .withConnectionString(testingCluster.getConnectString()) .withNamespace("test") .withServiceName("test-service") @@ -86,6 +87,7 @@ void testServiceProviderBuilder() { Assertions.assertTrue(exception instanceof IllegalArgumentException); val serviceProvider = ServiceProviderBuilders.unshardedServiceProviderBuilder() + .withMetricId("test-metric") .withConnectionString(testingCluster.getConnectString()) .withNamespace("test") .withServiceName("test-service") From 85888b2f0c463a8a8e593e0a172c9760d7048455 Mon Sep 17 00:00:00 2001 From: Jitendra Dhawan Date: Sun, 31 May 2026 19:22:18 +0530 Subject: [PATCH 09/24] Add integration tests for Drove API and Node Data Source metrics recording --- ...ApiCommunicatorMetricsIntegrationTest.java | 287 ++++++++++++++++++ ...eNodeDataSourceMetricsIntegrationTest.java | 200 ++++++++++++ ...rviceDataSourceMetricsIntegrationTest.java | 192 ++++++++++++ 3 files changed, 679 insertions(+) create mode 100644 ranger-drove/src/test/java/io/appform/ranger/drove/common/DroveApiCommunicatorMetricsIntegrationTest.java create mode 100644 ranger-drove/src/test/java/io/appform/ranger/drove/servicefinder/DroveNodeDataSourceMetricsIntegrationTest.java create mode 100644 ranger-drove/src/test/java/io/appform/ranger/drove/servicefinderhub/DroveServiceDataSourceMetricsIntegrationTest.java diff --git a/ranger-drove/src/test/java/io/appform/ranger/drove/common/DroveApiCommunicatorMetricsIntegrationTest.java b/ranger-drove/src/test/java/io/appform/ranger/drove/common/DroveApiCommunicatorMetricsIntegrationTest.java new file mode 100644 index 00000000..3727a9cd --- /dev/null +++ b/ranger-drove/src/test/java/io/appform/ranger/drove/common/DroveApiCommunicatorMetricsIntegrationTest.java @@ -0,0 +1,287 @@ +/* + * Copyright 2024 Authors, Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.appform.ranger.drove.common; + +import com.codahale.metrics.MetricRegistry; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.tomakehurst.wiremock.http.Fault; +import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo; +import com.github.tomakehurst.wiremock.junit5.WireMockTest; +import com.phonepe.drove.models.api.ApiResponse; +import com.phonepe.drove.models.api.AppSummary; +import com.phonepe.drove.models.api.ExposedAppInfo; +import com.phonepe.drove.models.application.ApplicationState; +import com.phonepe.drove.models.application.PortType; +import io.appform.ranger.core.model.Service; +import io.appform.ranger.core.util.MetricRecorder; +import io.appform.ranger.drove.config.DroveUpstreamConfig; +import io.appform.ranger.drove.utils.RangerDroveUtils; +import lombok.SneakyThrows; +import lombok.val; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.Date; +import java.util.List; +import java.util.Map; + +import static com.github.tomakehurst.wiremock.client.WireMock.*; +import static org.junit.jupiter.api.Assertions.*; + +/** + * Integration tests for DroveApiCommunicator metrics recording. + * Tests verify that metrics are pushed through actual Drove API calls via WireMock. + */ +@WireMockTest +class DroveApiCommunicatorMetricsIntegrationTest { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + private static final String METRIC_PREFIX = "io.appform.ranger"; + private MetricRegistry metricRegistry; + + @BeforeEach + void setUp() { + metricRegistry = new MetricRegistry(); + MetricRecorder.initialize(metricRegistry); + } + + // ==================== services() - Success with 200 ==================== + + @Test + @SneakyThrows + void testServices_success_recordsStatusCodeAndNoFailure(WireMockRuntimeInfo wm) { + val response = ApiResponse.success(Map.of( + "APP_1", new AppSummary("APP_1", "APP_1", 4, 4, 4, 1024, Map.of(), + ApplicationState.RUNNING, new Date(), new Date()), + "APP_2", new AppSummary("APP_2", "APP_2", 4, 4, 4, 1024, Map.of(), + ApplicationState.RUNNING, new Date(), new Date()))); + + stubFor(get("/apis/v1/applications") + .withBasicAuth("guest", "guest") + .willReturn(okJson(MAPPER.writeValueAsString(response)))); + + try (val client = buildClient(wm, "drove-api-1")) { + val services = client.services(); + assertNotNull(services); + assertEquals(2, services.size()); + + // Verify 200 status code meter + val statusMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.DROVE.dataSource.drove-api-1.httpCall.services.responseStatus.200"); + assertNotNull(statusMeter, "200 status meter should be recorded for services"); + assertEquals(1, statusMeter.getCount()); + } + } + + // ==================== services() - Empty data (null/empty services response) ==================== + + @Test + @SneakyThrows + void testServices_emptyData_recordsNullOrEmptyServicesResponse(WireMockRuntimeInfo wm) { + val response = ApiResponse.success(Map.of()); // empty map + + stubFor(get("/apis/v1/applications") + .withBasicAuth("guest", "guest") + .willReturn(okJson(MAPPER.writeValueAsString(response)))); + + try (val client = buildClient(wm, "drove-api-2")) { + val services = client.services(); + assertNotNull(services); + assertTrue(services.isEmpty()); + + // Verify null/empty services meter + val emptyMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.DROVE.dataSource.drove-api-2.httpCall.services.nullOrEmptyResponse"); + assertNotNull(emptyMeter, "Null/empty services response meter should be recorded"); + assertEquals(1, emptyMeter.getCount()); + } + } + + // ==================== services() - Non-200 (error) ==================== + + @Test + @SneakyThrows + void testServices_serverError_recordsStatusCode(WireMockRuntimeInfo wm) { + stubFor(get("/apis/v1/applications") + .withBasicAuth("guest", "guest") + .willReturn(aResponse().withStatus(500).withBody("Internal Server Error"))); + + try (val client = buildClient(wm, "drove-api-3")) { + assertThrows(DroveCommunicationException.class, client::services); + + // Verify 500 status code meter + val statusMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.DROVE.dataSource.drove-api-3.httpCall.services.responseStatus.500"); + assertNotNull(statusMeter, "500 status meter should be recorded"); + assertEquals(1, statusMeter.getCount()); + } + } + + // ==================== services() - Invalid JSON (parse failure) ==================== + + @Test + @SneakyThrows + void testServices_invalidJson_recordsParseFailure(WireMockRuntimeInfo wm) { + stubFor(get("/apis/v1/applications") + .withBasicAuth("guest", "guest") + .willReturn(aResponse().withStatus(200).withBody("not-valid-json{{{"))); + + try (val client = buildClient(wm, "drove-api-4")) { + assertThrows(DroveCommunicationException.class, client::services); + + // Verify parse failure meter + val parseMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.DROVE.dataSource.drove-api-4.httpCall.services.responseParseFailure"); + assertNotNull(parseMeter, "Services parse failure meter should be recorded"); + assertEquals(1, parseMeter.getCount()); + } + } + + // ==================== services() - Network error (unknown failure from OkHttp transport) ==================== + + @Test + @SneakyThrows + void testServices_networkError_recordsRemoteCallUnknownFailure(WireMockRuntimeInfo wm) { + stubFor(get("/apis/v1/applications") + .withBasicAuth("guest", "guest") + .willReturn(aResponse().withFault(Fault.MALFORMED_RESPONSE_CHUNK))); + + try (val client = buildClient(wm, "drove-api-5")) { + assertThrows(DroveCommunicationException.class, client::services); + + // Verify remote call unknown failure meter (from DroveOkHttpTransport.get()) + val failureMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.DROVE.dataSource.drove-api-5.httpCall.GET.unknownFailure"); + assertNotNull(failureMeter, "Remote call unknown failure meter should be recorded"); + assertTrue(failureMeter.getCount() >= 1); + } + } + + // ==================== listNodes() - Success with 200 ==================== + + @Test + @SneakyThrows + void testListNodes_success_recordsStatusCode(WireMockRuntimeInfo wm) { + val nodeResponse = ApiResponse.success(List.of( + new ExposedAppInfo("TEST_APP", "v1", "host.internal", Map.of(), + List.of(new ExposedAppInfo.ExposedHost("host1.internal", 32000, PortType.HTTP))))); + + stubFor(get(urlPathEqualTo("/apis/v1/endpoints")) + .withBasicAuth("guest", "guest") + .willReturn(aResponse() + .withBody(MAPPER.writeValueAsBytes(nodeResponse)) + .withStatus(200))); + + try (val client = buildClient(wm, "drove-api-6")) { + val service = new Service("testns", "TEST_APP"); + val nodes = client.listNodes(service); + + assertNotNull(nodes); + assertFalse(nodes.isEmpty()); + + // Verify 200 status code for listNodes + val statusMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.DROVE.dataSource.drove-api-6.httpCall.listNodes.responseStatus.200"); + assertNotNull(statusMeter, "200 status meter for listNodes should exist"); + assertEquals(1, statusMeter.getCount()); + } + } + + // ==================== listNodes() - Empty response ==================== + + @Test + @SneakyThrows + void testListNodes_emptyData_recordsNullOrEmptyListNodeResponse(WireMockRuntimeInfo wm) { + val nodeResponse = ApiResponse.success(List.of()); // empty list + + stubFor(get(urlPathEqualTo("/apis/v1/endpoints")) + .withBasicAuth("guest", "guest") + .willReturn(aResponse() + .withBody(MAPPER.writeValueAsBytes(nodeResponse)) + .withStatus(200))); + + try (val client = buildClient(wm, "drove-api-7")) { + val service = new Service("testns", "TEST_APP"); + val nodes = client.listNodes(service); + + assertNotNull(nodes); + assertTrue(nodes.isEmpty()); + + // Verify null/empty list node response + val emptyMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.DROVE.dataSource.drove-api-7.httpCall.listNodes.serviceName.TEST_APP.nullOrEmptyResponse"); + assertNotNull(emptyMeter, "Null/empty list node response meter should be recorded"); + assertEquals(1, emptyMeter.getCount()); + } + } + + // ==================== listNodes() - Invalid JSON (parse failure) ==================== + + @Test + @SneakyThrows + void testListNodes_invalidJson_recordsParseFailure(WireMockRuntimeInfo wm) { + stubFor(get(urlPathEqualTo("/apis/v1/endpoints")) + .withBasicAuth("guest", "guest") + .willReturn(aResponse().withStatus(200).withBody("invalid-json{{{{"))); + + try (val client = buildClient(wm, "drove-api-8")) { + val service = new Service("testns", "PARSE_FAIL_APP"); + assertThrows(DroveCommunicationException.class, () -> client.listNodes(service)); + + // Verify list nodes parse failure + val parseMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.DROVE.dataSource.drove-api-8.httpCall.listNodes.serviceName.PARSE_FAIL_APP.responseParseFailure"); + assertNotNull(parseMeter, "List nodes parse failure meter should be recorded"); + assertEquals(1, parseMeter.getCount()); + } + } + + // ==================== listNodes() - Non-200 (error) ==================== + + @Test + @SneakyThrows + void testListNodes_serverError_recordsStatusCode(WireMockRuntimeInfo wm) { + stubFor(get(urlPathEqualTo("/apis/v1/endpoints")) + .withBasicAuth("guest", "guest") + .willReturn(aResponse().withStatus(503).withBody("Service Unavailable"))); + + try (val client = buildClient(wm, "drove-api-9")) { + val service = new Service("testns", "ERROR_APP"); + assertThrows(DroveCommunicationException.class, () -> client.listNodes(service)); + + // Verify 503 status code + val statusMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.DROVE.dataSource.drove-api-9.httpCall.listNodes.responseStatus.503"); + assertNotNull(statusMeter, "503 status meter for listNodes should be recorded"); + assertEquals(1, statusMeter.getCount()); + } + } + + // ==================== Helper ==================== + + private DroveCommunicator buildClient(WireMockRuntimeInfo wm, String metricId) { + return RangerDroveUtils.buildDroveClient( + "testns", + DroveUpstreamConfig.builder() + .id(metricId) + .endpoints(List.of("http://localhost:" + wm.getHttpPort())) + .username("guest") + .password("guest") + .skipCaching(true) + .build(), + MAPPER); + } +} diff --git a/ranger-drove/src/test/java/io/appform/ranger/drove/servicefinder/DroveNodeDataSourceMetricsIntegrationTest.java b/ranger-drove/src/test/java/io/appform/ranger/drove/servicefinder/DroveNodeDataSourceMetricsIntegrationTest.java new file mode 100644 index 00000000..01003152 --- /dev/null +++ b/ranger-drove/src/test/java/io/appform/ranger/drove/servicefinder/DroveNodeDataSourceMetricsIntegrationTest.java @@ -0,0 +1,200 @@ +/* + * Copyright 2024 Authors, Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.appform.ranger.drove.servicefinder; + +import com.codahale.metrics.MetricRegistry; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo; +import com.github.tomakehurst.wiremock.junit5.WireMockTest; +import com.phonepe.drove.models.api.ApiResponse; +import com.phonepe.drove.models.api.ExposedAppInfo; +import com.phonepe.drove.models.application.PortType; +import io.appform.ranger.core.model.Service; +import io.appform.ranger.core.util.MetricRecorder; +import io.appform.ranger.drove.common.DroveCommunicator; +import io.appform.ranger.drove.config.DroveUpstreamConfig; +import io.appform.ranger.drove.serde.DroveResponseDataDeserializer; +import io.appform.ranger.drove.utils.RangerDroveUtils; +import lombok.SneakyThrows; +import lombok.val; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; + +import static com.github.tomakehurst.wiremock.client.WireMock.*; +import static org.junit.jupiter.api.Assertions.*; + +/** + * Integration tests for DroveNodeDataSource metrics recording. + * Tests verify metrics from actual Drove data source flows. + */ +@WireMockTest +class DroveNodeDataSourceMetricsIntegrationTest { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + private static final String METRIC_PREFIX = "io.appform.ranger"; + private MetricRegistry metricRegistry; + + private static final DroveResponseDataDeserializer STRING_DESERIALIZER = + new DroveResponseDataDeserializer<>() { + @Override + protected String translate(ExposedAppInfo appInfo, ExposedAppInfo.ExposedHost host) { + return host.getHost() + ":" + host.getPort(); + } + }; + + @BeforeEach + void setUp() { + metricRegistry = new MetricRegistry(); + MetricRecorder.initialize(metricRegistry); + } + + // ==================== isActive() - Healthy upstream ==================== + + @Test + @SneakyThrows + void testIsActive_healthyUpstream_recordsActiveStatus(WireMockRuntimeInfo wm) { + // Drove client starts with upstreamAvailable = true + val config = buildConfig(wm, "drove-node-src-1"); + try (val droveClient = buildClient(config)) { + val service = new Service("testns", "TEST_APP"); + val dataSource = new DroveNodeDataSource>( + "drove-node-src-1", service, config, MAPPER, droveClient); + + val active = dataSource.isActive(); + + assertTrue(active); + + val activeMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataSource.DROVE.drove-node-src-1.active"); + assertNotNull(activeMeter, "Active status meter should be recorded"); + assertEquals(1, activeMeter.getCount()); + } + } + + // ==================== isActive() - Unhealthy upstream ==================== + + @Test + @SneakyThrows + void testIsActive_unhealthyUpstream_recordsInactiveStatus(WireMockRuntimeInfo wm) { + stubFor(get("/apis/v1/applications") + .withBasicAuth("guest", "guest") + .willReturn(aResponse().withStatus(500).withBody("Error"))); + + val config = buildConfig(wm, "drove-node-src-2"); + try (val droveClient = buildClient(config)) { + val service = new Service("testns", "TEST_APP"); + val dataSource = new DroveNodeDataSource>( + "drove-node-src-2", service, config, MAPPER, droveClient); + + // First call to services fails, setting upstreamAvailable to false + try { + droveClient.services(); + } catch (Exception ignored) { + } + + val active = dataSource.isActive(); + + assertFalse(active); + + val inactiveMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataSource.DROVE.drove-node-src-2.inactive"); + assertNotNull(inactiveMeter, "Inactive status meter should be recorded"); + assertEquals(1, inactiveMeter.getCount()); + } + } + + // ==================== refresh() - Success ==================== + + @Test + @SneakyThrows + void testRefresh_success_returnsNodes(WireMockRuntimeInfo wm) { + val nodeResponse = ApiResponse.success(List.of( + new ExposedAppInfo("TEST_APP", "v1", "host.internal", Map.of(), + List.of(new ExposedAppInfo.ExposedHost("host1.internal", 32000, PortType.HTTP), + new ExposedAppInfo.ExposedHost("host2.internal", 32001, PortType.HTTP))))); + + stubFor(get(urlPathEqualTo("/apis/v1/endpoints")) + .withBasicAuth("guest", "guest") + .willReturn(aResponse() + .withBody(MAPPER.writeValueAsBytes(nodeResponse)) + .withStatus(200))); + + val config = buildConfig(wm, "drove-node-src-3"); + try (val droveClient = buildClient(config)) { + val service = new Service("testns", "TEST_APP"); + val dataSource = new DroveNodeDataSource>( + "drove-node-src-3", service, config, MAPPER, droveClient); + + val result = dataSource.refresh(STRING_DESERIALIZER); + + assertTrue(result.isPresent()); + assertEquals(2, result.get().size()); + + // Verify 200 status code was recorded (from DroveApiCommunicator) + val statusMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.DROVE.dataSource.drove-node-src-3.httpCall.listNodes.responseStatus.200"); + assertNotNull(statusMeter, "200 status meter should be recorded for listNodes"); + assertEquals(1, statusMeter.getCount()); + } + } + + // ==================== refresh() - Communication failure ==================== + + @Test + @SneakyThrows + void testRefresh_communicationFailure_returnsEmpty(WireMockRuntimeInfo wm) { + stubFor(get(urlPathEqualTo("/apis/v1/endpoints")) + .withBasicAuth("guest", "guest") + .willReturn(aResponse().withStatus(500).withBody("Error"))); + + val config = buildConfig(wm, "drove-node-src-4"); + try (val droveClient = buildClient(config)) { + val service = new Service("testns", "FAIL_APP"); + val dataSource = new DroveNodeDataSource>( + "drove-node-src-4", service, config, MAPPER, droveClient); + + val result = dataSource.refresh(STRING_DESERIALIZER); + + // On DroveCommunicationException, returns Optional.empty() to maintain old list + assertFalse(result.isPresent()); + + // Verify 500 status code was still recorded + val statusMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.DROVE.dataSource.drove-node-src-4.httpCall.listNodes.responseStatus.500"); + assertNotNull(statusMeter, "500 status meter should be recorded"); + assertEquals(1, statusMeter.getCount()); + } + } + + // ==================== Helpers ==================== + + private DroveUpstreamConfig buildConfig(WireMockRuntimeInfo wm, String metricId) { + return DroveUpstreamConfig.builder() + .id(metricId) + .endpoints(List.of("http://localhost:" + wm.getHttpPort())) + .username("guest") + .password("guest") + .skipCaching(true) + .build(); + } + + private DroveCommunicator buildClient(DroveUpstreamConfig config) { + return RangerDroveUtils.buildDroveClient("testns", config, MAPPER); + } +} diff --git a/ranger-drove/src/test/java/io/appform/ranger/drove/servicefinderhub/DroveServiceDataSourceMetricsIntegrationTest.java b/ranger-drove/src/test/java/io/appform/ranger/drove/servicefinderhub/DroveServiceDataSourceMetricsIntegrationTest.java new file mode 100644 index 00000000..47c26f9a --- /dev/null +++ b/ranger-drove/src/test/java/io/appform/ranger/drove/servicefinderhub/DroveServiceDataSourceMetricsIntegrationTest.java @@ -0,0 +1,192 @@ +/* + * Copyright 2024 Authors, Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.appform.ranger.drove.servicefinderhub; + +import com.codahale.metrics.MetricRegistry; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo; +import com.github.tomakehurst.wiremock.junit5.WireMockTest; +import com.phonepe.drove.models.api.ApiResponse; +import com.phonepe.drove.models.api.AppSummary; +import com.phonepe.drove.models.application.ApplicationState; +import io.appform.ranger.core.util.MetricRecorder; +import io.appform.ranger.drove.common.DroveCommunicationException; +import io.appform.ranger.drove.common.DroveCommunicator; +import io.appform.ranger.drove.config.DroveUpstreamConfig; +import io.appform.ranger.drove.utils.RangerDroveUtils; +import lombok.SneakyThrows; +import lombok.val; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.Date; +import java.util.List; +import java.util.Map; + +import static com.github.tomakehurst.wiremock.client.WireMock.*; +import static org.junit.jupiter.api.Assertions.*; + +/** + * Integration tests for DroveServiceDataSource metrics recording. + * Tests verify that metrics are pushed through actual Drove code flows. + */ +@WireMockTest +class DroveServiceDataSourceMetricsIntegrationTest { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + private static final String METRIC_PREFIX = "io.appform.ranger"; + private MetricRegistry metricRegistry; + + @BeforeEach + void setUp() { + metricRegistry = new MetricRegistry(); + MetricRecorder.initialize(metricRegistry); + } + + // ==================== services() - Success ==================== + + @Test + @SneakyThrows + void testServices_success_recordsFetchSuccess(WireMockRuntimeInfo wm) { + val response = ApiResponse.success(Map.of( + "SVC_A", new AppSummary("SVC_A", "SVC_A", 4, 4, 4, 1024, Map.of(), + ApplicationState.RUNNING, new Date(), new Date()), + "SVC_B", new AppSummary("SVC_B", "SVC_B", 4, 4, 4, 1024, Map.of(), + ApplicationState.RUNNING, new Date(), new Date()))); + + stubFor(get("/apis/v1/applications") + .withBasicAuth("guest", "guest") + .willReturn(okJson(MAPPER.writeValueAsString(response)))); + + val config = buildConfig(wm, "drove-svc-src-1"); + try (val droveClient = buildClient(wm, config)) { + val dataSource = new DroveServiceDataSource<>( + "drove-svc-src-1", config, MAPPER, "testns", droveClient); + + val services = dataSource.services(); + + assertNotNull(services); + assertEquals(2, services.size()); + + // Verify services fetch success + val successMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.DROVE.dataSource.drove-svc-src-1.services.fetch.success"); + assertNotNull(successMeter, "Services fetch success meter should be recorded"); + assertEquals(1, successMeter.getCount()); + + // No failure + assertNull(metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.DROVE.dataSource.drove-svc-src-1.services.fetch.failure")); + } + } + + // ==================== services() - Failure ==================== + + @Test + @SneakyThrows + void testServices_failure_recordsFetchFailure(WireMockRuntimeInfo wm) { + stubFor(get("/apis/v1/applications") + .withBasicAuth("guest", "guest") + .willReturn(aResponse().withStatus(500).withBody("Error"))); + + val config = buildConfig(wm, "drove-svc-src-2"); + try (val droveClient = buildClient(wm, config)) { + val dataSource = new DroveServiceDataSource<>( + "drove-svc-src-2", config, MAPPER, "testns", droveClient); + + assertThrows(DroveCommunicationException.class, dataSource::services); + + // Verify services fetch failure + val failureMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.DROVE.dataSource.drove-svc-src-2.services.fetch.failure"); + assertNotNull(failureMeter, "Services fetch failure meter should be recorded"); + assertEquals(1, failureMeter.getCount()); + } + } + + // ==================== isActive() - Healthy ==================== + + @Test + @SneakyThrows + void testIsActive_healthy_recordsActiveStatus(WireMockRuntimeInfo wm) { + // Stub services endpoint to make client healthy (upstreamAvailable defaults true) + val response = ApiResponse.success(Map.of()); + stubFor(get("/apis/v1/applications") + .withBasicAuth("guest", "guest") + .willReturn(okJson(MAPPER.writeValueAsString(response)))); + + val config = buildConfig(wm, "drove-svc-src-3"); + try (val droveClient = buildClient(wm, config)) { + val dataSource = new DroveServiceDataSource<>( + "drove-svc-src-3", config, MAPPER, "testns", droveClient); + + val active = dataSource.isActive(); + + assertTrue(active); + + // Verify active status + val activeMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataSource.DROVE.drove-svc-src-3.active"); + assertNotNull(activeMeter, "Active status meter should be recorded"); + assertEquals(1, activeMeter.getCount()); + } + } + + // ==================== isActive() - Unhealthy (after failed call) ==================== + + @Test + @SneakyThrows + void testIsActive_afterFailedCall_recordsInactiveStatus(WireMockRuntimeInfo wm) { + stubFor(get("/apis/v1/applications") + .withBasicAuth("guest", "guest") + .willReturn(aResponse().withStatus(500).withBody("Error"))); + + val config = buildConfig(wm, "drove-svc-src-4"); + try (val droveClient = buildClient(wm, config)) { + val dataSource = new DroveServiceDataSource<>( + "drove-svc-src-4", config, MAPPER, "testns", droveClient); + + // First call fails and sets upstreamAvailable to false + assertThrows(DroveCommunicationException.class, dataSource::services); + + val active = dataSource.isActive(); + + assertFalse(active); + + // Verify inactive status + val inactiveMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataSource.DROVE.drove-svc-src-4.inactive"); + assertNotNull(inactiveMeter, "Inactive status meter should be recorded"); + assertEquals(1, inactiveMeter.getCount()); + } + } + + // ==================== Helpers ==================== + + private DroveUpstreamConfig buildConfig(WireMockRuntimeInfo wm, String metricId) { + return DroveUpstreamConfig.builder() + .id(metricId) + .endpoints(List.of("http://localhost:" + wm.getHttpPort())) + .username("guest") + .password("guest") + .skipCaching(true) + .build(); + } + + private DroveCommunicator buildClient(WireMockRuntimeInfo wm, DroveUpstreamConfig config) { + return RangerDroveUtils.buildDroveClient("testns", config, MAPPER); + } +} From 9fbfe1c2019ccff0fd44d68606288c20d239f079 Mon Sep 17 00:00:00 2001 From: Jitendra Dhawan Date: Sun, 31 May 2026 19:22:31 +0530 Subject: [PATCH 10/24] Add integration tests for HttpApiCommunicator and HttpServiceDataSource metrics --- ...ApiCommunicatorMetricsIntegrationTest.java | 364 ++++++++++++++++++ ...rviceDataSourceMetricsIntegrationTest.java | 147 +++++++ 2 files changed, 511 insertions(+) create mode 100644 ranger-http/src/test/java/io/appform/ranger/http/servicefinder/HttpApiCommunicatorMetricsIntegrationTest.java create mode 100644 ranger-http/src/test/java/io/appform/ranger/http/servicefinderhub/HttpServiceDataSourceMetricsIntegrationTest.java diff --git a/ranger-http/src/test/java/io/appform/ranger/http/servicefinder/HttpApiCommunicatorMetricsIntegrationTest.java b/ranger-http/src/test/java/io/appform/ranger/http/servicefinder/HttpApiCommunicatorMetricsIntegrationTest.java new file mode 100644 index 00000000..0c23997f --- /dev/null +++ b/ranger-http/src/test/java/io/appform/ranger/http/servicefinder/HttpApiCommunicatorMetricsIntegrationTest.java @@ -0,0 +1,364 @@ +/* + * Copyright 2024 Authors, Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.appform.ranger.http.servicefinder; + +import com.codahale.metrics.MetricRegistry; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo; +import com.github.tomakehurst.wiremock.junit5.WireMockTest; +import io.appform.ranger.core.model.Service; +import io.appform.ranger.core.model.ServiceNode; +import io.appform.ranger.core.util.MetricRecorder; +import io.appform.ranger.http.config.HttpClientConfig; +import io.appform.ranger.http.model.ServiceDataSourceResponse; +import io.appform.ranger.http.model.ServiceNodesResponse; +import io.appform.ranger.http.serde.HTTPResponseDataDeserializer; +import io.appform.ranger.http.utils.RangerHttpUtils; +import lombok.val; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.List; +import java.util.Set; + +import static com.github.tomakehurst.wiremock.client.WireMock.*; +import static org.junit.jupiter.api.Assertions.*; + +@WireMockTest +class HttpApiCommunicatorMetricsIntegrationTest { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + private static final String METRIC_PREFIX = "io.appform.ranger"; + private MetricRegistry metricRegistry; + + @BeforeEach + void setUp() { + metricRegistry = new MetricRegistry(); + MetricRecorder.initialize(metricRegistry); + } + + // ==================== services() - Success ==================== + + @Test + void testServices_success_recordsStatusCodeAndNoFailure(WireMockRuntimeInfo wmInfo) throws Exception { + val responseObj = ServiceDataSourceResponse.builder() + .data(Set.of(new Service("ns", "svc1"), new Service("ns", "svc2"))) + .build(); + stubFor(get(urlPathEqualTo("/ranger/services/v1")) + .willReturn(aResponse() + .withBody(MAPPER.writeValueAsBytes(responseObj)) + .withStatus(200))); + + val config = buildConfig(wmInfo, "http-metric-1"); + val communicator = RangerHttpUtils.httpClient(config, MAPPER); + val services = communicator.services(); + + assertNotNull(services); + assertEquals(2, services.size()); + + // Verify status code 200 meter + val statusMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.HTTP.dataSource.http-metric-1.httpCall.services.responseStatus.200"); + assertNotNull(statusMeter, "200 status meter should be recorded"); + assertEquals(1, statusMeter.getCount()); + + // No unknown failure + assertNull(metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.HTTP.dataSource.http-metric-1.httpCall.services.unknownFailure")); + + // No parse failure + assertNull(metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.HTTP.dataSource.http-metric-1.httpCall.services.responseParseFailure")); + } + + // ==================== services() - Empty response ==================== + + @Test + void testServices_emptyData_recordsNullOrEmptyServicesResponse(WireMockRuntimeInfo wmInfo) throws Exception { + val responseObj = ServiceDataSourceResponse.builder() + .data(Set.of()) // empty set + .build(); + stubFor(get(urlPathEqualTo("/ranger/services/v1")) + .willReturn(aResponse() + .withBody(MAPPER.writeValueAsBytes(responseObj)) + .withStatus(200))); + + val config = buildConfig(wmInfo, "http-metric-2"); + val communicator = RangerHttpUtils.httpClient(config, MAPPER); + + // Empty set is valid (data != null), so services() returns successfully with empty set + val services = communicator.services(); + assertNotNull(services); + assertTrue(services.isEmpty()); + + // Verify null/empty services meter is still recorded + val emptyMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.HTTP.dataSource.http-metric-2.httpCall.services.nullOrEmptyResponse"); + assertNotNull(emptyMeter, "Null/empty services meter should be recorded"); + assertEquals(1, emptyMeter.getCount()); + } + + // ==================== services() - Non-200 response ==================== + + @Test + void testServices_serverError_recordsStatusCodeAndUnknownFailure(WireMockRuntimeInfo wmInfo) { + stubFor(get(urlPathEqualTo("/ranger/services/v1")) + .willReturn(aResponse() + .withStatus(500) + .withBody("Internal Server Error"))); + + val config = buildConfig(wmInfo, "http-metric-3"); + val communicator = RangerHttpUtils.httpClient(config, MAPPER); + + assertThrows(Exception.class, communicator::services); + + // Verify 500 status code meter + val statusMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.HTTP.dataSource.http-metric-3.httpCall.services.responseStatus.500"); + assertNotNull(statusMeter, "500 status meter should be recorded"); + assertEquals(1, statusMeter.getCount()); + + // Verify unknown failure meter (HttpCommunicationException re-thrown) + val failureMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.HTTP.dataSource.http-metric-3.httpCall.services.unknownFailure"); + assertNotNull(failureMeter, "Unknown failure meter should be recorded"); + assertTrue(failureMeter.getCount() >= 1); + } + + // ==================== services() - Invalid JSON (parse failure) ==================== + + @Test + void testServices_invalidJson_recordsParseFailure(WireMockRuntimeInfo wmInfo) { + stubFor(get(urlPathEqualTo("/ranger/services/v1")) + .willReturn(aResponse() + .withBody("not-valid-json{{{") + .withStatus(200))); + + val config = buildConfig(wmInfo, "http-metric-4"); + val communicator = RangerHttpUtils.httpClient(config, MAPPER); + + assertThrows(Exception.class, communicator::services); + + // Verify parse failure meter + val parseMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.HTTP.dataSource.http-metric-4.httpCall.services.responseParseFailure"); + assertNotNull(parseMeter, "Services parse failure meter should be recorded"); + assertEquals(1, parseMeter.getCount()); + } + + // ==================== services() - Connection refused (unknown failure) ==================== + + @Test + void testServices_connectionRefused_recordsUnknownFailure() { + // Use a port that is not listening + val config = HttpClientConfig.builder() + .id("http-metric-5") + .host("127.0.0.1") + .port(19999) // unlikely to have something listening + .connectionTimeoutMs(500) + .operationTimeoutMs(500) + .build(); + val communicator = RangerHttpUtils.httpClient(config, MAPPER); + + assertThrows(Exception.class, communicator::services); + + // Verify unknown failure meter + val failureMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.HTTP.dataSource.http-metric-5.httpCall.services.unknownFailure"); + assertNotNull(failureMeter, "Unknown failure meter should be recorded on connection error"); + assertEquals(1, failureMeter.getCount()); + } + + // ==================== listNodes() - Success ==================== + + @Test + void testListNodes_success_recordsStatusCode(WireMockRuntimeInfo wmInfo) throws Exception { + val nodeResponse = ServiceNodesResponse.builder() + .data(List.of( + ServiceNode.builder().host("h1").port(8080).nodeData("data1").build(), + ServiceNode.builder().host("h2").port(8081).nodeData("data2").build() + )) + .build(); + stubFor(get(urlPathEqualTo("/ranger/nodes/v1/test-ns/test-svc")) + .willReturn(aResponse() + .withBody(MAPPER.writeValueAsBytes(nodeResponse)) + .withStatus(200))); + + val config = buildConfig(wmInfo, "http-metric-6"); + val communicator = RangerHttpUtils.httpClient(config, MAPPER); + val service = new Service("test-ns", "test-svc"); + + HTTPResponseDataDeserializer deserializer = data -> { + try { + return MAPPER.readValue(data, MAPPER.getTypeFactory() + .constructParametricType(ServiceNodesResponse.class, String.class)); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }; + + val nodes = communicator.listNodes(service, deserializer); + + assertNotNull(nodes); + assertEquals(2, nodes.size()); + + // Verify 200 status code for listNodes + val statusMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.HTTP.dataSource.http-metric-6.httpCall.listNodes.responseStatus.200"); + assertNotNull(statusMeter, "200 status meter for listNodes should exist"); + assertEquals(1, statusMeter.getCount()); + } + + // ==================== listNodes() - Server error ==================== + + @Test + void testListNodes_serverError_recordsStatusCodeAndUnknownFailure(WireMockRuntimeInfo wmInfo) { + stubFor(get(urlPathEqualTo("/ranger/nodes/v1/test-ns/error-svc")) + .willReturn(aResponse() + .withStatus(503) + .withBody("Service Unavailable"))); + + val config = buildConfig(wmInfo, "http-metric-7"); + val communicator = RangerHttpUtils.httpClient(config, MAPPER); + val service = new Service("test-ns", "error-svc"); + + HTTPResponseDataDeserializer deserializer = data -> { + try { + return MAPPER.readValue(data, MAPPER.getTypeFactory() + .constructParametricType(ServiceNodesResponse.class, String.class)); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }; + + assertThrows(Exception.class, () -> communicator.listNodes(service, deserializer)); + + // Verify 503 status code + val statusMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.HTTP.dataSource.http-metric-7.httpCall.listNodes.responseStatus.503"); + assertNotNull(statusMeter, "503 status meter for listNodes should exist"); + assertEquals(1, statusMeter.getCount()); + + // Verify unknown failure (with service name) + val failureMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.HTTP.dataSource.http-metric-7.httpCall.listNodes.unknownFailure"); + assertNotNull(failureMeter, "Unknown failure meter for listNodes should exist"); + assertTrue(failureMeter.getCount() >= 1); + } + + // ==================== listNodes() - Empty body ==================== + + @Test + void testListNodes_emptyBody_recordsNullOrEmptyListNodeResponse(WireMockRuntimeInfo wmInfo) { + stubFor(get(urlPathEqualTo("/ranger/nodes/v1/test-ns/empty-svc")) + .willReturn(aResponse() + .withStatus(200) + .withBody(""))); // empty body + + val config = buildConfig(wmInfo, "http-metric-8"); + val communicator = RangerHttpUtils.httpClient(config, MAPPER); + val service = new Service("test-ns", "empty-svc"); + + HTTPResponseDataDeserializer deserializer = data -> { + try { + return MAPPER.readValue(data, MAPPER.getTypeFactory() + .constructParametricType(ServiceNodesResponse.class, String.class)); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }; + + assertThrows(Exception.class, () -> communicator.listNodes(service, deserializer)); + + // The 200 status code should be recorded + val statusMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.HTTP.dataSource.http-metric-8.httpCall.listNodes.responseStatus.200"); + assertNotNull(statusMeter, "200 status meter should exist even for empty body"); + assertEquals(1, statusMeter.getCount()); + } + + // ==================== listNodes() - Invalid JSON (parse failure) ==================== + + @Test + void testListNodes_invalidJson_recordsParseFailure(WireMockRuntimeInfo wmInfo) { + stubFor(get(urlPathEqualTo("/ranger/nodes/v1/test-ns/parse-fail-svc")) + .willReturn(aResponse() + .withStatus(200) + .withBody("{{invalid json}}"))); + + val config = buildConfig(wmInfo, "http-metric-9"); + val communicator = RangerHttpUtils.httpClient(config, MAPPER); + val service = new Service("test-ns", "parse-fail-svc"); + + HTTPResponseDataDeserializer deserializer = data -> { + throw new RuntimeException("Parse error simulation"); + }; + + assertThrows(Exception.class, () -> communicator.listNodes(service, deserializer)); + + // The listNodes parse failure meter should exist + val parseMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.HTTP.dataSource.http-metric-9.httpCall.listNodes.serviceName.parse-fail-svc.responseParseFailure"); + assertNotNull(parseMeter, "List nodes parse failure meter should be recorded"); + assertEquals(1, parseMeter.getCount()); + } + + // ==================== listNodes() - Connection refused ==================== + + @Test + void testListNodes_connectionRefused_recordsUnknownFailureWithServiceName() { + val config = HttpClientConfig.builder() + .id("http-metric-10") + .host("127.0.0.1") + .port(19998) + .connectionTimeoutMs(500) + .operationTimeoutMs(500) + .build(); + val communicator = RangerHttpUtils.httpClient(config, MAPPER); + val service = new Service("test-ns", "conn-fail-svc"); + + HTTPResponseDataDeserializer deserializer = data -> + ServiceNodesResponse.builder().data(List.of()).build(); + + assertThrows(Exception.class, () -> communicator.listNodes(service, deserializer)); + + // Verify unknown failure meter (generic) + val genericFailureMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.HTTP.dataSource.http-metric-10.httpCall.listNodes.unknownFailure"); + assertNotNull(genericFailureMeter, "Generic unknown failure meter should exist"); + assertEquals(1, genericFailureMeter.getCount()); + + // Verify service-specific unknown failure meter + val svcFailureMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.HTTP.dataSource.http-metric-10.httpCall.listNodes.serviceName.conn-fail-svc.unknownFailure"); + assertNotNull(svcFailureMeter, "Service-specific unknown failure meter should exist"); + assertEquals(1, svcFailureMeter.getCount()); + } + + // ==================== Helper ==================== + + private HttpClientConfig buildConfig(WireMockRuntimeInfo wmInfo, String metricId) { + return HttpClientConfig.builder() + .id(metricId) + .host("127.0.0.1") + .port(wmInfo.getHttpPort()) + .connectionTimeoutMs(30_000) + .operationTimeoutMs(30_000) + .build(); + } +} diff --git a/ranger-http/src/test/java/io/appform/ranger/http/servicefinderhub/HttpServiceDataSourceMetricsIntegrationTest.java b/ranger-http/src/test/java/io/appform/ranger/http/servicefinderhub/HttpServiceDataSourceMetricsIntegrationTest.java new file mode 100644 index 00000000..7b020184 --- /dev/null +++ b/ranger-http/src/test/java/io/appform/ranger/http/servicefinderhub/HttpServiceDataSourceMetricsIntegrationTest.java @@ -0,0 +1,147 @@ +/* + * Copyright 2024 Authors, Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.appform.ranger.http.servicefinderhub; + +import com.codahale.metrics.MetricRegistry; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.github.tomakehurst.wiremock.junit5.WireMockRuntimeInfo; +import com.github.tomakehurst.wiremock.junit5.WireMockTest; +import io.appform.ranger.core.model.Service; +import io.appform.ranger.core.util.MetricRecorder; +import io.appform.ranger.http.config.HttpClientConfig; +import io.appform.ranger.http.model.ServiceDataSourceResponse; +import io.appform.ranger.http.utils.RangerHttpUtils; +import lombok.val; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.Set; + +import static com.github.tomakehurst.wiremock.client.WireMock.*; +import static org.junit.jupiter.api.Assertions.*; + +@WireMockTest +class HttpServiceDataSourceMetricsIntegrationTest { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + private static final String METRIC_PREFIX = "io.appform.ranger"; + private MetricRegistry metricRegistry; + + @BeforeEach + void setUp() { + metricRegistry = new MetricRegistry(); + MetricRecorder.initialize(metricRegistry); + } + + @Test + void testServices_success_recordsServicesFetchSuccess(WireMockRuntimeInfo wmInfo) throws Exception { + val responseObj = ServiceDataSourceResponse.builder() + .data(Set.of(new Service("ns", "svc1"))) + .build(); + stubFor(get(urlPathEqualTo("/ranger/services/v1")) + .willReturn(aResponse() + .withBody(MAPPER.writeValueAsBytes(responseObj)) + .withStatus(200))); + + val config = buildConfig(wmInfo, "sds-metric-1"); + val httpServiceDataSource = new HttpServiceDataSource<>( + "sds-metric-1", config, RangerHttpUtils.httpClient(config, MAPPER)); + val services = httpServiceDataSource.services(); + + assertNotNull(services); + assertFalse(services.isEmpty()); + + // Verify services fetch success meter + val successMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.HTTP.dataSource.sds-metric-1.services.fetch.success"); + assertNotNull(successMeter, "Services fetch success meter should exist"); + assertEquals(1, successMeter.getCount()); + + // No failure + assertNull(metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.HTTP.dataSource.sds-metric-1.services.fetch.failure")); + } + + @Test + void testServices_failure_recordsServicesFetchFailure(WireMockRuntimeInfo wmInfo) { + stubFor(get(urlPathEqualTo("/ranger/services/v1")) + .willReturn(aResponse() + .withStatus(500) + .withBody("error"))); + + val config = buildConfig(wmInfo, "sds-metric-2"); + val httpServiceDataSource = new HttpServiceDataSource<>( + "sds-metric-2", config, RangerHttpUtils.httpClient(config, MAPPER)); + + assertThrows(Exception.class, httpServiceDataSource::services); + + // Verify services fetch failure meter + val failureMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.HTTP.dataSource.sds-metric-2.services.fetch.failure"); + assertNotNull(failureMeter, "Services fetch failure meter should exist"); + assertEquals(1, failureMeter.getCount()); + + // No success + assertNull(metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.HTTP.dataSource.sds-metric-2.services.fetch.success")); + } + + @Test + void testServices_connectionRefused_recordsServicesFetchFailure() { + val config = HttpClientConfig.builder() + .id("sds-metric-3") + .host("127.0.0.1") + .port(19997) // not listening + .connectionTimeoutMs(500) + .operationTimeoutMs(500) + .build(); + val httpServiceDataSource = new HttpServiceDataSource<>( + "sds-metric-3", config, RangerHttpUtils.httpClient(config, MAPPER)); + + assertThrows(Exception.class, httpServiceDataSource::services); + + val failureMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.HTTP.dataSource.sds-metric-3.services.fetch.failure"); + assertNotNull(failureMeter, "Services fetch failure meter should exist on connection error"); + assertEquals(1, failureMeter.getCount()); + } + + @Test + void testIsActive_recordsDataSourceStatus(WireMockRuntimeInfo wmInfo) { + val config = buildConfig(wmInfo, "sds-metric-4"); + val httpServiceDataSource = new HttpServiceDataSource<>( + "sds-metric-4", config, RangerHttpUtils.httpClient(config, MAPPER)); + + val active = httpServiceDataSource.isActive(); + assertTrue(active, "HTTP data source should always be active"); + + // HttpNodeDataStoreConnector.isActive() records status with config.getId() + val statusMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataSource.HTTP.sds-metric-4.active"); + assertNotNull(statusMeter, "Active status meter should be recorded"); + assertEquals(1, statusMeter.getCount()); + } + + private HttpClientConfig buildConfig(WireMockRuntimeInfo wmInfo, String metricId) { + return HttpClientConfig.builder() + .id(metricId) + .host("127.0.0.1") + .port(wmInfo.getHttpPort()) + .connectionTimeoutMs(30_000) + .operationTimeoutMs(30_000) + .build(); + } +} From 2cd0e2261f27195c82b1284610ec89f241b6e71e Mon Sep 17 00:00:00 2001 From: Jitendra Dhawan Date: Sun, 31 May 2026 19:22:42 +0530 Subject: [PATCH 11/24] Add integration tests for ZkNodeDataSink and ZkNodeDataSource metrics recording --- ...kNodeDataSourceMetricsIntegrationTest.java | 294 ++++++++++++++++++ ...rviceDataSourceMetricsIntegrationTest.java | 195 ++++++++++++ .../ZkNodeDataSinkMetricsIntegrationTest.java | 240 ++++++++++++++ 3 files changed, 729 insertions(+) create mode 100644 ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/servicefinder/ZkNodeDataSourceMetricsIntegrationTest.java create mode 100644 ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/servicefinderhub/ZkServiceDataSourceMetricsIntegrationTest.java create mode 100644 ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/serviceprovider/ZkNodeDataSinkMetricsIntegrationTest.java diff --git a/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/servicefinder/ZkNodeDataSourceMetricsIntegrationTest.java b/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/servicefinder/ZkNodeDataSourceMetricsIntegrationTest.java new file mode 100644 index 00000000..44afa554 --- /dev/null +++ b/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/servicefinder/ZkNodeDataSourceMetricsIntegrationTest.java @@ -0,0 +1,294 @@ +/* + * Copyright 2024 Authors, Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.appform.ranger.zookeeper.servicefinder; + +import com.codahale.metrics.MetricRegistry; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.appform.ranger.core.model.Service; +import io.appform.ranger.core.model.ServiceNode; +import io.appform.ranger.core.units.TestNodeData; +import io.appform.ranger.core.util.MetricRecorder; +import io.appform.ranger.zookeeper.serde.ZkNodeDataDeserializer; +import io.appform.ranger.zookeeper.util.PathBuilder; +import lombok.SneakyThrows; +import lombok.val; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.retry.ExponentialBackoffRetry; +import org.apache.curator.test.TestingCluster; +import org.apache.zookeeper.CreateMode; +import org.junit.jupiter.api.*; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Integration tests for ZkNodeDataSource metrics recording. + * Tests verify that metrics are pushed through actual ZK operations. + */ +class ZkNodeDataSourceMetricsIntegrationTest { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + private static final String METRIC_PREFIX = "io.appform.ranger"; + private static final String NAMESPACE = "test"; + private static final String SERVICE_NAME = "node-source-svc"; + + private TestingCluster testingCluster; + private CuratorFramework curatorFramework; + private MetricRegistry metricRegistry; + + @BeforeEach + void setUp() throws Exception { + metricRegistry = new MetricRegistry(); + MetricRecorder.initialize(metricRegistry); + + testingCluster = new TestingCluster(3); + testingCluster.start(); + + curatorFramework = CuratorFrameworkFactory.builder() + .namespace(NAMESPACE) + .connectString(testingCluster.getConnectString()) + .retryPolicy(new ExponentialBackoffRetry(1000, 3)) + .build(); + curatorFramework.start(); + curatorFramework.blockUntilConnected(); + } + + @AfterEach + void tearDown() throws Exception { + if (curatorFramework != null) { + curatorFramework.close(); + } + if (testingCluster != null) { + testingCluster.close(); + } + } + + // ==================== isActive() - Active connection ==================== + + @Test + void testIsActive_connectedZk_recordsActiveStatus() { + val service = new Service(NAMESPACE, SERVICE_NAME); + val dataSource = new ZkNodeDataSource>( + "zk-node-src-1", service, curatorFramework); + dataSource.start(); + + val active = dataSource.isActive(); + + assertTrue(active); + + // Verify active status metric + val activeMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataSource.ZK.zk-node-src-1.active"); + assertNotNull(activeMeter, "Active status meter should be recorded"); + assertEquals(1, activeMeter.getCount()); + + dataSource.stop(); + } + + // ==================== isActive() - Disconnected ==================== + + @Test + void testIsActive_disconnectedZk_recordsInactiveStatus() throws Exception { + val service = new Service(NAMESPACE, SERVICE_NAME); + val disconnectedCurator = CuratorFrameworkFactory.builder() + .namespace(NAMESPACE) + .connectString("127.0.0.1:19999") // non-existent ZK + .retryPolicy(new ExponentialBackoffRetry(100, 1)) + .sessionTimeoutMs(500) + .connectionTimeoutMs(500) + .build(); + disconnectedCurator.start(); + // Don't wait for connection — it should fail + + val dataSource = new ZkNodeDataSource>( + "zk-node-src-2", service, disconnectedCurator); + // Don't call start() — it would block waiting for connection + + val active = dataSource.isActive(); + + assertFalse(active); + + // Verify inactive status metric + val inactiveMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataSource.ZK.zk-node-src-2.inactive"); + assertNotNull(inactiveMeter, "Inactive status meter should be recorded"); + assertEquals(1, inactiveMeter.getCount()); + + disconnectedCurator.close(); + } + + // ==================== refresh() - Success with nodes ==================== + + @Test + void testRefresh_withNodes_noEmptyMetric() throws Exception { + val service = new Service(NAMESPACE, SERVICE_NAME); + val dataSource = new ZkNodeDataSource>( + "zk-node-src-3", service, curatorFramework); + dataSource.start(); + + // Create service path and child nodes in ZK + val servicePath = PathBuilder.servicePath(service); + curatorFramework.create().creatingParentContainersIfNeeded().forPath(servicePath); + + val node1 = ServiceNode.builder() + .host("host1").port(8080) + .nodeData(TestNodeData.builder().shardId(1).build()) + .build(); + val nodeData1 = MAPPER.writeValueAsBytes(node1); + curatorFramework.create().withMode(CreateMode.EPHEMERAL) + .forPath(servicePath + "/" + node1.representation(), nodeData1); + + val result = dataSource.refresh(validDeserializer()); + + assertTrue(result.isPresent()); + assertEquals(1, result.get().size()); + assertEquals("host1", result.get().get(0).getHost()); + + // No null/empty list node response metric should be recorded + val emptyMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.ZK.dataSource.zk-node-src-3.httpCall.listNodes.serviceName." + + SERVICE_NAME + ".nullOrEmptyResponse"); + assertNull(emptyMeter, "No empty list node metric should be recorded when nodes exist"); + + dataSource.stop(); + } + + // ==================== refresh() - Empty children ==================== + + @Test + void testRefresh_emptyChildren_recordsNullOrEmptyListNodeResponse() throws Exception { + val service = new Service(NAMESPACE, SERVICE_NAME); + val dataSource = new ZkNodeDataSource>( + "zk-node-src-4", service, curatorFramework); + dataSource.start(); + + // Create service path but no child nodes + val servicePath = PathBuilder.servicePath(service); + curatorFramework.create().creatingParentContainersIfNeeded().forPath(servicePath); + + val result = dataSource.refresh(validDeserializer()); + + assertTrue(result.isPresent()); + assertTrue(result.get().isEmpty()); + + // Verify null/empty list node response metric + val emptyMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.ZK.dataSource.zk-node-src-4.httpCall.listNodes.serviceName." + + SERVICE_NAME + ".nullOrEmptyResponse"); + assertNotNull(emptyMeter, "Empty list node meter should be recorded"); + assertEquals(1, emptyMeter.getCount()); + + dataSource.stop(); + } + + // ==================== refresh() - NoNodeException (path doesn't exist) ==================== + + @Test + void testRefresh_noServicePath_recordsNullOrEmptyListNodeResponse() throws Exception { + val service = new Service(NAMESPACE, "nonexistent-svc"); + val dataSource = new ZkNodeDataSource>( + "zk-node-src-5", service, curatorFramework); + dataSource.start(); + + // Don't create the service path — triggers NoNodeException + + val result = dataSource.refresh(validDeserializer()); + + assertTrue(result.isPresent()); + assertTrue(result.get().isEmpty()); + + // Verify null/empty list node response metric (NoNodeException path) + val emptyMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.ZK.dataSource.zk-node-src-5.httpCall.listNodes.serviceName.nonexistent-svc.nullOrEmptyResponse"); + assertNotNull(emptyMeter, "Empty list node meter should be recorded for NoNodeException"); + assertTrue(emptyMeter.getCount() >= 1); + + dataSource.stop(); + } + + // ==================== refresh() - Deserializer failure (parse failure) ==================== + + @Test + void testRefresh_deserializerThrows_recordsListNodesParseFailure() throws Exception { + val service = new Service(NAMESPACE, SERVICE_NAME); + val dataSource = new ZkNodeDataSource>( + "zk-node-src-6", service, curatorFramework); + dataSource.start(); + + // Create service path with a child node containing bad data + val servicePath = PathBuilder.servicePath(service); + curatorFramework.create().creatingParentContainersIfNeeded().forPath(servicePath); + curatorFramework.create().withMode(CreateMode.EPHEMERAL) + .forPath(servicePath + "/bad-node", "invalid-json{{{".getBytes()); + + // Deserializer that always throws + ZkNodeDataDeserializer badDeserializer = data -> { + throw new RuntimeException("Parse failure simulation"); + }; + + assertThrows(RuntimeException.class, () -> dataSource.refresh(badDeserializer)); + + // Verify list nodes parse failure metric + val parseMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.ZK.dataSource.zk-node-src-6.httpCall.listNodes.serviceName." + + SERVICE_NAME + ".responseParseFailure"); + assertNotNull(parseMeter, "List nodes parse failure meter should be recorded"); + assertEquals(1, parseMeter.getCount()); + + dataSource.stop(); + } + + // ==================== refresh() - Not started ==================== + + @Test + void testRefresh_notStarted_returnsEmpty() { + val service = new Service(NAMESPACE, SERVICE_NAME); + val dataSource = new ZkNodeDataSource>( + "zk-node-src-7", service, curatorFramework); + // Intentionally do NOT call start() + + val result = dataSource.refresh(validDeserializer()); + + assertFalse(result.isPresent()); + } + + // ==================== refresh() - Stopped ==================== + + @Test + void testRefresh_stopped_returnsEmpty() { + val service = new Service(NAMESPACE, SERVICE_NAME); + val dataSource = new ZkNodeDataSource>( + "zk-node-src-8", service, curatorFramework); + dataSource.start(); + dataSource.stop(); + + val result = dataSource.refresh(validDeserializer()); + + assertFalse(result.isPresent()); + } + + // ==================== Helper ==================== + + @SneakyThrows + private static ServiceNode deserializeNode(byte[] data) { + return MAPPER.readValue(data, new TypeReference>() {}); + } + + private ZkNodeDataDeserializer validDeserializer() { + return ZkNodeDataSourceMetricsIntegrationTest::deserializeNode; + } +} diff --git a/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/servicefinderhub/ZkServiceDataSourceMetricsIntegrationTest.java b/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/servicefinderhub/ZkServiceDataSourceMetricsIntegrationTest.java new file mode 100644 index 00000000..04b04920 --- /dev/null +++ b/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/servicefinderhub/ZkServiceDataSourceMetricsIntegrationTest.java @@ -0,0 +1,195 @@ +/* + * Copyright 2024 Authors, Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.appform.ranger.zookeeper.servicefinderhub; + +import com.codahale.metrics.MetricRegistry; +import io.appform.ranger.core.util.MetricRecorder; +import lombok.val; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.retry.ExponentialBackoffRetry; +import org.apache.curator.test.TestingCluster; +import org.junit.jupiter.api.*; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Integration tests for ZkServiceDataSource metrics recording. + * Tests verify that metrics are pushed through actual ZK operations. + */ +class ZkServiceDataSourceMetricsIntegrationTest { + + private static final String METRIC_PREFIX = "io.appform.ranger"; + private static final String NAMESPACE = "test"; + + private TestingCluster testingCluster; + private CuratorFramework curatorFramework; + private MetricRegistry metricRegistry; + + @BeforeEach + void setUp() throws Exception { + metricRegistry = new MetricRegistry(); + MetricRecorder.initialize(metricRegistry); + + testingCluster = new TestingCluster(3); + testingCluster.start(); + + curatorFramework = CuratorFrameworkFactory.builder() + .namespace(NAMESPACE) + .connectString(testingCluster.getConnectString()) + .retryPolicy(new ExponentialBackoffRetry(1000, 3)) + .build(); + curatorFramework.start(); + curatorFramework.blockUntilConnected(); + } + + @AfterEach + void tearDown() throws Exception { + if (curatorFramework != null) { + curatorFramework.close(); + } + if (testingCluster != null) { + testingCluster.close(); + } + } + + // ==================== services() - Success with children ==================== + + @Test + void testServices_withChildren_recordsSuccessStatus() throws Exception { + // Create some child nodes under "/" (the registered services path) + curatorFramework.create().creatingParentContainersIfNeeded().forPath("/service-a"); + curatorFramework.create().creatingParentContainersIfNeeded().forPath("/service-b"); + + val dataSource = new ZkServiceDataSource( + "zk-svc-src-1", NAMESPACE, testingCluster.getConnectString(), curatorFramework); + dataSource.start(); + + val services = dataSource.services(); + + assertNotNull(services); + assertEquals(2, services.size()); + + // Verify success fetch status + val successMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.ZK.dataSource.zk-svc-src-1.services.fetch.success"); + assertNotNull(successMeter, "Services fetch success meter should be recorded"); + assertEquals(1, successMeter.getCount()); + + // No empty response metric + val emptyMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.ZK.dataSource.zk-svc-src-1.httpCall.services.nullOrEmptyResponse"); + assertNull(emptyMeter, "No null/empty services response meter should be recorded when children exist"); + + dataSource.stop(); + } + + // ==================== services() - Empty children ==================== + + @Test + void testServices_noChildren_recordsNullOrEmptyServicesAndSuccess() throws Exception { + // Use a fresh curator with a different namespace that has no children + val emptyCurator = CuratorFrameworkFactory.builder() + .namespace("empty-ns") + .connectString(testingCluster.getConnectString()) + .retryPolicy(new ExponentialBackoffRetry(1000, 3)) + .build(); + emptyCurator.start(); + emptyCurator.blockUntilConnected(); + + // Ensure root path exists but is empty + if (emptyCurator.checkExists().forPath("/") == null) { + emptyCurator.create().forPath("/"); + } + + val dataSource = new ZkServiceDataSource( + "zk-svc-src-2", "empty-ns", testingCluster.getConnectString(), emptyCurator); + dataSource.start(); + + val services = dataSource.services(); + + assertNotNull(services); + assertTrue(services.isEmpty()); + + // Verify null/empty services response metric + val emptyMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.ZK.dataSource.zk-svc-src-2.httpCall.services.nullOrEmptyResponse"); + assertNotNull(emptyMeter, "Null/empty services response meter should be recorded"); + assertEquals(1, emptyMeter.getCount()); + + // Verify success fetch status (still marked success even if empty) + val successMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.ZK.dataSource.zk-svc-src-2.services.fetch.success"); + assertNotNull(successMeter, "Services fetch success meter should still be recorded"); + assertEquals(1, successMeter.getCount()); + + emptyCurator.close(); + } + + // ==================== services() - ZK failure ==================== + + @Test + void testServices_zkFailure_recordsUnknownFailureAndFailureStatus() throws Exception { + val dataSource = new ZkServiceDataSource( + "zk-svc-src-3", NAMESPACE, testingCluster.getConnectString(), curatorFramework); + dataSource.start(); + + // Close the curator to simulate connection failure + curatorFramework.close(); + + assertThrows(Exception.class, dataSource::services); + + // Verify failure fetch status + val failureMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.ZK.dataSource.zk-svc-src-3.services.fetch.failure"); + assertNotNull(failureMeter, "Services fetch failure meter should be recorded"); + assertEquals(1, failureMeter.getCount()); + + // Verify ZK read unknown failure + val unknownFailureMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.ZK.dataSource.zk-svc-src-3.zkRead.services.unknownFailure"); + assertNotNull(unknownFailureMeter, "ZK read unknown failure meter should be recorded"); + assertEquals(1, unknownFailureMeter.getCount()); + + // Null out curatorFramework to prevent double-close in tearDown + curatorFramework = null; + } + + // ==================== services() - Success with provided curator ==================== + + @Test + void testServices_providedCurator_recordsSuccess() throws Exception { + // Create a service node + curatorFramework.create().creatingParentContainersIfNeeded().forPath("/my-service"); + + val dataSource = new ZkServiceDataSource( + "zk-svc-src-4", NAMESPACE, testingCluster.getConnectString(), curatorFramework); + dataSource.start(); + + val services = dataSource.services(); + + assertNotNull(services); + assertTrue(services.stream().anyMatch(s -> "my-service".equals(s.getServiceName()))); + + // Verify success + val successMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.ZK.dataSource.zk-svc-src-4.services.fetch.success"); + assertNotNull(successMeter, "Success meter should be recorded"); + assertEquals(1, successMeter.getCount()); + + dataSource.stop(); + } +} diff --git a/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/serviceprovider/ZkNodeDataSinkMetricsIntegrationTest.java b/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/serviceprovider/ZkNodeDataSinkMetricsIntegrationTest.java new file mode 100644 index 00000000..48a36263 --- /dev/null +++ b/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/serviceprovider/ZkNodeDataSinkMetricsIntegrationTest.java @@ -0,0 +1,240 @@ +/* + * Copyright 2024 Authors, Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.appform.ranger.zookeeper.serviceprovider; + +import com.codahale.metrics.MetricRegistry; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.appform.ranger.core.model.Service; +import io.appform.ranger.core.model.ServiceNode; +import io.appform.ranger.core.units.TestNodeData; +import io.appform.ranger.core.util.MetricRecorder; +import io.appform.ranger.zookeeper.serde.ZkNodeDataSerializer; +import lombok.val; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.retry.ExponentialBackoffRetry; +import org.apache.curator.test.TestingCluster; +import org.junit.jupiter.api.*; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Integration tests for ZkNodeDataSink metrics recording. + * Tests verify that metrics are pushed through actual ZK write operations. + */ +class ZkNodeDataSinkMetricsIntegrationTest { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + private static final String METRIC_PREFIX = "io.appform.ranger"; + private static final String NAMESPACE = "test"; + private static final String SERVICE_NAME = "sink-test-svc"; + + private TestingCluster testingCluster; + private CuratorFramework curatorFramework; + private MetricRegistry metricRegistry; + + @BeforeEach + void setUp() throws Exception { + metricRegistry = new MetricRegistry(); + MetricRecorder.initialize(metricRegistry); + + testingCluster = new TestingCluster(3); + testingCluster.start(); + + curatorFramework = CuratorFrameworkFactory.builder() + .namespace(NAMESPACE) + .connectString(testingCluster.getConnectString()) + .retryPolicy(new ExponentialBackoffRetry(1000, 3)) + .build(); + curatorFramework.start(); + curatorFramework.blockUntilConnected(); + } + + @AfterEach + void tearDown() throws Exception { + if (curatorFramework != null) { + curatorFramework.close(); + } + if (testingCluster != null) { + testingCluster.close(); + } + } + + // ==================== updateState() - Success (create new node) ==================== + + @Test + void testUpdateState_createNewNode_recordsSuccess() { + val service = new Service(NAMESPACE, SERVICE_NAME); + val sink = new ZkNodeDataSink>( + "zk-sink-1", service, curatorFramework); + sink.start(); + + val serviceNode = ServiceNode.builder() + .host("host1").port(8080) + .nodeData(TestNodeData.builder().shardId(1).build()) + .build(); + + ZkNodeDataSerializer serializer = node -> { + try { + return MAPPER.writeValueAsBytes(node); + } catch (Exception e) { + throw new RuntimeException(e); + } + }; + + sink.updateState(serializer, serviceNode); + + // Verify success meter + val successMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataSource.ZK.dataSource.zk-sink-1.nodeDataSink.update.success"); + assertNotNull(successMeter, "Node data sink update success meter should be recorded"); + assertEquals(1, successMeter.getCount()); + + // No failure + val failureMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataSource.ZK.dataSource.zk-sink-1.nodeDataSink.update.failure"); + assertNull(failureMeter, "No failure meter should be recorded on success"); + + sink.stop(); + } + + // ==================== updateState() - Success (update existing node) ==================== + + @Test + void testUpdateState_updateExistingNode_recordsSuccess() { + val service = new Service(NAMESPACE, SERVICE_NAME); + val sink = new ZkNodeDataSink>( + "zk-sink-2", service, curatorFramework); + sink.start(); + + val serviceNode = ServiceNode.builder() + .host("host2").port(8081) + .nodeData(TestNodeData.builder().shardId(2).build()) + .build(); + + ZkNodeDataSerializer serializer = node -> { + try { + return MAPPER.writeValueAsBytes(node); + } catch (Exception e) { + throw new RuntimeException(e); + } + }; + + // First call creates + sink.updateState(serializer, serviceNode); + // Second call updates + sink.updateState(serializer, serviceNode); + + // Verify 2 success markers + val successMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataSource.ZK.dataSource.zk-sink-2.nodeDataSink.update.success"); + assertNotNull(successMeter, "Success meter should exist"); + assertEquals(2, successMeter.getCount()); + + sink.stop(); + } + + // ==================== updateState() - Serialization failure ==================== + + @Test + void testUpdateState_serializerThrows_recordsSerDeFailure() { + val service = new Service(NAMESPACE, SERVICE_NAME); + val sink = new ZkNodeDataSink>( + "zk-sink-3", service, curatorFramework); + sink.start(); + + val serviceNode = ServiceNode.builder() + .host("host3").port(8082) + .nodeData(TestNodeData.builder().shardId(3).build()) + .build(); + + // Serializer that throws + ZkNodeDataSerializer badSerializer = node -> { + throw new RuntimeException("Serialization failed"); + }; + + // updateState should propagate the exception (wrapped in IllegalStateException) + assertThrows(Exception.class, () -> sink.updateState(badSerializer, serviceNode)); + + // Verify serialization failure metric + val serdeMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.ZK.dataSource.zk-sink-3.nodeDataSink.serialization.failure"); + assertNotNull(serdeMeter, "Serialization failure meter should be recorded"); + assertTrue(serdeMeter.getCount() >= 1); + + // Verify service-specific serialization failure + val svcSerdeMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.ZK.dataSource.zk-sink-3.nodeDataSink.serialization.serviceName." + + SERVICE_NAME + ".failure"); + assertNotNull(svcSerdeMeter, "Service-specific serialization failure meter should be recorded"); + assertTrue(svcSerdeMeter.getCount() >= 1); + + sink.stop(); + } + + // ==================== updateState() - Stopped state (no-op) ==================== + + @Test + void testUpdateState_stoppedSink_noMetricsRecorded() { + val service = new Service(NAMESPACE, SERVICE_NAME); + val sink = new ZkNodeDataSink>( + "zk-sink-4", service, curatorFramework); + sink.start(); + sink.stop(); + + val serviceNode = ServiceNode.builder() + .host("host4").port(8083) + .nodeData(TestNodeData.builder().shardId(4).build()) + .build(); + + ZkNodeDataSerializer serializer = node -> { + try { + return MAPPER.writeValueAsBytes(node); + } catch (Exception e) { + throw new RuntimeException(e); + } + }; + + // Should not throw, just return early + sink.updateState(serializer, serviceNode); + + // No success or failure metrics + assertNull(metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataSource.ZK.dataSource.zk-sink-4.nodeDataSink.update.success")); + assertNull(metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataSource.ZK.dataSource.zk-sink-4.nodeDataSink.update.failure")); + } + + // ==================== isActive() via ZkNodeDataStoreConnector base class ==================== + + @Test + void testIsActive_connectedCurator_recordsActiveStatus() { + val service = new Service(NAMESPACE, SERVICE_NAME); + val sink = new ZkNodeDataSink>( + "zk-sink-5", service, curatorFramework); + sink.start(); + + val active = sink.isActive(); + + assertTrue(active); + val activeMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataSource.ZK.zk-sink-5.active"); + assertNotNull(activeMeter, "Active meter should be recorded"); + assertEquals(1, activeMeter.getCount()); + + sink.stop(); + } +} From adbcf41420ca083a50b9ac23156cf080c4ab0077 Mon Sep 17 00:00:00 2001 From: Jitendra Dhawan Date: Sun, 31 May 2026 19:23:00 +0530 Subject: [PATCH 12/24] Add integration tests for FinderUtils and HealthChecker metrics recording --- ...RegistryUpdaterMetricsIntegrationTest.java | 337 ++++++++++++++++++ .../HealthCheckerMetricsIntegrationTest.java | 196 ++++++++++ .../FinderUtilsMetricsIntegrationTest.java | 179 ++++++++++ 3 files changed, 712 insertions(+) create mode 100644 ranger-core/src/test/java/io/appform/ranger/core/finder/serviceregistry/ServiceRegistryUpdaterMetricsIntegrationTest.java create mode 100644 ranger-core/src/test/java/io/appform/ranger/core/healthcheck/HealthCheckerMetricsIntegrationTest.java create mode 100644 ranger-core/src/test/java/io/appform/ranger/core/util/FinderUtilsMetricsIntegrationTest.java diff --git a/ranger-core/src/test/java/io/appform/ranger/core/finder/serviceregistry/ServiceRegistryUpdaterMetricsIntegrationTest.java b/ranger-core/src/test/java/io/appform/ranger/core/finder/serviceregistry/ServiceRegistryUpdaterMetricsIntegrationTest.java new file mode 100644 index 00000000..d18480d4 --- /dev/null +++ b/ranger-core/src/test/java/io/appform/ranger/core/finder/serviceregistry/ServiceRegistryUpdaterMetricsIntegrationTest.java @@ -0,0 +1,337 @@ +/* + * Copyright 2024 Authors, Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.appform.ranger.core.finder.serviceregistry; + +import com.codahale.metrics.MetricRegistry; +import io.appform.ranger.core.healthcheck.HealthcheckStatus; +import io.appform.ranger.core.model.*; +import io.appform.ranger.core.signals.Signal; +import io.appform.ranger.core.units.TestNodeData; +import io.appform.ranger.core.util.MetricRecorder; +import lombok.val; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Consumer; +import java.util.function.Supplier; + +import static org.junit.jupiter.api.Assertions.*; + +class ServiceRegistryUpdaterMetricsIntegrationTest { + + private MetricRegistry metricRegistry; + private static final String METRIC_ID = "test-updater-metric"; + private static final Service TEST_SERVICE = new Service("test-ns", "test-svc"); + + private ServiceRegistryUpdater updater; + + @BeforeEach + void setUp() { + metricRegistry = new MetricRegistry(); + MetricRecorder.initialize(metricRegistry); + } + + @AfterEach + void tearDown() { + if (updater != null) { + updater.stop(); + } + } + + @Test + void testSuccessfulRefresh_recordsSuccessTimer() { + val registry = new MapBasedServiceRegistry(TEST_SERVICE); + val nodes = List.of( + ServiceNode.builder() + .host("host1") + .port(8080) + .nodeData(TestNodeData.builder().shardId(1).build()) + .healthcheckStatus(HealthcheckStatus.healthy) + .lastUpdatedTimeStamp(System.currentTimeMillis()) + .build() + ); + val dataSource = new TestNodeDataSource(METRIC_ID, DataStoreType.ZK, true, nodes, false); + val signal = new TestSignal(); + + updater = new ServiceRegistryUpdater<>(registry, dataSource, List.of(signal), new TestDeserializer()); + updater.start(); + + // Wait for initial update and a brief period for metric recording to complete + awaitRefresh(registry); + sleep(100); // Allow time for MetricRecorder call after updateNodes + + // Verify success timer + val timerName = "io.appform.ranger.dataSource.ZK." + METRIC_ID + ".nodeDataRefresh.success"; + val timer = metricRegistry.getTimers().get(timerName); + assertNotNull(timer, "Node data refresh success timer should exist"); + assertTrue(timer.getCount() >= 1, "Timer should have at least 1 update"); + + // No failure timer + val failureTimerName = "io.appform.ranger.dataSource.ZK." + METRIC_ID + ".nodeDataRefresh.failure"; + val failureTimer = metricRegistry.getTimers().get(failureTimerName); + assertTrue(failureTimer == null || failureTimer.getCount() == 0, + "Failure timer should not be recorded on success"); + } + + @Test + void testRefreshThrowsException_recordsFailureTimer() { + val registry = new MapBasedServiceRegistry(TEST_SERVICE); + val dataSource = new TestNodeDataSource(METRIC_ID, DataStoreType.HTTP, true, null, true); + val signal = new TestSignal(); + + updater = new ServiceRegistryUpdater<>(registry, dataSource, List.of(signal), new TestDeserializer()); + // Use try-catch since initial update will fail + try { + updater.start(); + } catch (Exception e) { + // Expected: initial update fails + } + + // Give some time for the updater thread to process + sleep(200); + + // Verify failure timer + val failureTimerName = "io.appform.ranger.dataSource.HTTP." + METRIC_ID + ".nodeDataRefresh.failure"; + val failureTimer = metricRegistry.getTimers().get(failureTimerName); + assertNotNull(failureTimer, "Node data refresh failure timer should exist"); + assertTrue(failureTimer.getCount() >= 1, "Failure timer should have at least 1 update"); + } + + @Test + void testInactiveDataSource_recordsStaleDataRetained() { + val registry = new MapBasedServiceRegistry(TEST_SERVICE); + // Start with an active source so initial update succeeds + val nodes = List.of( + ServiceNode.builder() + .host("host1") + .port(8080) + .nodeData(TestNodeData.builder().shardId(1).build()) + .healthcheckStatus(HealthcheckStatus.healthy) + .lastUpdatedTimeStamp(System.currentTimeMillis()) + .build() + ); + val dataSource = new TestNodeDataSource(METRIC_ID, DataStoreType.DROVE, true, nodes, false); + val signal = new TestSignal(); + + updater = new ServiceRegistryUpdater<>(registry, dataSource, List.of(signal), new TestDeserializer()); + updater.start(); + awaitRefresh(registry); + + // Now deactivate the data source and trigger another update + dataSource.setActive(false); + signal.fire(); + sleep(200); + + // Verify stale data retained meter + val meterName = "io.appform.ranger.dataSource.DROVE." + METRIC_ID + ".staleDataRetained"; + val meter = metricRegistry.getMeters().get(meterName); + assertNotNull(meter, "Stale data retained meter should exist"); + assertTrue(meter.getCount() >= 1, "Stale data retained should be recorded at least once"); + } + + @Test + void testSuccessfulRefresh_zombieNodesFiltered_recordsZombieMetric() { + val registry = new MapBasedServiceRegistry(TEST_SERVICE); + val nodes = List.of( + ServiceNode.builder() + .host("healthy-host") + .port(8080) + .nodeData(TestNodeData.builder().shardId(1).build()) + .healthcheckStatus(HealthcheckStatus.healthy) + .lastUpdatedTimeStamp(System.currentTimeMillis()) + .build(), + ServiceNode.builder() + .host("zombie-host") + .port(8081) + .nodeData(TestNodeData.builder().shardId(2).build()) + .healthcheckStatus(HealthcheckStatus.healthy) + .lastUpdatedTimeStamp(0L) // Very old => zombie + .build() + ); + val dataSource = new TestNodeDataSource(METRIC_ID, DataStoreType.ZK, true, nodes, false); + val signal = new TestSignal(); + + updater = new ServiceRegistryUpdater<>(registry, dataSource, List.of(signal), new TestDeserializer()); + updater.start(); + awaitRefresh(registry); + + // Verify zombie metric recorded (from FinderUtils.filterValidNodes called inside updateRegistry) + val zombieMeter = metricRegistry.getMeters().get("io.appform.ranger.zombieNodes"); + assertNotNull(zombieMeter, "Zombie nodes meter should exist"); + assertTrue(zombieMeter.getCount() >= 1); + + val svcZombieMeter = metricRegistry.getMeters().get("io.appform.ranger.zombieNodes.serviceName.test-svc"); + assertNotNull(svcZombieMeter); + assertTrue(svcZombieMeter.getCount() >= 1); + + // Also verify success timer still recorded + val timerName = "io.appform.ranger.dataSource.ZK." + METRIC_ID + ".nodeDataRefresh.success"; + val timer = metricRegistry.getTimers().get(timerName); + assertNotNull(timer); + assertTrue(timer.getCount() >= 1); + } + + @Test + void testRefreshReturnsNull_noSuccessOrFailureTimer_butStaleRetained() { + val registry = new MapBasedServiceRegistry(TEST_SERVICE); + // Return null from refresh (empty Optional) + val dataSource = new TestNodeDataSource(METRIC_ID, DataStoreType.ZK, true, + null, false); // null nodes => refresh returns empty + // Pre-populate registry so initial wait doesn't block forever + registry.updateNodes(List.of( + ServiceNode.builder() + .host("old-host") + .port(8080) + .nodeData(TestNodeData.builder().shardId(1).build()) + .healthcheckStatus(HealthcheckStatus.healthy) + .lastUpdatedTimeStamp(System.currentTimeMillis()) + .build() + )); + + val signal = new TestSignal(); + updater = new ServiceRegistryUpdater<>(registry, dataSource, List.of(signal), new TestDeserializer()); + + // Cannot call start() because initial refresh returns null which won't set refreshed=true + // Instead, test the scenario after start by triggering signal + // Use alternate approach: make first call return valid, then switch to null + dataSource.setNodeList(List.of( + ServiceNode.builder() + .host("valid-host") + .port(8080) + .nodeData(TestNodeData.builder().shardId(1).build()) + .healthcheckStatus(HealthcheckStatus.healthy) + .lastUpdatedTimeStamp(System.currentTimeMillis()) + .build() + )); + + updater = new ServiceRegistryUpdater<>(registry, dataSource, List.of(signal), new TestDeserializer()); + updater.start(); + awaitRefresh(registry); + + // Now set to null and trigger update + dataSource.setNodeList(null); + signal.fire(); + sleep(200); + + // When refresh returns null, no success timer should be incremented for that second call + // The success timer from the first call should be 1 + val timerName = "io.appform.ranger.dataSource.ZK." + METRIC_ID + ".nodeDataRefresh.success"; + val timer = metricRegistry.getTimers().get(timerName); + assertNotNull(timer); + assertEquals(1, timer.getCount(), "Only the first successful refresh should be counted"); + } + + // ==================== Test helpers ==================== + + private void awaitRefresh(ServiceRegistry registry) { + val start = System.currentTimeMillis(); + while (!registry.isRefreshed() && (System.currentTimeMillis() - start) < 5000) { + sleep(50); + } + assertTrue(registry.isRefreshed(), "Registry should be refreshed within timeout"); + } + + private void sleep(long ms) { + try { + Thread.sleep(ms); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + // ==================== Test implementations ==================== + + static class TestDeserializer implements Deserializer { + } + + static class TestNodeDataSource implements NodeDataSource { + private final String metricId; + private final DataStoreType dataStoreType; + private final AtomicBoolean active; + private volatile List> nodeList; + private final boolean throwOnRefresh; + + TestNodeDataSource(String metricId, DataStoreType dataStoreType, boolean active, + List> nodeList, boolean throwOnRefresh) { + this.metricId = metricId; + this.dataStoreType = dataStoreType; + this.active = new AtomicBoolean(active); + this.nodeList = nodeList; + this.throwOnRefresh = throwOnRefresh; + } + + void setActive(boolean active) { + this.active.set(active); + } + + void setNodeList(List> nodeList) { + this.nodeList = nodeList; + } + + @Override + public String getMetricId() { + return metricId; + } + + @Override + public DataStoreType getDataStoreType() { + return dataStoreType; + } + + @Override + public Optional>> refresh(TestDeserializer deserializer) { + if (throwOnRefresh) { + throw new RuntimeException("Simulated refresh failure"); + } + return Optional.ofNullable(nodeList); + } + + @Override + public void start() {} + + @Override + public void ensureConnected() {} + + @Override + public void stop() {} + + @Override + public boolean isActive() { + return active.get(); + } + } + + static class TestSignal extends Signal { + protected TestSignal() { + super(() -> null, Collections.emptyList()); + } + + public void fire() { + onSignalReceived(); + } + + @Override + public void start() {} + + @Override + public void stop() {} + } +} diff --git a/ranger-core/src/test/java/io/appform/ranger/core/healthcheck/HealthCheckerMetricsIntegrationTest.java b/ranger-core/src/test/java/io/appform/ranger/core/healthcheck/HealthCheckerMetricsIntegrationTest.java new file mode 100644 index 00000000..f3e0b972 --- /dev/null +++ b/ranger-core/src/test/java/io/appform/ranger/core/healthcheck/HealthCheckerMetricsIntegrationTest.java @@ -0,0 +1,196 @@ +/* + * Copyright 2024 Authors, Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.appform.ranger.core.healthcheck; + +import com.codahale.metrics.MetricRegistry; +import io.appform.ranger.core.util.MetricRecorder; +import lombok.val; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +class HealthCheckerMetricsIntegrationTest { + + private MetricRegistry metricRegistry; + private static final String METRIC_ID = "test-hc-metric"; + + @BeforeEach + void setUp() { + metricRegistry = new MetricRegistry(); + MetricRecorder.initialize(metricRegistry); + } + + @Test + void testHealthyCheck_recordsHealthyMetric() { + val healthChecker = new HealthChecker(METRIC_ID, + List.of(() -> HealthcheckStatus.healthy), 10000); + + val result = healthChecker.get(); + + assertNotNull(result, "First call should always return a result"); + assertEquals(HealthcheckStatus.healthy, result.getStatus()); + + // Verify healthy metric recorded + val healthyMeter = metricRegistry.getMeters().get("io.appform.ranger.healthcheck." + METRIC_ID + ".healthy"); + assertNotNull(healthyMeter, "Healthy meter should exist"); + assertEquals(1, healthyMeter.getCount()); + + // Verify no failure metric + assertNull(metricRegistry.getMeters().get("io.appform.ranger.healthcheck." + METRIC_ID + ".failure")); + // Verify no unhealthy metric + assertNull(metricRegistry.getMeters().get("io.appform.ranger.healthcheck." + METRIC_ID + ".unhealthy")); + } + + @Test + void testUnhealthyCheck_recordsUnhealthyMetric() { + val healthChecker = new HealthChecker(METRIC_ID, + List.of(() -> HealthcheckStatus.unhealthy), 10000); + + val result = healthChecker.get(); + + assertNotNull(result); + assertEquals(HealthcheckStatus.unhealthy, result.getStatus()); + + // Verify unhealthy metric recorded + val unhealthyMeter = metricRegistry.getMeters().get("io.appform.ranger.healthcheck." + METRIC_ID + ".unhealthy"); + assertNotNull(unhealthyMeter, "Unhealthy meter should exist"); + assertEquals(1, unhealthyMeter.getCount()); + + // No healthy metric + assertNull(metricRegistry.getMeters().get("io.appform.ranger.healthcheck." + METRIC_ID + ".healthy")); + } + + @Test + void testExceptionInHealthcheck_recordsFailureAndUnhealthyMetrics() { + val healthChecker = new HealthChecker(METRIC_ID, List.of(() -> { + throw new RuntimeException("Healthcheck error"); + }), 10000); + + val result = healthChecker.get(); + + assertNotNull(result); + assertEquals(HealthcheckStatus.unhealthy, result.getStatus()); + + // Verify failure metric recorded (exception path) + val failureMeter = metricRegistry.getMeters().get("io.appform.ranger.healthcheck." + METRIC_ID + ".failure"); + assertNotNull(failureMeter, "Failure meter should exist on exception"); + assertEquals(1, failureMeter.getCount()); + + // Verify unhealthy status metric also recorded + val unhealthyMeter = metricRegistry.getMeters().get("io.appform.ranger.healthcheck." + METRIC_ID + ".unhealthy"); + assertNotNull(unhealthyMeter, "Unhealthy meter should be recorded after exception"); + assertEquals(1, unhealthyMeter.getCount()); + } + + @Test + void testMultipleHealthchecks_firstUnhealthy_shortCircuits() { + val healthChecker = new HealthChecker(METRIC_ID, List.of( + () -> HealthcheckStatus.unhealthy, + () -> HealthcheckStatus.healthy // Should not be reached + ), 10000); + + val result = healthChecker.get(); + + assertNotNull(result); + assertEquals(HealthcheckStatus.unhealthy, result.getStatus()); + + val unhealthyMeter = metricRegistry.getMeters().get("io.appform.ranger.healthcheck." + METRIC_ID + ".unhealthy"); + assertNotNull(unhealthyMeter); + assertEquals(1, unhealthyMeter.getCount()); + } + + @Test + void testMultipleHealthchecks_allHealthy() { + val healthChecker = new HealthChecker(METRIC_ID, List.of( + () -> HealthcheckStatus.healthy, + () -> HealthcheckStatus.healthy, + () -> HealthcheckStatus.healthy + ), 10000); + + val result = healthChecker.get(); + + assertNotNull(result); + assertEquals(HealthcheckStatus.healthy, result.getStatus()); + + val healthyMeter = metricRegistry.getMeters().get("io.appform.ranger.healthcheck." + METRIC_ID + ".healthy"); + assertNotNull(healthyMeter); + assertEquals(1, healthyMeter.getCount()); + } + + @Test + void testRepeatedCalls_metricsAccumulate() { + val healthChecker = new HealthChecker(METRIC_ID, + List.of(() -> HealthcheckStatus.healthy), 0); // staleUpdateThreshold=0 => always returns result + + healthChecker.get(); + healthChecker.get(); + healthChecker.get(); + + val healthyMeter = metricRegistry.getMeters().get("io.appform.ranger.healthcheck." + METRIC_ID + ".healthy"); + assertNotNull(healthyMeter); + assertEquals(3, healthyMeter.getCount(), "Healthy metric count should accumulate"); + } + + @Test + void testHealthStatusTransition_bothMetricsRecorded() { + // Start healthy, then transition to unhealthy + val statusHolder = new HealthcheckStatus[]{HealthcheckStatus.healthy}; + val healthChecker = new HealthChecker(METRIC_ID, List.of(() -> statusHolder[0]), 0); + + // First call: healthy + val result1 = healthChecker.get(); + assertNotNull(result1); + assertEquals(HealthcheckStatus.healthy, result1.getStatus()); + + // Switch to unhealthy + statusHolder[0] = HealthcheckStatus.unhealthy; + val result2 = healthChecker.get(); + assertNotNull(result2); + assertEquals(HealthcheckStatus.unhealthy, result2.getStatus()); + + val healthyMeter = metricRegistry.getMeters().get("io.appform.ranger.healthcheck." + METRIC_ID + ".healthy"); + assertEquals(1, healthyMeter.getCount()); + + val unhealthyMeter = metricRegistry.getMeters().get("io.appform.ranger.healthcheck." + METRIC_ID + ".unhealthy"); + assertEquals(1, unhealthyMeter.getCount()); + } + + @Test + void testExceptionInSecondHealthcheck_recordsFailure() { + val healthChecker = new HealthChecker(METRIC_ID, List.of( + () -> HealthcheckStatus.healthy, // First passes + () -> { throw new RuntimeException("Second fails"); } // Second throws + ), 10000); + + val result = healthChecker.get(); + + assertNotNull(result); + assertEquals(HealthcheckStatus.unhealthy, result.getStatus()); + + // Failure meter for the exception + val failureMeter = metricRegistry.getMeters().get("io.appform.ranger.healthcheck." + METRIC_ID + ".failure"); + assertNotNull(failureMeter); + assertEquals(1, failureMeter.getCount()); + + // Unhealthy status because the second check failed + val unhealthyMeter = metricRegistry.getMeters().get("io.appform.ranger.healthcheck." + METRIC_ID + ".unhealthy"); + assertNotNull(unhealthyMeter); + assertEquals(1, unhealthyMeter.getCount()); + } +} diff --git a/ranger-core/src/test/java/io/appform/ranger/core/util/FinderUtilsMetricsIntegrationTest.java b/ranger-core/src/test/java/io/appform/ranger/core/util/FinderUtilsMetricsIntegrationTest.java new file mode 100644 index 00000000..438efdaa --- /dev/null +++ b/ranger-core/src/test/java/io/appform/ranger/core/util/FinderUtilsMetricsIntegrationTest.java @@ -0,0 +1,179 @@ +/* + * Copyright 2024 Authors, Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.appform.ranger.core.util; + +import com.codahale.metrics.MetricRegistry; +import io.appform.ranger.core.healthcheck.HealthcheckStatus; +import io.appform.ranger.core.model.Service; +import io.appform.ranger.core.model.ServiceNode; +import lombok.val; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +class FinderUtilsMetricsIntegrationTest { + + private MetricRegistry metricRegistry; + + @BeforeEach + void setUp() { + metricRegistry = new MetricRegistry(); + MetricRecorder.initialize(metricRegistry); + } + + @Test + void testZombieNodeDetection_recordsMetric() { + val service = new Service("test-ns", "test-svc"); + val zombieNode = ServiceNode.builder() + .host("localhost") + .port(8080) + .nodeData(1) + .healthcheckStatus(HealthcheckStatus.healthy) + .lastUpdatedTimeStamp(0L) // Very old timestamp => zombie + .build(); + + val thresholdTime = System.currentTimeMillis() - 60000; // 1 minute ago + + val result = FinderUtils.isValidNode(service, thresholdTime, zombieNode); + + assertFalse(result, "Zombie node should be invalid"); + + // Verify global zombie nodes meter + val globalMeter = metricRegistry.getMeters().get("io.appform.ranger.zombieNodes"); + assertNotNull(globalMeter, "Global zombie nodes meter should exist"); + assertEquals(1, globalMeter.getCount()); + + // Verify per-service zombie nodes meter + val svcMeter = metricRegistry.getMeters().get("io.appform.ranger.zombieNodes.serviceName.test-svc"); + assertNotNull(svcMeter, "Per-service zombie nodes meter should exist"); + assertEquals(1, svcMeter.getCount()); + } + + @Test + void testZombieNodeDetection_multipleZombies() { + val service = new Service("ns", "my-service"); + val thresholdTime = System.currentTimeMillis() - 60000; + + // Create 3 zombie nodes + for (int i = 0; i < 3; i++) { + val zombieNode = ServiceNode.builder() + .host("host-" + i) + .port(8080 + i) + .nodeData(i) + .healthcheckStatus(HealthcheckStatus.healthy) + .lastUpdatedTimeStamp(0L) + .build(); + FinderUtils.isValidNode(service, thresholdTime, zombieNode); + } + + val globalMeter = metricRegistry.getMeters().get("io.appform.ranger.zombieNodes"); + assertEquals(3, globalMeter.getCount()); + + val svcMeter = metricRegistry.getMeters().get("io.appform.ranger.zombieNodes.serviceName.my-service"); + assertEquals(3, svcMeter.getCount()); + } + + @Test + void testHealthyNode_noMetricRecorded() { + val service = new Service("ns", "svc"); + val healthyNode = ServiceNode.builder() + .host("localhost") + .port(8080) + .nodeData(1) + .healthcheckStatus(HealthcheckStatus.healthy) + .lastUpdatedTimeStamp(System.currentTimeMillis()) // Fresh timestamp + .build(); + + val thresholdTime = System.currentTimeMillis() - 60000; + val result = FinderUtils.isValidNode(service, thresholdTime, healthyNode); + + assertTrue(result, "Healthy node should be valid"); + assertTrue(metricRegistry.getMeters().isEmpty(), "No meters should be recorded for healthy nodes"); + } + + @Test + void testUnhealthyNode_noZombieMetricRecorded() { + val service = new Service("ns", "svc"); + val unhealthyNode = ServiceNode.builder() + .host("localhost") + .port(8080) + .nodeData(1) + .healthcheckStatus(HealthcheckStatus.unhealthy) + .lastUpdatedTimeStamp(System.currentTimeMillis()) + .build(); + + val thresholdTime = System.currentTimeMillis() - 60000; + val result = FinderUtils.isValidNode(service, thresholdTime, unhealthyNode); + + assertFalse(result, "Unhealthy node should be invalid"); + // Zombie metric should NOT be recorded for unhealthy nodes (only for stale healthy nodes) + assertTrue(metricRegistry.getMeters().isEmpty(), + "No zombie meters should be recorded for unhealthy nodes"); + } + + @Test + void testFilterValidNodes_zombiesFiltered_metricsRecorded() { + val service = new Service("ns", "filter-svc"); + val thresholdTime = System.currentTimeMillis() - 60000; + + val nodes = List.of( + ServiceNode.builder() + .host("good-host") + .port(8080) + .nodeData(1) + .healthcheckStatus(HealthcheckStatus.healthy) + .lastUpdatedTimeStamp(System.currentTimeMillis()) + .build(), + ServiceNode.builder() + .host("zombie-host") + .port(8081) + .nodeData(2) + .healthcheckStatus(HealthcheckStatus.healthy) + .lastUpdatedTimeStamp(0L) // zombie + .build(), + ServiceNode.builder() + .host("zombie-host-2") + .port(8082) + .nodeData(3) + .healthcheckStatus(HealthcheckStatus.healthy) + .lastUpdatedTimeStamp(0L) // zombie + .build() + ); + + val filtered = FinderUtils.filterValidNodes(service, nodes, thresholdTime); + + assertEquals(1, filtered.size(), "Only 1 valid node should remain"); + assertEquals("good-host", filtered.get(0).getHost()); + + val globalMeter = metricRegistry.getMeters().get("io.appform.ranger.zombieNodes"); + assertEquals(2, globalMeter.getCount(), "2 zombie detections should be recorded"); + + val svcMeter = metricRegistry.getMeters().get("io.appform.ranger.zombieNodes.serviceName.filter-svc"); + assertEquals(2, svcMeter.getCount()); + } + + @Test + void testNullNode_noMetricRecorded() { + val service = new Service("ns", "svc"); + val result = FinderUtils.isValidNode(service, System.currentTimeMillis() - 60000, null); + + assertFalse(result, "Null node should be invalid"); + assertTrue(metricRegistry.getMeters().isEmpty(), "No meters should be recorded for null nodes"); + } +} From 59dfd9cc6aa43d8bb2ae874532751598983262c7 Mon Sep 17 00:00:00 2001 From: Jitendra Dhawan Date: Sun, 31 May 2026 19:23:49 +0530 Subject: [PATCH 13/24] Update performance test mean operations for IdGenerator --- .../io.appform.ranger.id.IdGeneratorPerfTest.testGenerate.json | 2 +- ...ppform.ranger.id.IdGeneratorPerfTest.testGenerateBase36.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ranger-id/perf/results/io.appform.ranger.id.IdGeneratorPerfTest.testGenerate.json b/ranger-id/perf/results/io.appform.ranger.id.IdGeneratorPerfTest.testGenerate.json index 709b8f13..1aa29ce0 100644 --- a/ranger-id/perf/results/io.appform.ranger.id.IdGeneratorPerfTest.testGenerate.json +++ b/ranger-id/perf/results/io.appform.ranger.id.IdGeneratorPerfTest.testGenerate.json @@ -4,5 +4,5 @@ "iterations" : 4, "threads" : 1, "forks" : 3, - "mean_ops" : 665263.0845729186 + "mean_ops" : 658557.7599494386 } \ No newline at end of file diff --git a/ranger-id/perf/results/io.appform.ranger.id.IdGeneratorPerfTest.testGenerateBase36.json b/ranger-id/perf/results/io.appform.ranger.id.IdGeneratorPerfTest.testGenerateBase36.json index efbfb290..fdbe58d2 100644 --- a/ranger-id/perf/results/io.appform.ranger.id.IdGeneratorPerfTest.testGenerateBase36.json +++ b/ranger-id/perf/results/io.appform.ranger.id.IdGeneratorPerfTest.testGenerateBase36.json @@ -4,5 +4,5 @@ "iterations" : 4, "threads" : 1, "forks" : 3, - "mean_ops" : 506179.5641038627 + "mean_ops" : 509709.58246890694 } \ No newline at end of file From bf89c3287d305c0d5129132950570354d6c3b249 Mon Sep 17 00:00:00 2001 From: Jitendra Dhawan Date: Mon, 1 Jun 2026 09:54:32 +0530 Subject: [PATCH 14/24] remove redundant MonitoredFunction annotations and function metrics dependency --- ranger-core/pom.xml | 4 ---- .../ranger/drove/servicefinderhub/DroveServiceDataSource.java | 3 +-- .../appform/ranger/http/servicefinder/HttpNodeDataSource.java | 3 +-- .../ranger/http/servicefinderhub/HttpServiceDataSource.java | 3 +-- .../appform/ranger/http/serviceprovider/HttpNodeDataSink.java | 3 +-- .../ranger/zookeeper/servicefinder/ZkNodeDataSource.java | 3 --- .../zookeeper/servicefinderhub/ZkServiceDataSource.java | 3 +-- .../ranger/zookeeper/serviceprovider/ZkNodeDataSink.java | 3 +-- 8 files changed, 6 insertions(+), 19 deletions(-) diff --git a/ranger-core/pom.xml b/ranger-core/pom.xml index cafd21fa..ec9f0b72 100644 --- a/ranger-core/pom.xml +++ b/ranger-core/pom.xml @@ -53,10 +53,6 @@ dropwizard-metrics ${dropwizard.version} - - io.appform.functionmetrics - function-metrics - org.mockito mockito-core diff --git a/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinderhub/DroveServiceDataSource.java b/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinderhub/DroveServiceDataSource.java index 2c0bf0dd..3b6a3d52 100644 --- a/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinderhub/DroveServiceDataSource.java +++ b/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinderhub/DroveServiceDataSource.java @@ -16,7 +16,7 @@ package io.appform.ranger.drove.servicefinderhub; import com.fasterxml.jackson.databind.ObjectMapper; -import io.appform.functionmetrics.MonitoredFunction; + import io.appform.ranger.core.finderhub.ServiceDataSource; import io.appform.ranger.core.model.Service; import io.appform.ranger.core.model.DataStoreType; @@ -49,7 +49,6 @@ public DroveServiceDataSource( } @Override - @MonitoredFunction public Collection services() { requireNonNull(config, "client config has not been set for node data"); requireNonNull(mapper, "mapper has not been set for node data"); diff --git a/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpNodeDataSource.java b/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpNodeDataSource.java index 900fa63f..7b171c54 100644 --- a/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpNodeDataSource.java +++ b/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpNodeDataSource.java @@ -15,7 +15,7 @@ */ package io.appform.ranger.http.servicefinder; -import io.appform.functionmetrics.MonitoredFunction; + import io.appform.ranger.core.model.DataStoreType; import io.appform.ranger.core.model.NodeDataSource; import io.appform.ranger.core.model.Service; @@ -68,7 +68,6 @@ public DataStoreType getDataStoreType() { } @Override - @MonitoredFunction public Optional>> refresh(D deserializer) { return Optional.of(httpCommunicator.listNodes(service, deserializer)); } diff --git a/ranger-http/src/main/java/io/appform/ranger/http/servicefinderhub/HttpServiceDataSource.java b/ranger-http/src/main/java/io/appform/ranger/http/servicefinderhub/HttpServiceDataSource.java index 4107556f..3181694f 100644 --- a/ranger-http/src/main/java/io/appform/ranger/http/servicefinderhub/HttpServiceDataSource.java +++ b/ranger-http/src/main/java/io/appform/ranger/http/servicefinderhub/HttpServiceDataSource.java @@ -15,7 +15,7 @@ */ package io.appform.ranger.http.servicefinderhub; -import io.appform.functionmetrics.MonitoredFunction; + import io.appform.ranger.core.finderhub.ServiceDataSource; import io.appform.ranger.core.model.Service; import io.appform.ranger.core.model.DataStoreType; @@ -43,7 +43,6 @@ public HttpServiceDataSource(String metricId, HttpClientConfig config, HttpCommu } @Override - @MonitoredFunction public Collection services() { Objects.requireNonNull(config, "client config has not been set for node data"); try { diff --git a/ranger-http/src/main/java/io/appform/ranger/http/serviceprovider/HttpNodeDataSink.java b/ranger-http/src/main/java/io/appform/ranger/http/serviceprovider/HttpNodeDataSink.java index d295efcb..0d16b1d6 100644 --- a/ranger-http/src/main/java/io/appform/ranger/http/serviceprovider/HttpNodeDataSink.java +++ b/ranger-http/src/main/java/io/appform/ranger/http/serviceprovider/HttpNodeDataSink.java @@ -17,7 +17,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import io.appform.functionmetrics.MonitoredFunction; + import io.appform.ranger.core.model.DataStoreType; import io.appform.ranger.core.model.NodeDataSink; import io.appform.ranger.core.model.Service; @@ -57,7 +57,6 @@ public HttpNodeDataSink(String metricId, Service service, HttpClientConfig confi } @Override - @MonitoredFunction public void updateState(S serializer, ServiceNode serviceNode) { requireNonNull(config, "client config has not been set for node data"); requireNonNull(mapper, "mapper has not been set for node data"); diff --git a/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinder/ZkNodeDataSource.java b/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinder/ZkNodeDataSource.java index b50b590f..339219eb 100644 --- a/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinder/ZkNodeDataSource.java +++ b/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinder/ZkNodeDataSource.java @@ -15,8 +15,6 @@ */ package io.appform.ranger.zookeeper.servicefinder; -import com.fasterxml.jackson.core.JsonProcessingException; -import io.appform.functionmetrics.MonitoredFunction; import io.appform.ranger.core.model.DataStoreType; import io.appform.ranger.core.model.NodeDataSource; import io.appform.ranger.core.model.Service; @@ -63,7 +61,6 @@ public DataStoreType getDataStoreType() { } @Override - @MonitoredFunction public Optional>> refresh(D deserializer) { return checkForUpdateOnZookeeper(deserializer); } diff --git a/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinderhub/ZkServiceDataSource.java b/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinderhub/ZkServiceDataSource.java index f35f4e15..4894ee10 100644 --- a/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinderhub/ZkServiceDataSource.java +++ b/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinderhub/ZkServiceDataSource.java @@ -15,7 +15,7 @@ */ package io.appform.ranger.zookeeper.servicefinderhub; -import io.appform.functionmetrics.MonitoredFunction; + import io.appform.ranger.core.finderhub.ServiceDataSource; import io.appform.ranger.core.model.Service; import io.appform.ranger.core.model.DataStoreType; @@ -59,7 +59,6 @@ public ZkServiceDataSource(String metricId, @Override @SneakyThrows - @MonitoredFunction public Collection services() { try { val children = curatorFramework.getChildren() diff --git a/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/serviceprovider/ZkNodeDataSink.java b/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/serviceprovider/ZkNodeDataSink.java index 39abf3c3..34e94265 100644 --- a/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/serviceprovider/ZkNodeDataSink.java +++ b/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/serviceprovider/ZkNodeDataSink.java @@ -15,7 +15,7 @@ */ package io.appform.ranger.zookeeper.serviceprovider; -import io.appform.functionmetrics.MonitoredFunction; + import io.appform.ranger.core.model.DataStoreType; import io.appform.ranger.core.model.NodeDataSink; import io.appform.ranger.core.model.Service; @@ -48,7 +48,6 @@ public ZkNodeDataSink( } @Override - @MonitoredFunction public void updateState(S serializer, ServiceNode serviceNode) { if (isStopped()) { log.warn("Node has been stopped already for service: {}. No update will be possible.", From 3e2df50f88ae35ddffaef2e23e962dae48c89f31 Mon Sep 17 00:00:00 2001 From: Jitendra Dhawan Date: Mon, 1 Jun 2026 18:13:46 +0530 Subject: [PATCH 15/24] Refactor HealthChecker and MetricRecorder to include DataStoreType in metrics recording --- .../ServiceRegistryUpdater.java | 6 +- .../core/healthcheck/HealthChecker.java | 10 +- .../ranger/core/model/NodeDataSink.java | 5 + .../BaseServiceProviderBuilder.java | 3 +- .../ranger/core/util/MetricRecorder.java | 190 ++++++++++++------ .../HealthCheckerMetricsIntegrationTest.java | 17 +- .../serviceprovider/ServiceProviderTest.java | 15 +- .../serviceprovider/HttpNodeDataSink.java | 12 +- ...r.id.IdGeneratorPerfTest.testGenerate.json | 2 +- ...dGeneratorPerfTest.testGenerateBase36.json | 2 +- .../serviceprovider/ZkNodeDataSink.java | 12 +- 11 files changed, 186 insertions(+), 88 deletions(-) diff --git a/ranger-core/src/main/java/io/appform/ranger/core/finder/serviceregistry/ServiceRegistryUpdater.java b/ranger-core/src/main/java/io/appform/ranger/core/finder/serviceregistry/ServiceRegistryUpdater.java index b2f25ebc..8bb538c6 100644 --- a/ranger-core/src/main/java/io/appform/ranger/core/finder/serviceregistry/ServiceRegistryUpdater.java +++ b/ranger-core/src/main/java/io/appform/ranger/core/finder/serviceregistry/ServiceRegistryUpdater.java @@ -164,7 +164,8 @@ private void updateRegistry() throws InterruptedException { e.getClass().getSimpleName(), e.getMessage()); callFailed = true; - MetricRecorder.recordNodeDataRefreshFailure(nodeDataSource.getDataStoreType(), nodeDataSource.getMetricId(), + MetricRecorder.recordNodeDataRefreshFailure(serviceRegistry.getService().getServiceName(), + nodeDataSource.getDataStoreType(), nodeDataSource.getMetricId(), stopwatch.elapsed(TimeUnit.MILLISECONDS)); } finally { stopwatch.stop(); @@ -175,7 +176,8 @@ private void updateRegistry() throws InterruptedException { log.warn("Node data source seems to be down. Keeping old list for {}." + " Will update timestamp to keep stale date relevant.", serviceRegistry.getService().getServiceName()); - MetricRecorder.recordStaleDataRetained(nodeDataSource.getDataStoreType(), nodeDataSource.getMetricId()); + MetricRecorder.recordStaleDataRetained(serviceRegistry.getService().getServiceName(), + nodeDataSource.getDataStoreType(), nodeDataSource.getMetricId()); serviceRegistry.updateNodes(serviceRegistry.nodeList() .stream() .filter(node -> HealthcheckStatus.healthy == node.getHealthcheckStatus()) diff --git a/ranger-core/src/main/java/io/appform/ranger/core/healthcheck/HealthChecker.java b/ranger-core/src/main/java/io/appform/ranger/core/healthcheck/HealthChecker.java index 9f1c37f8..891ee53c 100644 --- a/ranger-core/src/main/java/io/appform/ranger/core/healthcheck/HealthChecker.java +++ b/ranger-core/src/main/java/io/appform/ranger/core/healthcheck/HealthChecker.java @@ -15,6 +15,7 @@ */ package io.appform.ranger.core.healthcheck; +import io.appform.ranger.core.model.DataStoreType; import io.appform.ranger.core.util.MetricRecorder; import lombok.extern.slf4j.Slf4j; import lombok.val; @@ -28,13 +29,16 @@ @Slf4j public class HealthChecker implements Supplier { + private final DataStoreType dataStoreType; private final String metricId; private final List healthChecks; private final int staleUpdateThreshold; private HealthcheckStatus lastHealthcheckStatus; private long lastUpdatedTime; - public HealthChecker(String metricId, List healthChecks, int staleUpdateThreshold) { + public HealthChecker(DataStoreType dataStoreType, String metricId, + List healthChecks, int staleUpdateThreshold) { + this.dataStoreType = dataStoreType; this.metricId = metricId; this.healthChecks = healthChecks; this.staleUpdateThreshold = staleUpdateThreshold; @@ -60,13 +64,13 @@ private boolean refreshHealth() { catch (Exception e) { log.error("Error running healthcheck. Setting node to unhealthy", e); healthcheckStatus = HealthcheckStatus.unhealthy; - MetricRecorder.recordHealthcheckFailure(metricId); + MetricRecorder.recordHealthcheckFailure(dataStoreType, metricId); } if (HealthcheckStatus.unhealthy == healthcheckStatus) { break; } } - MetricRecorder.recordHealthcheckStatus(metricId, HealthcheckStatus.healthy == healthcheckStatus); + MetricRecorder.recordHealthcheckStatus(dataStoreType, metricId, HealthcheckStatus.healthy == healthcheckStatus); //Trigger update only if state change has happened //Conditions on which update will be triggered //1. First time diff --git a/ranger-core/src/main/java/io/appform/ranger/core/model/NodeDataSink.java b/ranger-core/src/main/java/io/appform/ranger/core/model/NodeDataSink.java index faa7c4ad..1a9cde8e 100644 --- a/ranger-core/src/main/java/io/appform/ranger/core/model/NodeDataSink.java +++ b/ranger-core/src/main/java/io/appform/ranger/core/model/NodeDataSink.java @@ -19,5 +19,10 @@ * */ public interface NodeDataSink> extends NodeDataStoreConnector { + + DataStoreType getDataStoreType(); + + String getMetricId(); + void updateState(S serializer, ServiceNode serviceNode); } diff --git a/ranger-core/src/main/java/io/appform/ranger/core/serviceprovider/BaseServiceProviderBuilder.java b/ranger-core/src/main/java/io/appform/ranger/core/serviceprovider/BaseServiceProviderBuilder.java index 398c40df..e869fe84 100644 --- a/ranger-core/src/main/java/io/appform/ranger/core/serviceprovider/BaseServiceProviderBuilder.java +++ b/ranger-core/src/main/java/io/appform/ranger/core/serviceprovider/BaseServiceProviderBuilder.java @@ -211,7 +211,8 @@ protected final ServiceProvider buildProvider() { val healthcheckUpdateSignalGenerator = new ScheduledSignal<>( service, - new HealthChecker(metricId, healthchecks, staleUpdateThresholdMs), + new HealthChecker(usableNodeDataSource.getDataStoreType(), metricId, healthchecks, + staleUpdateThresholdMs), Collections.emptyList(), healthUpdateIntervalMs ); diff --git a/ranger-core/src/main/java/io/appform/ranger/core/util/MetricRecorder.java b/ranger-core/src/main/java/io/appform/ranger/core/util/MetricRecorder.java index a0e61a14..5e3903d2 100644 --- a/ranger-core/src/main/java/io/appform/ranger/core/util/MetricRecorder.java +++ b/ranger-core/src/main/java/io/appform/ranger/core/util/MetricRecorder.java @@ -32,8 +32,10 @@ public class MetricRecorder { public static final String HEALTHCHECK = "healthcheck"; public static final String NODE_DATA_SINK = "nodeDataSink"; public static final String REGISTER_SERVICE = "registerService"; - public static final String SERIALIZAION = "serialization"; + public static final String SERIALIZATION = "serialization"; public static final String DESERIALIZATION = "deserialization"; + public static final String STALE_DATA_RETAINED = "staleDataRetained"; + public static final String ZK_READ = "zkRead"; private static MetricRegistry metricRegistry = new MetricRegistry(); @@ -42,125 +44,181 @@ public static void initialize(MetricRegistry registry) { } public static void recordZombieNodeFound(String serviceName) { - metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, ZOMBIE_NODES)).mark(); - metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, ZOMBIE_NODES, SERVICE_NAME, serviceName)).mark(); + if (metricRegistry != null) { + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, ZOMBIE_NODES)).mark(); + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, ZOMBIE_NODES, SERVICE_NAME, serviceName)).mark(); + } } public static void recordNoteDataSourceStatus(DataStoreType dataStoreType, String metricId, boolean active) { - metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_SOURCE, dataStoreType.name(), metricId, - active ? ACTIVE : INACTIVE)).mark(); + if (metricRegistry != null) { + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_SOURCE, dataStoreType.name(), + metricId, active ? ACTIVE : INACTIVE)).mark(); + } } public static void recordNodeDataRefreshSuccess(DataStoreType dataStoreType, String metricId, long elapsed) { - metricRegistry.timer(MetricRegistry.name(PACKAGE_PREFIX, DATA_SOURCE, dataStoreType.name(), metricId, NODE_DATA_REFRESH, SUCCESS)) - .update(elapsed, MILLISECONDS); + if (metricRegistry != null) { + metricRegistry.timer(MetricRegistry.name(PACKAGE_PREFIX, DATA_SOURCE, dataStoreType.name(), + metricId, NODE_DATA_REFRESH, SUCCESS)).update(elapsed, MILLISECONDS); + } } - public static void recordNodeDataRefreshFailure(DataStoreType dataStoreType, String metricId, long elapsed) { - metricRegistry.timer(MetricRegistry.name(PACKAGE_PREFIX, DATA_SOURCE, dataStoreType.name(), metricId, NODE_DATA_REFRESH, FAILURE)) - .update(elapsed, MILLISECONDS); + public static void recordNodeDataRefreshFailure(String serviceName, DataStoreType dataStoreType, + String metricId, long elapsed) { + if (metricRegistry != null) { + metricRegistry.timer(MetricRegistry.name(PACKAGE_PREFIX, DATA_SOURCE, dataStoreType.name(), + metricId, NODE_DATA_REFRESH, FAILURE)).update(elapsed, MILLISECONDS); + metricRegistry.timer(MetricRegistry.name(PACKAGE_PREFIX, DATA_SOURCE, dataStoreType.name(), + metricId, SERVICE_NAME, serviceName, NODE_DATA_REFRESH, FAILURE)) + .update(elapsed, MILLISECONDS); + } } - public static void recordStaleDataRetained(DataStoreType dataStoreType, String metricId) { - metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_SOURCE, dataStoreType.name(), metricId, "staleDataRetained")).mark(); + public static void recordStaleDataRetained(String serviceName, DataStoreType dataStoreType, String metricId) { + if (metricRegistry != null) { + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_SOURCE, dataStoreType.name(), + metricId, STALE_DATA_RETAINED)).mark(); + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_SOURCE, dataStoreType.name(), + metricId, SERVICE_NAME, serviceName, STALE_DATA_RETAINED)).mark(); + } } public static void recordNodeDataSinkUpdateStatus(DataStoreType dataStoreType, String metricId, String status) { - metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_SOURCE, dataStoreType.name(), DATA_SOURCE, metricId, NODE_DATA_SINK, UPDATE, status)).mark(); + if (metricRegistry != null) { + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_SOURCE, dataStoreType.name(), + DATA_SOURCE, metricId, NODE_DATA_SINK, UPDATE, status)).mark(); + } } - public static void recordHealthcheckFailure(String metricId) { - metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, HEALTHCHECK, metricId, FAILURE)).mark(); + public static void recordHealthcheckFailure(DataStoreType dataStoreType, String metricId) { + if (metricRegistry != null) { + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, HEALTHCHECK, metricId, FAILURE)).mark(); + } } - public static void recordHealthcheckStatus(String metricId, boolean healthy) { - metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, HEALTHCHECK, metricId, healthy ? HEALTHY : UNHEALTHY)).mark(); + public static void recordHealthcheckStatus(DataStoreType dataStoreType, String metricId, boolean healthy) { + if (metricRegistry != null) { + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, HEALTHCHECK, metricId, + healthy ? HEALTHY : UNHEALTHY)).mark(); + } } public static void recordServicesFetchStatus(DataStoreType dataStoreType, String metricId, String success) { - metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), - DATA_SOURCE, metricId, SERVICES_LIST, "fetch", success)).mark(); + if (metricRegistry != null) { + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), + DATA_SOURCE, metricId, SERVICES_LIST, "fetch", success)).mark(); + } } - public static void recordRemoteCallStatusCode(DataStoreType dataStoreType, String metricId, String remoteCall, int statusCode) { - metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), - DATA_SOURCE, metricId, HTTP_CALL, remoteCall, "responseStatus", Integer.toString(statusCode))).mark(); + public static void recordRemoteCallStatusCode(DataStoreType dataStoreType, String metricId, + String remoteCall, int statusCode) { + if (metricRegistry != null) { + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), + DATA_SOURCE, metricId, HTTP_CALL, remoteCall, "responseStatus", Integer.toString(statusCode))).mark(); + } } - public static void recordCacheUpdateOnDroveEvent(DataStoreType dataStoreType, String metricId, String eventName, String serviceName) { - metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), - DATA_SOURCE, metricId, "cacheUpdateOnDroveEvent", eventName, SERVICE_NAME, serviceName)).mark(); + public static void recordCacheUpdateOnDroveEvent(DataStoreType dataStoreType, String metricId, + String eventName, String serviceName) { + if (metricRegistry != null) { + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), + DATA_SOURCE, metricId, "cacheUpdateOnDroveEvent", eventName, SERVICE_NAME, serviceName)).mark(); + } } public static void recordServicesParseFailure(DataStoreType dataStoreType, String metricId) { - metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), - DATA_SOURCE, metricId, HTTP_CALL, SERVICES_LIST, RESPONSE_PARSE_FAILURE)).mark(); + if (metricRegistry != null) { + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), + DATA_SOURCE, metricId, HTTP_CALL, SERVICES_LIST, RESPONSE_PARSE_FAILURE)).mark(); + } } public static void recordListNodesParseFailure(DataStoreType dataStoreType, String metricId, String serviceName) { - metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), - DATA_SOURCE, metricId, HTTP_CALL, LIST_NODES, SERVICE_NAME, serviceName, RESPONSE_PARSE_FAILURE)).mark(); + if (metricRegistry != null) { + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), + DATA_SOURCE, metricId, HTTP_CALL, LIST_NODES, SERVICE_NAME, serviceName, RESPONSE_PARSE_FAILURE)).mark(); + } } public static void recordZookeeperReadUnknownFailure(DataStoreType dataStoreType, String metricId, String operation, String exceptionName) { - metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), - DATA_SOURCE, metricId, "zkRead", operation, UNKNOWN_FAILURE)).mark(); - metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), - DATA_SOURCE, metricId, "zkRead", operation, UNKNOWN_FAILURE, exceptionName)).mark(); + if (metricRegistry != null) { + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), + DATA_SOURCE, metricId, ZK_READ, operation, UNKNOWN_FAILURE)).mark(); + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), + DATA_SOURCE, metricId, ZK_READ, operation, UNKNOWN_FAILURE, exceptionName)).mark(); + } } public static void recordRemoteCallUnknownFailure(DataStoreType dataStoreType, String metricId, String httpMethodName, String exceptionName) { - metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), - DATA_SOURCE, metricId, HTTP_CALL, httpMethodName, UNKNOWN_FAILURE)).mark(); - metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), - DATA_SOURCE, metricId, HTTP_CALL, httpMethodName, UNKNOWN_FAILURE, exceptionName)).mark(); - + if (metricRegistry != null) { + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), + DATA_SOURCE, metricId, HTTP_CALL, httpMethodName, UNKNOWN_FAILURE)).mark(); + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), + DATA_SOURCE, metricId, HTTP_CALL, httpMethodName, UNKNOWN_FAILURE, exceptionName)).mark(); + } } public static void recordRemoteCallUnknownFailure(DataStoreType dataStoreType, String metricId, String httpMethodName, String exceptionName, String serviceName) { recordRemoteCallUnknownFailure(dataStoreType, metricId, httpMethodName, exceptionName); - metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), - DATA_SOURCE, metricId, HTTP_CALL, httpMethodName, SERVICE_NAME, serviceName, - UNKNOWN_FAILURE)).mark(); - metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), - DATA_SOURCE, metricId, HTTP_CALL, httpMethodName, SERVICE_NAME, serviceName, - UNKNOWN_FAILURE, exceptionName)).mark(); - + if (metricRegistry != null) { + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), + DATA_SOURCE, metricId, HTTP_CALL, httpMethodName, SERVICE_NAME, serviceName, + UNKNOWN_FAILURE)).mark(); + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), + DATA_SOURCE, metricId, HTTP_CALL, httpMethodName, SERVICE_NAME, serviceName, + UNKNOWN_FAILURE, exceptionName)).mark(); + } } public static void recordNullOrEmptyServicesListResponse(DataStoreType dataStoreType, String metricId) { - metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), - DATA_SOURCE, metricId, HTTP_CALL, SERVICES_LIST, NULL_OR_EMPTY_RESPONSE)).mark(); + if (metricRegistry != null) { + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), + DATA_SOURCE, metricId, HTTP_CALL, SERVICES_LIST, NULL_OR_EMPTY_RESPONSE)).mark(); + } } - public static void recordNullOrEmptyListNodeResponse(DataStoreType dataStoreType, String metricId, String serviceName) { - metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), - DATA_SOURCE, metricId, HTTP_CALL, LIST_NODES, SERVICE_NAME, serviceName, NULL_OR_EMPTY_RESPONSE)).mark(); + public static void recordNullOrEmptyListNodeResponse(DataStoreType dataStoreType, String metricId, + String serviceName) { + if (metricRegistry != null) { + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), + DATA_SOURCE, metricId, HTTP_CALL, LIST_NODES, SERVICE_NAME, serviceName, NULL_OR_EMPTY_RESPONSE)).mark(); + } } - public static void recordNodeDataSinkUnknownFailure(DataStoreType dataStoreType, String metricId, String serviceName, String exceptionName) { - metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), - DATA_SOURCE, metricId, NODE_DATA_SINK, SERVICE_NAME, serviceName, UNKNOWN_FAILURE)).mark(); - metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), - DATA_SOURCE, metricId, NODE_DATA_SINK, SERVICE_NAME, serviceName, UNKNOWN_FAILURE, exceptionName)).mark(); + public static void recordNodeDataSinkUnknownFailure(DataStoreType dataStoreType, String metricId, + String serviceName, String exceptionName) { + if (metricRegistry != null) { + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), + DATA_SOURCE, metricId, NODE_DATA_SINK, SERVICE_NAME, serviceName, UNKNOWN_FAILURE)).mark(); + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), + DATA_SOURCE, metricId, NODE_DATA_SINK, SERVICE_NAME, serviceName, UNKNOWN_FAILURE, exceptionName)).mark(); + } } - public static void recordNodeDataSinkSerDeFailure(DataStoreType dataStoreType, String metricId, String serDe, String serviceName, String exceptionName) { - metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), DATA_SOURCE, metricId, NODE_DATA_SINK, serDe, FAILURE)).mark(); - metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), - DATA_SOURCE, metricId, NODE_DATA_SINK, serDe,SERVICE_NAME, serviceName, FAILURE)).mark(); - metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), - DATA_SOURCE, metricId, NODE_DATA_SINK, serDe,SERVICE_NAME, serviceName, FAILURE, exceptionName)).mark(); + public static void recordNodeDataSinkSerDeFailure(DataStoreType dataStoreType, String metricId, + String serDe, String serviceName, String exceptionName) { + if (metricRegistry != null) { + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), + DATA_SOURCE, metricId, NODE_DATA_SINK, serDe, FAILURE)).mark(); + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), + DATA_SOURCE, metricId, NODE_DATA_SINK, serDe, SERVICE_NAME, serviceName, FAILURE)).mark(); + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), + DATA_SOURCE, metricId, NODE_DATA_SINK, serDe, SERVICE_NAME, serviceName, FAILURE, exceptionName)).mark(); + } } - public static void recordNullOrEmptyRegisterServiceResponse(DataStoreType dataStoreType, String metricId, String serviceName) { - metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), - DATA_SOURCE, metricId, HTTP_CALL, REGISTER_SERVICE, NULL_OR_EMPTY_RESPONSE)).mark(); - metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), - DATA_SOURCE, metricId, HTTP_CALL, REGISTER_SERVICE, SERVICE_NAME, serviceName, NULL_OR_EMPTY_RESPONSE)).mark(); + public static void recordNullOrEmptyRegisterServiceResponse(DataStoreType dataStoreType, String metricId, + String serviceName) { + if (metricRegistry != null) { + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), + DATA_SOURCE, metricId, HTTP_CALL, REGISTER_SERVICE, NULL_OR_EMPTY_RESPONSE)).mark(); + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), + DATA_SOURCE, metricId, HTTP_CALL, REGISTER_SERVICE, SERVICE_NAME, serviceName, NULL_OR_EMPTY_RESPONSE)).mark(); } + } } diff --git a/ranger-core/src/test/java/io/appform/ranger/core/healthcheck/HealthCheckerMetricsIntegrationTest.java b/ranger-core/src/test/java/io/appform/ranger/core/healthcheck/HealthCheckerMetricsIntegrationTest.java index f3e0b972..7f3d862f 100644 --- a/ranger-core/src/test/java/io/appform/ranger/core/healthcheck/HealthCheckerMetricsIntegrationTest.java +++ b/ranger-core/src/test/java/io/appform/ranger/core/healthcheck/HealthCheckerMetricsIntegrationTest.java @@ -16,6 +16,7 @@ package io.appform.ranger.core.healthcheck; import com.codahale.metrics.MetricRegistry; +import io.appform.ranger.core.model.DataStoreType; import io.appform.ranger.core.util.MetricRecorder; import lombok.val; import org.junit.jupiter.api.BeforeEach; @@ -38,7 +39,7 @@ void setUp() { @Test void testHealthyCheck_recordsHealthyMetric() { - val healthChecker = new HealthChecker(METRIC_ID, + val healthChecker = new HealthChecker(DataStoreType.ZK, METRIC_ID, List.of(() -> HealthcheckStatus.healthy), 10000); val result = healthChecker.get(); @@ -59,7 +60,7 @@ void testHealthyCheck_recordsHealthyMetric() { @Test void testUnhealthyCheck_recordsUnhealthyMetric() { - val healthChecker = new HealthChecker(METRIC_ID, + val healthChecker = new HealthChecker(DataStoreType.ZK, METRIC_ID, List.of(() -> HealthcheckStatus.unhealthy), 10000); val result = healthChecker.get(); @@ -78,7 +79,7 @@ void testUnhealthyCheck_recordsUnhealthyMetric() { @Test void testExceptionInHealthcheck_recordsFailureAndUnhealthyMetrics() { - val healthChecker = new HealthChecker(METRIC_ID, List.of(() -> { + val healthChecker = new HealthChecker(DataStoreType.ZK, METRIC_ID, List.of(() -> { throw new RuntimeException("Healthcheck error"); }), 10000); @@ -100,7 +101,7 @@ void testExceptionInHealthcheck_recordsFailureAndUnhealthyMetrics() { @Test void testMultipleHealthchecks_firstUnhealthy_shortCircuits() { - val healthChecker = new HealthChecker(METRIC_ID, List.of( + val healthChecker = new HealthChecker(DataStoreType.ZK, METRIC_ID, List.of( () -> HealthcheckStatus.unhealthy, () -> HealthcheckStatus.healthy // Should not be reached ), 10000); @@ -117,7 +118,7 @@ void testMultipleHealthchecks_firstUnhealthy_shortCircuits() { @Test void testMultipleHealthchecks_allHealthy() { - val healthChecker = new HealthChecker(METRIC_ID, List.of( + val healthChecker = new HealthChecker(DataStoreType.ZK, METRIC_ID, List.of( () -> HealthcheckStatus.healthy, () -> HealthcheckStatus.healthy, () -> HealthcheckStatus.healthy @@ -135,7 +136,7 @@ void testMultipleHealthchecks_allHealthy() { @Test void testRepeatedCalls_metricsAccumulate() { - val healthChecker = new HealthChecker(METRIC_ID, + val healthChecker = new HealthChecker(DataStoreType.ZK, METRIC_ID, List.of(() -> HealthcheckStatus.healthy), 0); // staleUpdateThreshold=0 => always returns result healthChecker.get(); @@ -151,7 +152,7 @@ void testRepeatedCalls_metricsAccumulate() { void testHealthStatusTransition_bothMetricsRecorded() { // Start healthy, then transition to unhealthy val statusHolder = new HealthcheckStatus[]{HealthcheckStatus.healthy}; - val healthChecker = new HealthChecker(METRIC_ID, List.of(() -> statusHolder[0]), 0); + val healthChecker = new HealthChecker(DataStoreType.ZK, METRIC_ID, List.of(() -> statusHolder[0]), 0); // First call: healthy val result1 = healthChecker.get(); @@ -173,7 +174,7 @@ void testHealthStatusTransition_bothMetricsRecorded() { @Test void testExceptionInSecondHealthcheck_recordsFailure() { - val healthChecker = new HealthChecker(METRIC_ID, List.of( + val healthChecker = new HealthChecker(DataStoreType.ZK, METRIC_ID, List.of( () -> HealthcheckStatus.healthy, // First passes () -> { throw new RuntimeException("Second fails"); } // Second throws ), 10000); diff --git a/ranger-core/src/test/java/io/appform/ranger/core/serviceprovider/ServiceProviderTest.java b/ranger-core/src/test/java/io/appform/ranger/core/serviceprovider/ServiceProviderTest.java index b0c1cae6..9d9d4387 100644 --- a/ranger-core/src/test/java/io/appform/ranger/core/serviceprovider/ServiceProviderTest.java +++ b/ranger-core/src/test/java/io/appform/ranger/core/serviceprovider/ServiceProviderTest.java @@ -21,10 +21,7 @@ import io.appform.ranger.core.healthcheck.updater.HealthStatusHandler; import io.appform.ranger.core.healthcheck.updater.HealthUpdateHandler; import io.appform.ranger.core.healthcheck.updater.LastUpdatedHandler; -import io.appform.ranger.core.model.NodeDataSink; -import io.appform.ranger.core.model.Serializer; -import io.appform.ranger.core.model.Service; -import io.appform.ranger.core.model.ServiceNode; +import io.appform.ranger.core.model.*; import io.appform.ranger.core.units.TestNodeData; import lombok.val; import org.junit.jupiter.api.Assertions; @@ -61,6 +58,16 @@ static class TestNodeDataSink serviceNode) { testNodeData = serviceNode.getNodeData(); diff --git a/ranger-http/src/main/java/io/appform/ranger/http/serviceprovider/HttpNodeDataSink.java b/ranger-http/src/main/java/io/appform/ranger/http/serviceprovider/HttpNodeDataSink.java index 0d16b1d6..19f93194 100644 --- a/ranger-http/src/main/java/io/appform/ranger/http/serviceprovider/HttpNodeDataSink.java +++ b/ranger-http/src/main/java/io/appform/ranger/http/serviceprovider/HttpNodeDataSink.java @@ -56,6 +56,16 @@ public HttpNodeDataSink(String metricId, Service service, HttpClientConfig confi this.mapper = mapper; } + @Override + public DataStoreType getDataStoreType() { + return DataStoreType.HTTP; + } + + @Override + public String getMetricId() { + return metricId; + } + @Override public void updateState(S serializer, ServiceNode serviceNode) { requireNonNull(config, "client config has not been set for node data"); @@ -90,7 +100,7 @@ private > byte[] getSerializedData(Str try { return serializer.serialize(serviceNode); } catch (Exception e) { - MetricRecorder.recordNodeDataSinkSerDeFailure(DataStoreType.HTTP, metricId, MetricRecorder.SERIALIZAION, serviceName, e.getClass().getSimpleName()); + MetricRecorder.recordNodeDataSinkSerDeFailure(DataStoreType.HTTP, metricId, MetricRecorder.SERIALIZATION, serviceName, e.getClass().getSimpleName()); log.error("Error serializing data for service {} with node {} with exception", serviceName, serviceNode, e); throw e; } diff --git a/ranger-id/perf/results/io.appform.ranger.id.IdGeneratorPerfTest.testGenerate.json b/ranger-id/perf/results/io.appform.ranger.id.IdGeneratorPerfTest.testGenerate.json index 1aa29ce0..d65eceb0 100644 --- a/ranger-id/perf/results/io.appform.ranger.id.IdGeneratorPerfTest.testGenerate.json +++ b/ranger-id/perf/results/io.appform.ranger.id.IdGeneratorPerfTest.testGenerate.json @@ -4,5 +4,5 @@ "iterations" : 4, "threads" : 1, "forks" : 3, - "mean_ops" : 658557.7599494386 + "mean_ops" : 663978.5789639627 } \ No newline at end of file diff --git a/ranger-id/perf/results/io.appform.ranger.id.IdGeneratorPerfTest.testGenerateBase36.json b/ranger-id/perf/results/io.appform.ranger.id.IdGeneratorPerfTest.testGenerateBase36.json index fdbe58d2..524b0dcd 100644 --- a/ranger-id/perf/results/io.appform.ranger.id.IdGeneratorPerfTest.testGenerateBase36.json +++ b/ranger-id/perf/results/io.appform.ranger.id.IdGeneratorPerfTest.testGenerateBase36.json @@ -4,5 +4,5 @@ "iterations" : 4, "threads" : 1, "forks" : 3, - "mean_ops" : 509709.58246890694 + "mean_ops" : 522058.9846283437 } \ No newline at end of file diff --git a/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/serviceprovider/ZkNodeDataSink.java b/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/serviceprovider/ZkNodeDataSink.java index 34e94265..1c351b52 100644 --- a/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/serviceprovider/ZkNodeDataSink.java +++ b/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/serviceprovider/ZkNodeDataSink.java @@ -47,6 +47,16 @@ public ZkNodeDataSink( super(metricId, service, curatorFramework, ZkStoreType.SINK); } + @Override + public DataStoreType getDataStoreType() { + return DataStoreType.ZK; + } + + @Override + public String getMetricId() { + return metricId; + } + @Override public void updateState(S serializer, ServiceNode serviceNode) { if (isStopped()) { @@ -78,7 +88,7 @@ private > byte[] getSerializedData(String s try { return serializer.serialize(serviceNode); } catch (Exception e) { - MetricRecorder.recordNodeDataSinkSerDeFailure(DataStoreType.ZK, metricId, MetricRecorder.SERIALIZAION, serviceName, e.getClass().getSimpleName()); + MetricRecorder.recordNodeDataSinkSerDeFailure(DataStoreType.ZK, metricId, MetricRecorder.SERIALIZATION, serviceName, e.getClass().getSimpleName()); throw e; } } From 647ea5a13ff78fc24c7568f56400a42b7a884f80 Mon Sep 17 00:00:00 2001 From: Jitendra Dhawan Date: Mon, 1 Jun 2026 18:38:18 +0530 Subject: [PATCH 16/24] Update performance test metrics and enhance timer assertions in ServiceRegistryUpdaterMetricsIntegrationTest --- ...iceRegistryUpdaterMetricsIntegrationTest.java | 16 +++++++++++++--- ...nger.id.IdGeneratorPerfTest.testGenerate.json | 2 +- ...d.IdGeneratorPerfTest.testGenerateBase36.json | 2 +- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/ranger-core/src/test/java/io/appform/ranger/core/finder/serviceregistry/ServiceRegistryUpdaterMetricsIntegrationTest.java b/ranger-core/src/test/java/io/appform/ranger/core/finder/serviceregistry/ServiceRegistryUpdaterMetricsIntegrationTest.java index d18480d4..763ab4a0 100644 --- a/ranger-core/src/test/java/io/appform/ranger/core/finder/serviceregistry/ServiceRegistryUpdaterMetricsIntegrationTest.java +++ b/ranger-core/src/test/java/io/appform/ranger/core/finder/serviceregistry/ServiceRegistryUpdaterMetricsIntegrationTest.java @@ -234,9 +234,19 @@ void testRefreshReturnsNull_noSuccessOrFailureTimer_butStaleRetained() { // When refresh returns null, no success timer should be incremented for that second call // The success timer from the first call should be 1 val timerName = "io.appform.ranger.dataSource.ZK." + METRIC_ID + ".nodeDataRefresh.success"; - val timer = metricRegistry.getTimers().get(timerName); - assertNotNull(timer); - assertEquals(1, timer.getCount(), "Only the first successful refresh should be counted"); + val beforeTimer = metricRegistry.getTimers().get(timerName); + final long beforeCount = beforeTimer == null ? 0L : beforeTimer.getCount(); + + // Now set to null and trigger update + dataSource.setNodeList(null); + signal.fire(); + sleep(200); + + val afterTimer = metricRegistry.getTimers().get(timerName); + final long afterCount = afterTimer == null ? 0L : afterTimer.getCount(); + + assertEquals(beforeCount, afterCount, + "Success timer count should not increase when refresh returns null"); } // ==================== Test helpers ==================== diff --git a/ranger-id/perf/results/io.appform.ranger.id.IdGeneratorPerfTest.testGenerate.json b/ranger-id/perf/results/io.appform.ranger.id.IdGeneratorPerfTest.testGenerate.json index d65eceb0..46d9a3f1 100644 --- a/ranger-id/perf/results/io.appform.ranger.id.IdGeneratorPerfTest.testGenerate.json +++ b/ranger-id/perf/results/io.appform.ranger.id.IdGeneratorPerfTest.testGenerate.json @@ -4,5 +4,5 @@ "iterations" : 4, "threads" : 1, "forks" : 3, - "mean_ops" : 663978.5789639627 + "mean_ops" : 689917.755171293 } \ No newline at end of file diff --git a/ranger-id/perf/results/io.appform.ranger.id.IdGeneratorPerfTest.testGenerateBase36.json b/ranger-id/perf/results/io.appform.ranger.id.IdGeneratorPerfTest.testGenerateBase36.json index 524b0dcd..909e6d10 100644 --- a/ranger-id/perf/results/io.appform.ranger.id.IdGeneratorPerfTest.testGenerateBase36.json +++ b/ranger-id/perf/results/io.appform.ranger.id.IdGeneratorPerfTest.testGenerateBase36.json @@ -4,5 +4,5 @@ "iterations" : 4, "threads" : 1, "forks" : 3, - "mean_ops" : 522058.9846283437 + "mean_ops" : 523288.58490266046 } \ No newline at end of file From 6287778c26ae0419576477a80aeca32bc50c8f97 Mon Sep 17 00:00:00 2001 From: Jitendra Dhawan Date: Mon, 1 Jun 2026 20:00:06 +0530 Subject: [PATCH 17/24] Fix all the sonar issues --- ranger-bom/pom.xml | 23 +++++++---- ranger-core/pom.xml | 5 +++ .../ServiceRegistryUpdater.java | 4 +- .../BaseServiceProviderBuilder.java | 8 ++-- .../ranger/core/util/MetricRecorder.java | 7 ++-- ...RegistryUpdaterMetricsIntegrationTest.java | 31 ++++++++++----- .../core/finderhub/ServiceFinderHubTest.java | 12 +++--- .../HealthCheckerMetricsIntegrationTest.java | 38 ++++++++++++------- .../bundle/ServiceDiscoveryBundle.java | 2 +- ...rviceDiscoveryBundleLocalHostPortTest.java | 1 - .../DefaultPortSchemeResolverTest.java | 4 +- .../bundle/ServiceDiscoveryBundle.java | 15 ++++---- ...viceDiscoveryBundleCustomHostPortTest.java | 1 - .../ServiceDiscoveryBundleDwMonitorTest.java | 1 - ...DiscoveryBundleDwStalenessMonitorTest.java | 1 - ...scoveryBundleHierarchicalSelectorTest.java | 1 - ...rviceDiscoveryBundleLocalHostPortTest.java | 6 --- .../ServiceDiscoveryBundleRotationTest.java | 1 - .../bundle/ServiceDiscoveryBundleTest.java | 1 - .../DefaultNodeInfoResolverTest.java | 1 - .../DefaultPortSchemeResolverTest.java | 4 +- .../drove/AbstractRangerDroveHubClient.java | 2 +- .../servicefinder/DroveNodeDataSource.java | 3 -- .../DroveShardedServiceFinderBuilder.java | 2 +- .../DroveUnshardedServiceFinderBuilider.java | 2 +- .../DroveServiceDataSource.java | 3 -- ...eNodeDataSourceMetricsIntegrationTest.java | 9 +++-- .../DroveNodeDataSourceTest.java | 4 +- ...rviceDataSourceMetricsIntegrationTest.java | 18 ++++----- .../DroveServiceDataSourceTest.java | 2 +- ...r.id.IdGeneratorPerfTest.testGenerate.json | 2 +- ...dGeneratorPerfTest.testGenerateBase36.json | 2 +- ranger-server-bundle/README.md | 2 +- ranger-server-dw5-bundle/README.md | 2 +- 34 files changed, 121 insertions(+), 99 deletions(-) diff --git a/ranger-bom/pom.xml b/ranger-bom/pom.xml index e91b5c87..15d0c481 100644 --- a/ranger-bom/pom.xml +++ b/ranger-bom/pom.xml @@ -1,4 +1,20 @@ + + @@ -13,13 +29,6 @@ pom Bill of Materials for Ranger - - 17 - 17 - UTF-8 - 1.7.0 - - diff --git a/ranger-core/pom.xml b/ranger-core/pom.xml index ec9f0b72..ff4d8162 100644 --- a/ranger-core/pom.xml +++ b/ranger-core/pom.xml @@ -48,6 +48,11 @@ failsafe ${failsafe.version} + + jakarta.validation + jakarta.validation-api + 2.0.2 + io.dropwizard dropwizard-metrics diff --git a/ranger-core/src/main/java/io/appform/ranger/core/finder/serviceregistry/ServiceRegistryUpdater.java b/ranger-core/src/main/java/io/appform/ranger/core/finder/serviceregistry/ServiceRegistryUpdater.java index 8bb538c6..1df2fa7c 100644 --- a/ranger-core/src/main/java/io/appform/ranger/core/finder/serviceregistry/ServiceRegistryUpdater.java +++ b/ranger-core/src/main/java/io/appform/ranger/core/finder/serviceregistry/ServiceRegistryUpdater.java @@ -176,13 +176,13 @@ private void updateRegistry() throws InterruptedException { log.warn("Node data source seems to be down. Keeping old list for {}." + " Will update timestamp to keep stale date relevant.", serviceRegistry.getService().getServiceName()); - MetricRecorder.recordStaleDataRetained(serviceRegistry.getService().getServiceName(), - nodeDataSource.getDataStoreType(), nodeDataSource.getMetricId()); serviceRegistry.updateNodes(serviceRegistry.nodeList() .stream() .filter(node -> HealthcheckStatus.healthy == node.getHealthcheckStatus()) .map(node -> node.setLastUpdatedTimeStamp(currTime)) .toList()); + MetricRecorder.recordStaleDataRetained(serviceRegistry.getService().getServiceName(), + nodeDataSource.getDataStoreType(), nodeDataSource.getMetricId()); } } diff --git a/ranger-core/src/main/java/io/appform/ranger/core/serviceprovider/BaseServiceProviderBuilder.java b/ranger-core/src/main/java/io/appform/ranger/core/serviceprovider/BaseServiceProviderBuilder.java index e869fe84..24d1d3b6 100644 --- a/ranger-core/src/main/java/io/appform/ranger/core/serviceprovider/BaseServiceProviderBuilder.java +++ b/ranger-core/src/main/java/io/appform/ranger/core/serviceprovider/BaseServiceProviderBuilder.java @@ -62,10 +62,10 @@ public abstract class BaseServiceProviderBuilder nodeDataSource = null; protected HealthUpdateHandler healthUpdateHandler; - protected final List healthchecks = Lists.newArrayList(); - protected final List> startSignalHandlers = Lists.newArrayList(); - protected final List> stopSignalHandlers = Lists.newArrayList(); - protected final List> additionalRefreshSignals = Lists.newArrayList(); + protected final List healthchecks = new ArrayList<>(); + protected final List> startSignalHandlers = new ArrayList<>(); + protected final List> stopSignalHandlers = new ArrayList<>(); + protected final List> additionalRefreshSignals = new ArrayList<>(); /* list of isolated monitors */ private final List> isolatedMonitors = new ArrayList<>(); diff --git a/ranger-core/src/main/java/io/appform/ranger/core/util/MetricRecorder.java b/ranger-core/src/main/java/io/appform/ranger/core/util/MetricRecorder.java index 5e3903d2..1874c9af 100644 --- a/ranger-core/src/main/java/io/appform/ranger/core/util/MetricRecorder.java +++ b/ranger-core/src/main/java/io/appform/ranger/core/util/MetricRecorder.java @@ -93,14 +93,15 @@ public static void recordNodeDataSinkUpdateStatus(DataStoreType dataStoreType, S public static void recordHealthcheckFailure(DataStoreType dataStoreType, String metricId) { if (metricRegistry != null) { - metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, HEALTHCHECK, metricId, FAILURE)).mark(); + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_SOURCE, dataStoreType.name(), + DATA_SOURCE, metricId, HEALTHCHECK, FAILURE)).mark(); } } public static void recordHealthcheckStatus(DataStoreType dataStoreType, String metricId, boolean healthy) { if (metricRegistry != null) { - metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, HEALTHCHECK, metricId, - healthy ? HEALTHY : UNHEALTHY)).mark(); + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_SOURCE, dataStoreType.name(), + DATA_SOURCE, metricId, HEALTHCHECK, "status", healthy ? HEALTHY : UNHEALTHY)).mark(); } } diff --git a/ranger-core/src/test/java/io/appform/ranger/core/finder/serviceregistry/ServiceRegistryUpdaterMetricsIntegrationTest.java b/ranger-core/src/test/java/io/appform/ranger/core/finder/serviceregistry/ServiceRegistryUpdaterMetricsIntegrationTest.java index 763ab4a0..53829555 100644 --- a/ranger-core/src/test/java/io/appform/ranger/core/finder/serviceregistry/ServiceRegistryUpdaterMetricsIntegrationTest.java +++ b/ranger-core/src/test/java/io/appform/ranger/core/finder/serviceregistry/ServiceRegistryUpdaterMetricsIntegrationTest.java @@ -22,17 +22,22 @@ import io.appform.ranger.core.units.TestNodeData; import io.appform.ranger.core.util.MetricRecorder; import lombok.val; +import org.awaitility.Awaitility; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import java.time.Duration; +import java.time.temporal.TemporalUnit; import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; import java.util.function.Supplier; +import static org.awaitility.Awaitility.await; import static org.junit.jupiter.api.Assertions.*; class ServiceRegistryUpdaterMetricsIntegrationTest { @@ -260,11 +265,7 @@ private void awaitRefresh(ServiceRegistry registry) { } private void sleep(long ms) { - try { - Thread.sleep(ms); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } + await().pollDelay(Duration.ofMillis(ms)).until(() -> true); } // ==================== Test implementations ==================== @@ -315,13 +316,19 @@ public Optional>> refresh(TestDeserializer deseri } @Override - public void start() {} + public void start() { + // No-op for test + } @Override - public void ensureConnected() {} + public void ensureConnected() { + // No-op for test + } @Override - public void stop() {} + public void stop() { + // No-op for test + } @Override public boolean isActive() { @@ -339,9 +346,13 @@ public void fire() { } @Override - public void start() {} + public void start() { + // No-op for test + } @Override - public void stop() {} + public void stop() { + // No-op for test + } } } diff --git a/ranger-core/src/test/java/io/appform/ranger/core/finderhub/ServiceFinderHubTest.java b/ranger-core/src/test/java/io/appform/ranger/core/finderhub/ServiceFinderHubTest.java index 29fa77c7..291a25ea 100644 --- a/ranger-core/src/test/java/io/appform/ranger/core/finderhub/ServiceFinderHubTest.java +++ b/ranger-core/src/test/java/io/appform/ranger/core/finderhub/ServiceFinderHubTest.java @@ -44,7 +44,7 @@ class ServiceFinderHubTest { private final ServiceFinderHub> serviceFinderHub = new ServiceFinderHub<>( - new DynamicDataSource(Lists.newArrayList(new Service("NS", "PRE_REGISTERED_SERVICE"))), + new DynamicDataSource(List.of(new Service("NS", "PRE_REGISTERED_SERVICE"))), service -> new TestServiceFinderBuilder() .withMetricId("test-metric") @@ -74,7 +74,7 @@ void testDynamicServiceAddition() { @Test void testTimeoutOnHubStartup() { var testServiceFinderHub = new TestServiceFinderHubBuilder() - .withServiceDataSource(new DynamicDataSource(Lists.newArrayList(new Service("NS", "SERVICE")))) + .withServiceDataSource(new DynamicDataSource(List.of(new Service("NS", "SERVICE")))) .withServiceFinderFactory(new TestServiceFinderFactory()) .withRefreshFrequencyMs(5_000) .withHubStartTimeout(1_000) @@ -92,7 +92,7 @@ void testTimeoutOnHubStartup() { @Test void testDelayedServiceAddition() { - val delayedHub = new ServiceFinderHub<>(new DynamicDataSource(Lists.newArrayList(new Service("NS", "SERVICE"))), + val delayedHub = new ServiceFinderHub<>(new DynamicDataSource(List.of(new Service("NS", "SERVICE"))), service -> new TestServiceFinderBuilder() .withMetricId("test-metric") .withNamespace(service.getNamespace()) @@ -101,7 +101,7 @@ void testDelayedServiceAddition() { .withSleepDuration(5) .build(), 1_000, 5_000, Set.of()); Assertions.assertThrows(IllegalStateException.class, delayedHub::start); - val serviceFinderHub = new ServiceFinderHub<>(new DynamicDataSource(Lists.newArrayList(new Service("NS", "SERVICE"))), + val serviceFinderHub = new ServiceFinderHub<>(new DynamicDataSource(List.of(new Service("NS", "SERVICE"))), service -> new TestServiceFinderBuilder() .withMetricId("test-metric") .withNamespace(service.getNamespace()) @@ -136,7 +136,7 @@ void testDynamicServiceAdditionWithNonDynamicDataSource() { void testWeightedNodeSelectionWithVaryingWeights() { final ServiceFinderHub> serviceFinderHubVaryingWeights = new ServiceFinderHub<>( - new DynamicDataSource(Lists.newArrayList(new Service("NS", "PRE_REGISTERED_SERVICE"))), + new DynamicDataSource(List.of(new Service("NS", "PRE_REGISTERED_SERVICE"))), service -> new TestServiceFinderBuilder() .withMetricId("test-metric") @@ -241,7 +241,7 @@ public boolean isActive() { void testWeightedNodeSelectionWithVaryingNodeAge() { final ServiceFinderHub> serviceFinderHubVaryingNodeAge = new ServiceFinderHub<>( - new DynamicDataSource(Lists.newArrayList(new Service("NS", "PRE_REGISTERED_SERVICE"))), + new DynamicDataSource(List.of(new Service("NS", "PRE_REGISTERED_SERVICE"))), service -> new TestServiceFinderBuilder() .withMetricId("test-metric") diff --git a/ranger-core/src/test/java/io/appform/ranger/core/healthcheck/HealthCheckerMetricsIntegrationTest.java b/ranger-core/src/test/java/io/appform/ranger/core/healthcheck/HealthCheckerMetricsIntegrationTest.java index 7f3d862f..691093ad 100644 --- a/ranger-core/src/test/java/io/appform/ranger/core/healthcheck/HealthCheckerMetricsIntegrationTest.java +++ b/ranger-core/src/test/java/io/appform/ranger/core/healthcheck/HealthCheckerMetricsIntegrationTest.java @@ -31,6 +31,14 @@ class HealthCheckerMetricsIntegrationTest { private MetricRegistry metricRegistry; private static final String METRIC_ID = "test-hc-metric"; + // Metric keys produced by MetricRecorder for DataStoreType.ZK and METRIC_ID: + // recordHealthcheckStatus -> io.appform.ranger.dataSource.ZK.dataSource..healthcheck.status. + // recordHealthcheckFailure -> io.appform.ranger.dataSource.ZK.dataSource..healthcheck.failure + private static final String HC_PREFIX = "io.appform.ranger.dataSource.ZK.dataSource." + METRIC_ID + ".healthcheck"; + private static final String HEALTHY_KEY = HC_PREFIX + ".status.healthy"; + private static final String UNHEALTHY_KEY = HC_PREFIX + ".status.unhealthy"; + private static final String FAILURE_KEY = HC_PREFIX + ".failure"; + @BeforeEach void setUp() { metricRegistry = new MetricRegistry(); @@ -48,14 +56,14 @@ void testHealthyCheck_recordsHealthyMetric() { assertEquals(HealthcheckStatus.healthy, result.getStatus()); // Verify healthy metric recorded - val healthyMeter = metricRegistry.getMeters().get("io.appform.ranger.healthcheck." + METRIC_ID + ".healthy"); + val healthyMeter = metricRegistry.getMeters().get(HEALTHY_KEY); assertNotNull(healthyMeter, "Healthy meter should exist"); assertEquals(1, healthyMeter.getCount()); // Verify no failure metric - assertNull(metricRegistry.getMeters().get("io.appform.ranger.healthcheck." + METRIC_ID + ".failure")); + assertNull(metricRegistry.getMeters().get(FAILURE_KEY)); // Verify no unhealthy metric - assertNull(metricRegistry.getMeters().get("io.appform.ranger.healthcheck." + METRIC_ID + ".unhealthy")); + assertNull(metricRegistry.getMeters().get(UNHEALTHY_KEY)); } @Test @@ -69,12 +77,12 @@ void testUnhealthyCheck_recordsUnhealthyMetric() { assertEquals(HealthcheckStatus.unhealthy, result.getStatus()); // Verify unhealthy metric recorded - val unhealthyMeter = metricRegistry.getMeters().get("io.appform.ranger.healthcheck." + METRIC_ID + ".unhealthy"); + val unhealthyMeter = metricRegistry.getMeters().get(UNHEALTHY_KEY); assertNotNull(unhealthyMeter, "Unhealthy meter should exist"); assertEquals(1, unhealthyMeter.getCount()); // No healthy metric - assertNull(metricRegistry.getMeters().get("io.appform.ranger.healthcheck." + METRIC_ID + ".healthy")); + assertNull(metricRegistry.getMeters().get(HEALTHY_KEY)); } @Test @@ -89,12 +97,12 @@ void testExceptionInHealthcheck_recordsFailureAndUnhealthyMetrics() { assertEquals(HealthcheckStatus.unhealthy, result.getStatus()); // Verify failure metric recorded (exception path) - val failureMeter = metricRegistry.getMeters().get("io.appform.ranger.healthcheck." + METRIC_ID + ".failure"); + val failureMeter = metricRegistry.getMeters().get(FAILURE_KEY); assertNotNull(failureMeter, "Failure meter should exist on exception"); assertEquals(1, failureMeter.getCount()); // Verify unhealthy status metric also recorded - val unhealthyMeter = metricRegistry.getMeters().get("io.appform.ranger.healthcheck." + METRIC_ID + ".unhealthy"); + val unhealthyMeter = metricRegistry.getMeters().get(UNHEALTHY_KEY); assertNotNull(unhealthyMeter, "Unhealthy meter should be recorded after exception"); assertEquals(1, unhealthyMeter.getCount()); } @@ -111,7 +119,7 @@ void testMultipleHealthchecks_firstUnhealthy_shortCircuits() { assertNotNull(result); assertEquals(HealthcheckStatus.unhealthy, result.getStatus()); - val unhealthyMeter = metricRegistry.getMeters().get("io.appform.ranger.healthcheck." + METRIC_ID + ".unhealthy"); + val unhealthyMeter = metricRegistry.getMeters().get(UNHEALTHY_KEY); assertNotNull(unhealthyMeter); assertEquals(1, unhealthyMeter.getCount()); } @@ -129,7 +137,7 @@ void testMultipleHealthchecks_allHealthy() { assertNotNull(result); assertEquals(HealthcheckStatus.healthy, result.getStatus()); - val healthyMeter = metricRegistry.getMeters().get("io.appform.ranger.healthcheck." + METRIC_ID + ".healthy"); + val healthyMeter = metricRegistry.getMeters().get(HEALTHY_KEY); assertNotNull(healthyMeter); assertEquals(1, healthyMeter.getCount()); } @@ -143,7 +151,7 @@ void testRepeatedCalls_metricsAccumulate() { healthChecker.get(); healthChecker.get(); - val healthyMeter = metricRegistry.getMeters().get("io.appform.ranger.healthcheck." + METRIC_ID + ".healthy"); + val healthyMeter = metricRegistry.getMeters().get(HEALTHY_KEY); assertNotNull(healthyMeter); assertEquals(3, healthyMeter.getCount(), "Healthy metric count should accumulate"); } @@ -165,10 +173,12 @@ void testHealthStatusTransition_bothMetricsRecorded() { assertNotNull(result2); assertEquals(HealthcheckStatus.unhealthy, result2.getStatus()); - val healthyMeter = metricRegistry.getMeters().get("io.appform.ranger.healthcheck." + METRIC_ID + ".healthy"); + val healthyMeter = metricRegistry.getMeters().get(HEALTHY_KEY); + assertNotNull(healthyMeter); assertEquals(1, healthyMeter.getCount()); - val unhealthyMeter = metricRegistry.getMeters().get("io.appform.ranger.healthcheck." + METRIC_ID + ".unhealthy"); + val unhealthyMeter = metricRegistry.getMeters().get(UNHEALTHY_KEY); + assertNotNull(unhealthyMeter); assertEquals(1, unhealthyMeter.getCount()); } @@ -185,12 +195,12 @@ void testExceptionInSecondHealthcheck_recordsFailure() { assertEquals(HealthcheckStatus.unhealthy, result.getStatus()); // Failure meter for the exception - val failureMeter = metricRegistry.getMeters().get("io.appform.ranger.healthcheck." + METRIC_ID + ".failure"); + val failureMeter = metricRegistry.getMeters().get(FAILURE_KEY); assertNotNull(failureMeter); assertEquals(1, failureMeter.getCount()); // Unhealthy status because the second check failed - val unhealthyMeter = metricRegistry.getMeters().get("io.appform.ranger.healthcheck." + METRIC_ID + ".unhealthy"); + val unhealthyMeter = metricRegistry.getMeters().get(UNHEALTHY_KEY); assertNotNull(unhealthyMeter); assertEquals(1, unhealthyMeter.getCount()); } diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundle.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundle.java index a4c627b8..24f57be6 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundle.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundle.java @@ -97,7 +97,7 @@ @Slf4j public abstract class ServiceDiscoveryBundle implements ConfiguredBundle { - private final List healthchecks = Lists.newArrayList(); + private final List healthchecks = new ArrayList<>(); private final List globalIdConstraints; private ServiceDiscoveryConfiguration serviceDiscoveryConfiguration; private ServiceProvider> serviceProvider; diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleLocalHostPortTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleLocalHostPortTest.java index ea9466f4..7f625b9a 100644 --- a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleLocalHostPortTest.java +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleLocalHostPortTest.java @@ -129,7 +129,6 @@ void shouldThrowExceptionForInvalidZkHost() { when(environment.admin()).thenReturn(adminEnvironment); serviceDiscoveryConfiguration = ServiceDiscoveryConfiguration.builder() - .zookeeper(String.format("%s:2181", UUID.randomUUID())) .namespace("test") .environment("testing") diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/resolvers/DefaultPortSchemeResolverTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/resolvers/DefaultPortSchemeResolverTest.java index 8e79e40c..1a678867 100644 --- a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/resolvers/DefaultPortSchemeResolverTest.java +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/resolvers/DefaultPortSchemeResolverTest.java @@ -25,6 +25,8 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.util.List; + import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -34,7 +36,7 @@ class DefaultPortSchemeResolverTest { void testPortSchemeDefaultServerFactory() { val server = mock(DefaultServerFactory.class); val connectorFactory = mock(HttpConnectorFactory.class); - when(server.getApplicationConnectors()).thenReturn(Lists.newArrayList(connectorFactory)); + when(server.getApplicationConnectors()).thenReturn(List.of(connectorFactory)); val resolver = new DefaultPortSchemeResolver<>(); val configuration = mock(Configuration.class); when(configuration.getServerFactory()).thenReturn(server); diff --git a/ranger-discovery-dw5-bundle/src/main/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundle.java b/ranger-discovery-dw5-bundle/src/main/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundle.java index a3e428fa..5523d8b2 100644 --- a/ranger-discovery-dw5-bundle/src/main/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundle.java +++ b/ranger-discovery-dw5-bundle/src/main/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundle.java @@ -77,6 +77,7 @@ import java.io.IOException; import java.net.InetAddress; import java.net.UnknownHostException; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.concurrent.TimeUnit; @@ -95,7 +96,7 @@ @Slf4j public abstract class ServiceDiscoveryBundle implements ConfiguredBundle { - private final List healthchecks = Lists.newArrayList(); + private final List healthchecks = new ArrayList<>(); private final List globalIdConstraints; private ServiceDiscoveryConfiguration serviceDiscoveryConfiguration; private ServiceProvider> serviceProvider; @@ -117,8 +118,8 @@ protected ServiceDiscoveryBundle() { protected ServiceDiscoveryBundle(List globalIdConstraints) { this.globalIdConstraints = globalIdConstraints != null - ? globalIdConstraints - : Collections.emptyList(); + ? globalIdConstraints + : Collections.emptyList(); } @Override @@ -179,6 +180,10 @@ protected ServiceNodeSelector getServiceNodeSelector(T configuration) protected abstract String getServiceName(T configuration); + protected Supplier getWeightSupplier() { + return () -> 1.0; + } + protected NodeInfoResolver createNodeInfoResolver() { return new DefaultNodeInfoResolver(); } @@ -202,10 +207,6 @@ protected List> getHealthMonitors() { return Collections.emptyList(); } - protected Supplier getWeightSupplier() { - return () -> 1.0; - } - @SuppressWarnings("unused") protected int getPort(T configuration) { Preconditions.checkArgument(Constants.DEFAULT_PORT != serviceDiscoveryConfiguration.getPublishedPort() diff --git a/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleCustomHostPortTest.java b/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleCustomHostPortTest.java index d8b4b646..35c48310 100644 --- a/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleCustomHostPortTest.java +++ b/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleCustomHostPortTest.java @@ -107,7 +107,6 @@ void setup() throws Exception { testingCluster.start(); serviceDiscoveryConfiguration = ServiceDiscoveryConfiguration.builder() - .zookeeper(testingCluster.getConnectString()) .namespace("test") .environment("testing") diff --git a/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleDwMonitorTest.java b/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleDwMonitorTest.java index e58b847f..2cb13f4a 100644 --- a/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleDwMonitorTest.java +++ b/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleDwMonitorTest.java @@ -122,7 +122,6 @@ protected Result check() { testingCluster.start(); serviceDiscoveryConfiguration = ServiceDiscoveryConfiguration.builder() - .zookeeper(testingCluster.getConnectString()) .namespace("test") .environment("testing") diff --git a/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleDwStalenessMonitorTest.java b/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleDwStalenessMonitorTest.java index 764c1db9..823455e8 100644 --- a/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleDwStalenessMonitorTest.java +++ b/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleDwStalenessMonitorTest.java @@ -121,7 +121,6 @@ protected Result check() { testingCluster.start(); serviceDiscoveryConfiguration = ServiceDiscoveryConfiguration.builder() - .zookeeper(testingCluster.getConnectString()) .namespace("test") .environment("testing") diff --git a/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleHierarchicalSelectorTest.java b/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleHierarchicalSelectorTest.java index ce77e561..4f0f1cb2 100644 --- a/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleHierarchicalSelectorTest.java +++ b/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleHierarchicalSelectorTest.java @@ -101,7 +101,6 @@ void setup() throws Exception { DnsCacheManipulator.setDnsCache("TestHost", "127.0.0.1"); serviceDiscoveryConfiguration = ServiceDiscoveryConfiguration.builder() - .zookeeper(testingCluster.getConnectString()) .namespace("test") .environment("x.y") diff --git a/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleLocalHostPortTest.java b/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleLocalHostPortTest.java index 67e3accd..d8838aeb 100644 --- a/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleLocalHostPortTest.java +++ b/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleLocalHostPortTest.java @@ -100,7 +100,6 @@ void shouldFailLocalhostPublish() { when(environment.admin()).thenReturn(adminEnvironment); serviceDiscoveryConfiguration = ServiceDiscoveryConfiguration.builder() - .zookeeper("myzookeeper:2181,myfavzookeeper:2181") .namespace("test") .environment("testing") @@ -130,7 +129,6 @@ void shouldThrowExceptionForInvalidZkHost() { when(environment.admin()).thenReturn(adminEnvironment); serviceDiscoveryConfiguration = ServiceDiscoveryConfiguration.builder() - .zookeeper(String.format("%s:2181", UUID.randomUUID())) .namespace("test") .environment("testing") @@ -168,7 +166,6 @@ void testPublishWithEmptyZkHost() throws UnknownHostException { when(environment.admin()).thenReturn(adminEnvironment); serviceDiscoveryConfiguration = ServiceDiscoveryConfiguration.builder() - .zookeeper("myzookeeper:2181,myfavzookeeper:2181") .namespace("test") .environment("testing") @@ -206,7 +203,6 @@ void testPublishWithNullZkHost() throws UnknownHostException { when(environment.admin()).thenReturn(adminEnvironment); serviceDiscoveryConfiguration = ServiceDiscoveryConfiguration.builder() - .zookeeper("myzookeeper:2181,myfavzookeeper:2181") .namespace("test") .environment("testing") @@ -243,7 +239,6 @@ void shouldPublishingToLocalZk() { when(environment.admin()).thenReturn(adminEnvironment); serviceDiscoveryConfiguration = ServiceDiscoveryConfiguration.builder() - .zookeeper("localhost:2181,myfavzookeeper:2181") .namespace("test") .environment("testing") @@ -275,7 +270,6 @@ void shouldPublishToRemoteZk() { when(environment.admin()).thenReturn(adminEnvironment); serviceDiscoveryConfiguration = ServiceDiscoveryConfiguration.builder() - .zookeeper("myfavzookeeper:2181") .namespace("test") .environment("testing") diff --git a/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleRotationTest.java b/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleRotationTest.java index 2735ff6e..88c7bec7 100644 --- a/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleRotationTest.java +++ b/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleRotationTest.java @@ -102,7 +102,6 @@ void setup() throws Exception { testingCluster.start(); serviceDiscoveryConfiguration = ServiceDiscoveryConfiguration.builder() - .zookeeper(testingCluster.getConnectString()) .namespace("test") .environment("testing") diff --git a/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleTest.java b/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleTest.java index c4b4bdd9..17efe152 100644 --- a/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleTest.java +++ b/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleTest.java @@ -105,7 +105,6 @@ void setup() throws Exception { testingCluster.start(); serviceDiscoveryConfiguration = ServiceDiscoveryConfiguration.builder() - .zookeeper(testingCluster.getConnectString()) .namespace("test") .environment("testing") diff --git a/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/resolvers/DefaultNodeInfoResolverTest.java b/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/resolvers/DefaultNodeInfoResolverTest.java index fd577bae..fdf07928 100644 --- a/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/resolvers/DefaultNodeInfoResolverTest.java +++ b/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/resolvers/DefaultNodeInfoResolverTest.java @@ -27,7 +27,6 @@ class DefaultNodeInfoResolverTest { @Test void testNodeInfoResolver() { val configuration = ServiceDiscoveryConfiguration.builder() - .zookeeper("connectionString") .namespace("test") .environment("testing") diff --git a/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/resolvers/DefaultPortSchemeResolverTest.java b/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/resolvers/DefaultPortSchemeResolverTest.java index a2efd010..14503b3a 100644 --- a/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/resolvers/DefaultPortSchemeResolverTest.java +++ b/ranger-discovery-dw5-bundle/src/test/java/io/appform/ranger/discovery/bundle/resolvers/DefaultPortSchemeResolverTest.java @@ -25,6 +25,8 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.util.List; + import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -34,7 +36,7 @@ class DefaultPortSchemeResolverTest { void testPortSchemeDefaultServerFactory() { val server = mock(DefaultServerFactory.class); val connectorFactory = mock(HttpConnectorFactory.class); - when(server.getApplicationConnectors()).thenReturn(Lists.newArrayList(connectorFactory)); + when(server.getApplicationConnectors()).thenReturn(List.of(connectorFactory)); val resolver = new DefaultPortSchemeResolver<>(); val configuration = mock(Configuration.class); when(configuration.getServerFactory()).thenReturn(server); diff --git a/ranger-drove-client/src/main/java/io/appform/ranger/client/drove/AbstractRangerDroveHubClient.java b/ranger-drove-client/src/main/java/io/appform/ranger/client/drove/AbstractRangerDroveHubClient.java index fdf14d27..0c8cd508 100644 --- a/ranger-drove-client/src/main/java/io/appform/ranger/client/drove/AbstractRangerDroveHubClient.java +++ b/ranger-drove-client/src/main/java/io/appform/ranger/client/drove/AbstractRangerDroveHubClient.java @@ -46,7 +46,7 @@ public abstract class AbstractRangerDroveHubClient(metricId, clientConfig, getMapper(), getNamespace(), droveCommunicator); + return new DroveServiceDataSource<>(clientConfig, getMapper(), getNamespace(), droveCommunicator); } @Override diff --git a/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinder/DroveNodeDataSource.java b/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinder/DroveNodeDataSource.java index a56c7fd7..17111810 100644 --- a/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinder/DroveNodeDataSource.java +++ b/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinder/DroveNodeDataSource.java @@ -41,17 +41,14 @@ @Slf4j public class DroveNodeDataSource> extends DroveNodeDataStoreConnector implements NodeDataSource { - private final String metricId; private final Service service; public DroveNodeDataSource( - String metricId, Service service, final DroveUpstreamConfig config, ObjectMapper mapper, DroveCommunicator droveClient) { super(config, mapper, droveClient); - this.metricId = metricId; this.service = service; } diff --git a/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinder/DroveShardedServiceFinderBuilder.java b/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinder/DroveShardedServiceFinderBuilder.java index 96741e7d..472f617c 100644 --- a/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinder/DroveShardedServiceFinderBuilder.java +++ b/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinder/DroveShardedServiceFinderBuilder.java @@ -60,7 +60,7 @@ public SimpleShardedServiceFinder build() { @Override protected NodeDataSource> dataSource(String metricId, Service service) { - return new DroveNodeDataSource<>(metricId, service, clientConfig, + return new DroveNodeDataSource<>(service, clientConfig, mapper, Objects.requireNonNullElseGet(droveClient, () -> RangerDroveUtils.buildDroveClient( namespace, clientConfig, mapper))); diff --git a/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinder/DroveUnshardedServiceFinderBuilider.java b/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinder/DroveUnshardedServiceFinderBuilider.java index 2cb63459..e8173583 100644 --- a/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinder/DroveUnshardedServiceFinderBuilider.java +++ b/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinder/DroveUnshardedServiceFinderBuilider.java @@ -57,7 +57,7 @@ public SimpleUnshardedServiceFinder build() { @Override protected NodeDataSource> dataSource(String metricId, Service service) { - return new DroveNodeDataSource<>(metricId, + return new DroveNodeDataSource<>( service, clientConfig, mapper, Objects.requireNonNullElseGet(droveCommunicator, diff --git a/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinderhub/DroveServiceDataSource.java b/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinderhub/DroveServiceDataSource.java index 3b6a3d52..2b9c9365 100644 --- a/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinderhub/DroveServiceDataSource.java +++ b/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinderhub/DroveServiceDataSource.java @@ -34,17 +34,14 @@ @Slf4j public class DroveServiceDataSource extends DroveNodeDataStoreConnector implements ServiceDataSource { - private final String metricId; private final String namespace; public DroveServiceDataSource( - final String metricId, final DroveUpstreamConfig config, final ObjectMapper mapper, final String namespace, final DroveCommunicator droveClient) { super(config, mapper, droveClient); - this.metricId = metricId; this.namespace = namespace; } diff --git a/ranger-drove/src/test/java/io/appform/ranger/drove/servicefinder/DroveNodeDataSourceMetricsIntegrationTest.java b/ranger-drove/src/test/java/io/appform/ranger/drove/servicefinder/DroveNodeDataSourceMetricsIntegrationTest.java index 01003152..5ddae2e4 100644 --- a/ranger-drove/src/test/java/io/appform/ranger/drove/servicefinder/DroveNodeDataSourceMetricsIntegrationTest.java +++ b/ranger-drove/src/test/java/io/appform/ranger/drove/servicefinder/DroveNodeDataSourceMetricsIntegrationTest.java @@ -74,7 +74,7 @@ void testIsActive_healthyUpstream_recordsActiveStatus(WireMockRuntimeInfo wm) { try (val droveClient = buildClient(config)) { val service = new Service("testns", "TEST_APP"); val dataSource = new DroveNodeDataSource>( - "drove-node-src-1", service, config, MAPPER, droveClient); + service, config, MAPPER, droveClient); val active = dataSource.isActive(); @@ -100,12 +100,13 @@ void testIsActive_unhealthyUpstream_recordsInactiveStatus(WireMockRuntimeInfo wm try (val droveClient = buildClient(config)) { val service = new Service("testns", "TEST_APP"); val dataSource = new DroveNodeDataSource>( - "drove-node-src-2", service, config, MAPPER, droveClient); + service, config, MAPPER, droveClient); // First call to services fails, setting upstreamAvailable to false try { droveClient.services(); } catch (Exception ignored) { + // Ignored } val active = dataSource.isActive(); @@ -139,7 +140,7 @@ void testRefresh_success_returnsNodes(WireMockRuntimeInfo wm) { try (val droveClient = buildClient(config)) { val service = new Service("testns", "TEST_APP"); val dataSource = new DroveNodeDataSource>( - "drove-node-src-3", service, config, MAPPER, droveClient); + service, config, MAPPER, droveClient); val result = dataSource.refresh(STRING_DESERIALIZER); @@ -167,7 +168,7 @@ void testRefresh_communicationFailure_returnsEmpty(WireMockRuntimeInfo wm) { try (val droveClient = buildClient(config)) { val service = new Service("testns", "FAIL_APP"); val dataSource = new DroveNodeDataSource>( - "drove-node-src-4", service, config, MAPPER, droveClient); + service, config, MAPPER, droveClient); val result = dataSource.refresh(STRING_DESERIALIZER); diff --git a/ranger-drove/src/test/java/io/appform/ranger/drove/servicefinder/DroveNodeDataSourceTest.java b/ranger-drove/src/test/java/io/appform/ranger/drove/servicefinder/DroveNodeDataSourceTest.java index aec348a7..7559d376 100644 --- a/ranger-drove/src/test/java/io/appform/ranger/drove/servicefinder/DroveNodeDataSourceTest.java +++ b/ranger-drove/src/test/java/io/appform/ranger/drove/servicefinder/DroveNodeDataSourceTest.java @@ -75,7 +75,7 @@ void testSuccess() { Map.of(), List.of(new ExposedAppInfo.ExposedHost("h1", 100, TCP), new ExposedAppInfo.ExposedHost("h2", 100, TCP))))); - val ds = new DroveNodeDataSource("testDroveSource", + val ds = new DroveNodeDataSource( service, config, MAPPER, droveClient); @@ -100,7 +100,7 @@ void testUpstreamFailure() { when(droveClient.listNodes(any(Service.class))) .thenThrow(new DroveCommunicationException("test")); - val ds = new DroveNodeDataSource("testDroveSource", + val ds = new DroveNodeDataSource( service, config, MAPPER, droveClient); diff --git a/ranger-drove/src/test/java/io/appform/ranger/drove/servicefinderhub/DroveServiceDataSourceMetricsIntegrationTest.java b/ranger-drove/src/test/java/io/appform/ranger/drove/servicefinderhub/DroveServiceDataSourceMetricsIntegrationTest.java index 47c26f9a..c2f22754 100644 --- a/ranger-drove/src/test/java/io/appform/ranger/drove/servicefinderhub/DroveServiceDataSourceMetricsIntegrationTest.java +++ b/ranger-drove/src/test/java/io/appform/ranger/drove/servicefinderhub/DroveServiceDataSourceMetricsIntegrationTest.java @@ -72,9 +72,9 @@ ApplicationState.RUNNING, new Date(), new Date()), .willReturn(okJson(MAPPER.writeValueAsString(response)))); val config = buildConfig(wm, "drove-svc-src-1"); - try (val droveClient = buildClient(wm, config)) { + try (val droveClient = buildClient(config)) { val dataSource = new DroveServiceDataSource<>( - "drove-svc-src-1", config, MAPPER, "testns", droveClient); + config, MAPPER, "testns", droveClient); val services = dataSource.services(); @@ -103,9 +103,9 @@ void testServices_failure_recordsFetchFailure(WireMockRuntimeInfo wm) { .willReturn(aResponse().withStatus(500).withBody("Error"))); val config = buildConfig(wm, "drove-svc-src-2"); - try (val droveClient = buildClient(wm, config)) { + try (val droveClient = buildClient(config)) { val dataSource = new DroveServiceDataSource<>( - "drove-svc-src-2", config, MAPPER, "testns", droveClient); + config, MAPPER, "testns", droveClient); assertThrows(DroveCommunicationException.class, dataSource::services); @@ -129,9 +129,9 @@ void testIsActive_healthy_recordsActiveStatus(WireMockRuntimeInfo wm) { .willReturn(okJson(MAPPER.writeValueAsString(response)))); val config = buildConfig(wm, "drove-svc-src-3"); - try (val droveClient = buildClient(wm, config)) { + try (val droveClient = buildClient(config)) { val dataSource = new DroveServiceDataSource<>( - "drove-svc-src-3", config, MAPPER, "testns", droveClient); + config, MAPPER, "testns", droveClient); val active = dataSource.isActive(); @@ -155,9 +155,9 @@ void testIsActive_afterFailedCall_recordsInactiveStatus(WireMockRuntimeInfo wm) .willReturn(aResponse().withStatus(500).withBody("Error"))); val config = buildConfig(wm, "drove-svc-src-4"); - try (val droveClient = buildClient(wm, config)) { + try (val droveClient = buildClient(config)) { val dataSource = new DroveServiceDataSource<>( - "drove-svc-src-4", config, MAPPER, "testns", droveClient); + config, MAPPER, "testns", droveClient); // First call fails and sets upstreamAvailable to false assertThrows(DroveCommunicationException.class, dataSource::services); @@ -186,7 +186,7 @@ private DroveUpstreamConfig buildConfig(WireMockRuntimeInfo wm, String metricId) .build(); } - private DroveCommunicator buildClient(WireMockRuntimeInfo wm, DroveUpstreamConfig config) { + private DroveCommunicator buildClient(DroveUpstreamConfig config) { return RangerDroveUtils.buildDroveClient("testns", config, MAPPER); } } diff --git a/ranger-drove/src/test/java/io/appform/ranger/drove/servicefinderhub/DroveServiceDataSourceTest.java b/ranger-drove/src/test/java/io/appform/ranger/drove/servicefinderhub/DroveServiceDataSourceTest.java index 0f025fa6..fe5fac64 100644 --- a/ranger-drove/src/test/java/io/appform/ranger/drove/servicefinderhub/DroveServiceDataSourceTest.java +++ b/ranger-drove/src/test/java/io/appform/ranger/drove/servicefinderhub/DroveServiceDataSourceTest.java @@ -100,7 +100,7 @@ void testServiceDataSource(WireMockRuntimeInfo wireMockRuntimeInfo) { val namespace = "test"; try(val droveClient = RangerDroveUtils.buildDroveClient(namespace, clientConfig, MAPPER)) { val finder = new DroveServiceDataSource( - null, clientConfig, + clientConfig, MAPPER, namespace, droveClient); diff --git a/ranger-id/perf/results/io.appform.ranger.id.IdGeneratorPerfTest.testGenerate.json b/ranger-id/perf/results/io.appform.ranger.id.IdGeneratorPerfTest.testGenerate.json index 46d9a3f1..5be9813c 100644 --- a/ranger-id/perf/results/io.appform.ranger.id.IdGeneratorPerfTest.testGenerate.json +++ b/ranger-id/perf/results/io.appform.ranger.id.IdGeneratorPerfTest.testGenerate.json @@ -4,5 +4,5 @@ "iterations" : 4, "threads" : 1, "forks" : 3, - "mean_ops" : 689917.755171293 + "mean_ops" : 678336.7125441622 } \ No newline at end of file diff --git a/ranger-id/perf/results/io.appform.ranger.id.IdGeneratorPerfTest.testGenerateBase36.json b/ranger-id/perf/results/io.appform.ranger.id.IdGeneratorPerfTest.testGenerateBase36.json index 909e6d10..08d82286 100644 --- a/ranger-id/perf/results/io.appform.ranger.id.IdGeneratorPerfTest.testGenerateBase36.json +++ b/ranger-id/perf/results/io.appform.ranger.id.IdGeneratorPerfTest.testGenerateBase36.json @@ -4,5 +4,5 @@ "iterations" : 4, "threads" : 1, "forks" : 3, - "mean_ops" : 523288.58490266046 + "mean_ops" : 535260.4479626919 } \ No newline at end of file diff --git a/ranger-server-bundle/README.md b/ranger-server-bundle/README.md index eae598f5..24a9736b 100644 --- a/ranger-server-bundle/README.md +++ b/ranger-server-bundle/README.md @@ -15,7 +15,7 @@ Ranger server bundle is a common dropwizard bundle atop which we could implement @Override protected List>> withHubs(AppConfiguration configuration) { - return Lists.newArrayList( + return List.of( RangerServerUtils.buildRangerHub(curatorFramework, rangerConfiguration, environment.getObjectMapper()) ); } diff --git a/ranger-server-dw5-bundle/README.md b/ranger-server-dw5-bundle/README.md index eae598f5..24a9736b 100644 --- a/ranger-server-dw5-bundle/README.md +++ b/ranger-server-dw5-bundle/README.md @@ -15,7 +15,7 @@ Ranger server bundle is a common dropwizard bundle atop which we could implement @Override protected List>> withHubs(AppConfiguration configuration) { - return Lists.newArrayList( + return List.of( RangerServerUtils.buildRangerHub(curatorFramework, rangerConfiguration, environment.getObjectMapper()) ); } From f08051d419f35c1ad9f95511a633c4d4667ed30f Mon Sep 17 00:00:00 2001 From: Jitendra Dhawan Date: Tue, 2 Jun 2026 09:27:25 +0530 Subject: [PATCH 18/24] Add aggregate meters for null/empty responses and parse failures --- .../ranger/core/util/MetricRecorder.java | 16 ++++ ...rviceDiscoveryBundleLocalHostPortTest.java | 1 - .../ranger/discovery/core/Constants.java | 2 +- .../drove/common/DroveApiCommunicator.java | 2 + ...ApiCommunicatorMetricsIntegrationTest.java | 83 +++++++++++++++++++ .../servicefinder/HttpApiCommunicator.java | 3 + ...ApiCommunicatorMetricsIntegrationTest.java | 5 ++ .../servicefinder/ZkNodeDataSource.java | 5 ++ ...kNodeDataSourceMetricsIntegrationTest.java | 15 ++++ 9 files changed, 130 insertions(+), 2 deletions(-) diff --git a/ranger-core/src/main/java/io/appform/ranger/core/util/MetricRecorder.java b/ranger-core/src/main/java/io/appform/ranger/core/util/MetricRecorder.java index 1874c9af..b0bfb4ae 100644 --- a/ranger-core/src/main/java/io/appform/ranger/core/util/MetricRecorder.java +++ b/ranger-core/src/main/java/io/appform/ranger/core/util/MetricRecorder.java @@ -135,6 +135,13 @@ public static void recordServicesParseFailure(DataStoreType dataStoreType, Strin } } + public static void recordListNodesParseFailure(DataStoreType dataStoreType, String metricId) { + if (metricRegistry != null) { + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), + DATA_SOURCE, metricId, HTTP_CALL, LIST_NODES, RESPONSE_PARSE_FAILURE)).mark(); + } + } + public static void recordListNodesParseFailure(DataStoreType dataStoreType, String metricId, String serviceName) { if (metricRegistry != null) { metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), @@ -183,6 +190,13 @@ public static void recordNullOrEmptyServicesListResponse(DataStoreType dataStore } } + public static void recordNullOrEmptyListNodeResponse(DataStoreType dataStoreType, String metricId) { + if (metricRegistry != null) { + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), + DATA_SOURCE, metricId, HTTP_CALL, LIST_NODES, NULL_OR_EMPTY_RESPONSE)).mark(); + } + } + public static void recordNullOrEmptyListNodeResponse(DataStoreType dataStoreType, String metricId, String serviceName) { if (metricRegistry != null) { @@ -194,6 +208,8 @@ public static void recordNullOrEmptyListNodeResponse(DataStoreType dataStoreType public static void recordNodeDataSinkUnknownFailure(DataStoreType dataStoreType, String metricId, String serviceName, String exceptionName) { if (metricRegistry != null) { + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), + DATA_SOURCE, metricId, NODE_DATA_SINK, UNKNOWN_FAILURE)).mark(); metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), DATA_SOURCE, metricId, NODE_DATA_SINK, SERVICE_NAME, serviceName, UNKNOWN_FAILURE)).mark(); metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), diff --git a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleLocalHostPortTest.java b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleLocalHostPortTest.java index 7f625b9a..b2ad909f 100644 --- a/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleLocalHostPortTest.java +++ b/ranger-discovery-bundle/src/test/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundleLocalHostPortTest.java @@ -40,7 +40,6 @@ import java.util.UUID; import static io.appform.ranger.discovery.bundle.Constants.LOCAL_ADDRESSES; -import static io.appform.ranger.discovery.core.Constants.DEFAULT_DATA_SINK_ID; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; diff --git a/ranger-discovery-core/src/main/java/io/appform/ranger/discovery/core/Constants.java b/ranger-discovery-core/src/main/java/io/appform/ranger/discovery/core/Constants.java index 36da39b5..2a57a6b6 100644 --- a/ranger-discovery-core/src/main/java/io/appform/ranger/discovery/core/Constants.java +++ b/ranger-discovery-core/src/main/java/io/appform/ranger/discovery/core/Constants.java @@ -24,7 +24,7 @@ @UtilityClass public class Constants { public static final String DEFAULT_NAMESPACE = "default"; - public static final String DEFAULT_DATA_SINK_ID = "RANGER_DISCOVERY_ZOOKEEPER"; + public static final String DEFAULT_DATA_SINK_ID = "zkServiceDiscovery"; public static final String DEFAULT_HOST = "__DEFAULT_SERVICE_HOST"; public static final int DEFAULT_PORT = -1; diff --git a/ranger-drove/src/main/java/io/appform/ranger/drove/common/DroveApiCommunicator.java b/ranger-drove/src/main/java/io/appform/ranger/drove/common/DroveApiCommunicator.java index 11b5310e..96384a0e 100644 --- a/ranger-drove/src/main/java/io/appform/ranger/drove/common/DroveApiCommunicator.java +++ b/ranger-drove/src/main/java/io/appform/ranger/drove/common/DroveApiCommunicator.java @@ -195,12 +195,14 @@ private ApiResponse> parseListNodesResponse(Iterable MetricRecorder.recordNullOrEmptyListNodeResponse(DataStoreType.DROVE, metricId, service.getServiceName())); log.warn("Received empty services list from drove. Response body: {}", response.body()); } return apiResponse; } catch (JsonProcessingException e) { + MetricRecorder.recordListNodesParseFailure(DataStoreType.DROVE, metricId); services.forEach(service -> MetricRecorder.recordListNodesParseFailure(DataStoreType.DROVE, metricId, service.getServiceName())); log.error("Error parsing response from drove: {}. Response body: {}", e.getMessage(), response.body(), e); throw new DroveCommunicationException("Error parsing response from drove: " + e.getMessage()); diff --git a/ranger-drove/src/test/java/io/appform/ranger/drove/common/DroveApiCommunicatorMetricsIntegrationTest.java b/ranger-drove/src/test/java/io/appform/ranger/drove/common/DroveApiCommunicatorMetricsIntegrationTest.java index 3727a9cd..170c56f8 100644 --- a/ranger-drove/src/test/java/io/appform/ranger/drove/common/DroveApiCommunicatorMetricsIntegrationTest.java +++ b/ranger-drove/src/test/java/io/appform/ranger/drove/common/DroveApiCommunicatorMetricsIntegrationTest.java @@ -221,6 +221,11 @@ void testListNodes_emptyData_recordsNullOrEmptyListNodeResponse(WireMockRuntimeI assertTrue(nodes.isEmpty()); // Verify null/empty list node response + val aggregateMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.DROVE.dataSource.drove-api-7.httpCall.listNodes.nullOrEmptyResponse"); + assertNotNull(aggregateMeter, "Aggregate null/empty list node response meter should be recorded"); + assertEquals(1, aggregateMeter.getCount()); + val emptyMeter = metricRegistry.getMeters().get( METRIC_PREFIX + ".dataStoreType.DROVE.dataSource.drove-api-7.httpCall.listNodes.serviceName.TEST_APP.nullOrEmptyResponse"); assertNotNull(emptyMeter, "Null/empty list node response meter should be recorded"); @@ -242,6 +247,11 @@ void testListNodes_invalidJson_recordsParseFailure(WireMockRuntimeInfo wm) { assertThrows(DroveCommunicationException.class, () -> client.listNodes(service)); // Verify list nodes parse failure + val aggregateParseMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.DROVE.dataSource.drove-api-8.httpCall.listNodes.responseParseFailure"); + assertNotNull(aggregateParseMeter, "Aggregate list nodes parse failure meter should be recorded"); + assertEquals(1, aggregateParseMeter.getCount()); + val parseMeter = metricRegistry.getMeters().get( METRIC_PREFIX + ".dataStoreType.DROVE.dataSource.drove-api-8.httpCall.listNodes.serviceName.PARSE_FAIL_APP.responseParseFailure"); assertNotNull(parseMeter, "List nodes parse failure meter should be recorded"); @@ -270,6 +280,79 @@ void testListNodes_serverError_recordsStatusCode(WireMockRuntimeInfo wm) { } } + // ==================== listNodes() - Multi-service: aggregate pushed only once ==================== + + @Test + @SneakyThrows + void testListNodes_multiServiceEmptyResponse_aggregateMetricPushedOnce(WireMockRuntimeInfo wm) { + val nodeResponse = ApiResponse.success(List.of()); // empty list + + stubFor(get(urlPathEqualTo("/apis/v1/endpoints")) + .withBasicAuth("guest", "guest") + .willReturn(aResponse() + .withBody(MAPPER.writeValueAsBytes(nodeResponse)) + .withStatus(200))); + + try (val client = buildClient(wm, "drove-api-10")) { + val service1 = new Service("testns", "APP_ONE"); + val service2 = new Service("testns", "APP_TWO"); + val result = client.listNodes(List.of(service1, service2)); + + assertNotNull(result); + assertTrue(result.isEmpty()); + + // Aggregate metric must be pushed exactly once regardless of how many services were in the batch + val aggregateMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.DROVE.dataSource.drove-api-10.httpCall.listNodes.nullOrEmptyResponse"); + assertNotNull(aggregateMeter, "Aggregate null/empty metric should be recorded"); + assertEquals(1, aggregateMeter.getCount(), + "Aggregate metric must be pushed exactly once even for multi-service batches"); + + // Service-level metrics pushed once per service + val svc1Meter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.DROVE.dataSource.drove-api-10.httpCall.listNodes.serviceName.APP_ONE.nullOrEmptyResponse"); + assertNotNull(svc1Meter, "Service-level metric for APP_ONE should be recorded"); + assertEquals(1, svc1Meter.getCount()); + + val svc2Meter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.DROVE.dataSource.drove-api-10.httpCall.listNodes.serviceName.APP_TWO.nullOrEmptyResponse"); + assertNotNull(svc2Meter, "Service-level metric for APP_TWO should be recorded"); + assertEquals(1, svc2Meter.getCount()); + } + } + + @Test + @SneakyThrows + void testListNodes_multiServiceParseFailure_aggregateMetricPushedOnce(WireMockRuntimeInfo wm) { + stubFor(get(urlPathEqualTo("/apis/v1/endpoints")) + .withBasicAuth("guest", "guest") + .willReturn(aResponse().withStatus(200).withBody("invalid-json{{{{"))); + + try (val client = buildClient(wm, "drove-api-11")) { + val service1 = new Service("testns", "APP_ONE"); + val service2 = new Service("testns", "APP_TWO"); + assertThrows(DroveCommunicationException.class, () -> client.listNodes(List.of(service1, service2))); + + // Aggregate metric must be pushed exactly once + val aggregateParseMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.DROVE.dataSource.drove-api-11.httpCall.listNodes.responseParseFailure"); + assertNotNull(aggregateParseMeter, "Aggregate parse failure metric should be recorded"); + assertEquals(1, aggregateParseMeter.getCount(), + "Aggregate metric must be pushed exactly once even for multi-service batches"); + + // Service-level metrics pushed once per service + val svc1Meter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.DROVE.dataSource.drove-api-11.httpCall.listNodes.serviceName.APP_ONE.responseParseFailure"); + assertNotNull(svc1Meter, "Service-level parse failure metric for APP_ONE should be recorded"); + assertEquals(1, svc1Meter.getCount()); + + val svc2Meter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.DROVE.dataSource.drove-api-11.httpCall.listNodes.serviceName.APP_TWO.responseParseFailure"); + assertNotNull(svc2Meter, "Service-level parse failure metric for APP_TWO should be recorded"); + assertEquals(1, svc2Meter.getCount()); + } + } + // ==================== Helper ==================== private DroveCommunicator buildClient(WireMockRuntimeInfo wm, String metricId) { diff --git a/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpApiCommunicator.java b/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpApiCommunicator.java index f6ab5b1c..557c0bd1 100644 --- a/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpApiCommunicator.java +++ b/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpApiCommunicator.java @@ -202,6 +202,7 @@ private List> parseNodeList( try (val body = response.body()) { if (null == body) { log.warn("HTTP call to {} returned empty body", httpUrl); + MetricRecorder.recordNullOrEmptyListNodeResponse(DataStoreType.HTTP, metricId); MetricRecorder.recordNullOrEmptyListNodeResponse(DataStoreType.HTTP, metricId, service.getServiceName()); throw new HttpCommunicationException("Empty response received for call to " + httpUrl); } @@ -210,6 +211,7 @@ private List> parseNodeList( val serviceNodesResponse = deserializer.deserialize(bytes); if(serviceNodesResponse == null || serviceNodesResponse.getData() == null || serviceNodesResponse.getData().isEmpty()) { + MetricRecorder.recordNullOrEmptyListNodeResponse(DataStoreType.HTTP, metricId); MetricRecorder.recordNullOrEmptyListNodeResponse(DataStoreType.HTTP, metricId, service.getServiceName()); log.warn("Received empty node list from http. Response body: {}", response.body()); } @@ -225,6 +227,7 @@ private List> parseNodeList( } catch (HttpCommunicationException httpCommunicationException){ throw httpCommunicationException; } catch (Exception e) { + MetricRecorder.recordListNodesParseFailure(DataStoreType.HTTP, metricId); MetricRecorder.recordListNodesParseFailure(DataStoreType.HTTP, metricId, service.getServiceName()); throw new HttpCommunicationException( "Error parsing node data from server. Url: " + httpUrl + "Error: " + e.getMessage()); diff --git a/ranger-http/src/test/java/io/appform/ranger/http/servicefinder/HttpApiCommunicatorMetricsIntegrationTest.java b/ranger-http/src/test/java/io/appform/ranger/http/servicefinder/HttpApiCommunicatorMetricsIntegrationTest.java index 0c23997f..f492e6e0 100644 --- a/ranger-http/src/test/java/io/appform/ranger/http/servicefinder/HttpApiCommunicatorMetricsIntegrationTest.java +++ b/ranger-http/src/test/java/io/appform/ranger/http/servicefinder/HttpApiCommunicatorMetricsIntegrationTest.java @@ -312,6 +312,11 @@ void testListNodes_invalidJson_recordsParseFailure(WireMockRuntimeInfo wmInfo) { assertThrows(Exception.class, () -> communicator.listNodes(service, deserializer)); // The listNodes parse failure meter should exist + val aggregateParseMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.HTTP.dataSource.http-metric-9.httpCall.listNodes.responseParseFailure"); + assertNotNull(aggregateParseMeter, "Aggregate list nodes parse failure meter should be recorded"); + assertEquals(1, aggregateParseMeter.getCount()); + val parseMeter = metricRegistry.getMeters().get( METRIC_PREFIX + ".dataStoreType.HTTP.dataSource.http-metric-9.httpCall.listNodes.serviceName.parse-fail-svc.responseParseFailure"); assertNotNull(parseMeter, "List nodes parse failure meter should be recorded"); diff --git a/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinder/ZkNodeDataSource.java b/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinder/ZkNodeDataSource.java index 339219eb..41bdbb42 100644 --- a/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinder/ZkNodeDataSource.java +++ b/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinder/ZkNodeDataSource.java @@ -90,6 +90,7 @@ private Optional>> checkForUpdateOnZookeeper(D deserializer) List> nodes = new ArrayList<>(children.size()); log.debug("Found {} nodes for [{}]", children.size(), serviceName); if(children.isEmpty()){ + MetricRecorder.recordNullOrEmptyListNodeResponse(DataStoreType.ZK, metricId); MetricRecorder.recordNullOrEmptyListNodeResponse(DataStoreType.ZK, metricId, serviceName); } for (val child : children) { @@ -103,6 +104,7 @@ private Optional>> checkForUpdateOnZookeeper(D deserializer) return Optional.of(nodes); } catch (NoNodeException e) { + MetricRecorder.recordNullOrEmptyListNodeResponse(DataStoreType.ZK, metricId); MetricRecorder.recordNullOrEmptyListNodeResponse(DataStoreType.ZK, metricId, serviceName); log.error( "No ZK container node found for service: {}. Will return empty list for now. Please doublecheck service name", @@ -121,6 +123,7 @@ private > ServiceNode parseServiceNode try { return deserializer.deserialize(data); } catch (Exception e) { + MetricRecorder.recordListNodesParseFailure(DataStoreType.ZK, metricId); MetricRecorder.recordListNodesParseFailure(DataStoreType.ZK, metricId, serviceName); log.error("Error deserializing node data : {} for service name: {} ", new String(data), serviceName, e); throw e; @@ -133,11 +136,13 @@ private Optional readChild(String serviceName, String parentPath, String return Optional.ofNullable(curatorFramework.getData().forPath(path)); } catch (KeeperException.NoNodeException e) { + MetricRecorder.recordNullOrEmptyListNodeResponse(DataStoreType.ZK, metricId); MetricRecorder.recordNullOrEmptyListNodeResponse(DataStoreType.ZK, metricId, serviceName); log.warn("Node not found for path {}", path); return Optional.empty(); } catch (KeeperException e) { + MetricRecorder.recordNullOrEmptyListNodeResponse(DataStoreType.ZK, metricId); MetricRecorder.recordNullOrEmptyListNodeResponse(DataStoreType.ZK, metricId, serviceName); log.error("Could not get data for node: {}", path, e); return Optional.empty(); diff --git a/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/servicefinder/ZkNodeDataSourceMetricsIntegrationTest.java b/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/servicefinder/ZkNodeDataSourceMetricsIntegrationTest.java index 44afa554..e976cafc 100644 --- a/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/servicefinder/ZkNodeDataSourceMetricsIntegrationTest.java +++ b/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/servicefinder/ZkNodeDataSourceMetricsIntegrationTest.java @@ -186,6 +186,11 @@ void testRefresh_emptyChildren_recordsNullOrEmptyListNodeResponse() throws Excep assertTrue(result.get().isEmpty()); // Verify null/empty list node response metric + val aggregateMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.ZK.dataSource.zk-node-src-4.httpCall.listNodes.nullOrEmptyResponse"); + assertNotNull(aggregateMeter, "Aggregate empty list node meter should be recorded"); + assertEquals(1, aggregateMeter.getCount()); + val emptyMeter = metricRegistry.getMeters().get( METRIC_PREFIX + ".dataStoreType.ZK.dataSource.zk-node-src-4.httpCall.listNodes.serviceName." + SERVICE_NAME + ".nullOrEmptyResponse"); @@ -212,6 +217,11 @@ void testRefresh_noServicePath_recordsNullOrEmptyListNodeResponse() throws Excep assertTrue(result.get().isEmpty()); // Verify null/empty list node response metric (NoNodeException path) + val aggregateMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.ZK.dataSource.zk-node-src-5.httpCall.listNodes.nullOrEmptyResponse"); + assertNotNull(aggregateMeter, "Aggregate empty list node meter should be recorded for NoNodeException"); + assertTrue(aggregateMeter.getCount() >= 1); + val emptyMeter = metricRegistry.getMeters().get( METRIC_PREFIX + ".dataStoreType.ZK.dataSource.zk-node-src-5.httpCall.listNodes.serviceName.nonexistent-svc.nullOrEmptyResponse"); assertNotNull(emptyMeter, "Empty list node meter should be recorded for NoNodeException"); @@ -243,6 +253,11 @@ void testRefresh_deserializerThrows_recordsListNodesParseFailure() throws Except assertThrows(RuntimeException.class, () -> dataSource.refresh(badDeserializer)); // Verify list nodes parse failure metric + val aggregateParseMeter = metricRegistry.getMeters().get( + METRIC_PREFIX + ".dataStoreType.ZK.dataSource.zk-node-src-6.httpCall.listNodes.responseParseFailure"); + assertNotNull(aggregateParseMeter, "Aggregate list nodes parse failure meter should be recorded"); + assertEquals(1, aggregateParseMeter.getCount()); + val parseMeter = metricRegistry.getMeters().get( METRIC_PREFIX + ".dataStoreType.ZK.dataSource.zk-node-src-6.httpCall.listNodes.serviceName." + SERVICE_NAME + ".responseParseFailure"); From 7b6941bb3f741f45ead0fa4d91c2ee9311c2df69 Mon Sep 17 00:00:00 2001 From: Jitendra Dhawan Date: Tue, 2 Jun 2026 12:01:49 +0530 Subject: [PATCH 19/24] Enhance metrics recording for service nodes and services returned; update log messages for clarity --- .../ranger/core/util/MetricRecorder.java | 18 ++++++++- .../drove/common/DroveApiCommunicator.java | 2 +- .../servicefinder/HttpApiCommunicator.java | 1 - ...r.id.IdGeneratorPerfTest.testGenerate.json | 2 +- ...dGeneratorPerfTest.testGenerateBase36.json | 2 +- .../bundle/resources/RangerResource.java | 37 +++++++++++-------- 6 files changed, 41 insertions(+), 21 deletions(-) diff --git a/ranger-core/src/main/java/io/appform/ranger/core/util/MetricRecorder.java b/ranger-core/src/main/java/io/appform/ranger/core/util/MetricRecorder.java index b0bfb4ae..4e9c7115 100644 --- a/ranger-core/src/main/java/io/appform/ranger/core/util/MetricRecorder.java +++ b/ranger-core/src/main/java/io/appform/ranger/core/util/MetricRecorder.java @@ -235,7 +235,23 @@ public static void recordNullOrEmptyRegisterServiceResponse(DataStoreType dataSt metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), DATA_SOURCE, metricId, HTTP_CALL, REGISTER_SERVICE, NULL_OR_EMPTY_RESPONSE)).mark(); metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), - DATA_SOURCE, metricId, HTTP_CALL, REGISTER_SERVICE, SERVICE_NAME, serviceName, NULL_OR_EMPTY_RESPONSE)).mark(); + DATA_SOURCE, metricId, HTTP_CALL, REGISTER_SERVICE, SERVICE_NAME, serviceName, NULL_OR_EMPTY_RESPONSE)) + .mark(); } } + + public static void recordServiceNodesReturned(String serviceName, int serviceNodes) { + if (metricRegistry != null) { + metricRegistry.histogram(MetricRegistry.name(PACKAGE_PREFIX, SERVICE_NAME, + serviceName, "nodesReturned")).update(serviceNodes); + } + } + + public static void recordServicesReturned(int services) { + if (metricRegistry != null) { + metricRegistry.histogram(MetricRegistry.name(PACKAGE_PREFIX,"servicesReturned")) + .update(services); + } + } + } diff --git a/ranger-drove/src/main/java/io/appform/ranger/drove/common/DroveApiCommunicator.java b/ranger-drove/src/main/java/io/appform/ranger/drove/common/DroveApiCommunicator.java index 96384a0e..46b87064 100644 --- a/ranger-drove/src/main/java/io/appform/ranger/drove/common/DroveApiCommunicator.java +++ b/ranger-drove/src/main/java/io/appform/ranger/drove/common/DroveApiCommunicator.java @@ -197,7 +197,7 @@ private ApiResponse> parseListNodesResponse(Iterable MetricRecorder.recordNullOrEmptyListNodeResponse(DataStoreType.DROVE, metricId, service.getServiceName())); - log.warn("Received empty services list from drove. Response body: {}", response.body()); + log.warn("Received empty node list from drove. Response body: {}", response.body()); } return apiResponse; diff --git a/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpApiCommunicator.java b/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpApiCommunicator.java index 557c0bd1..39833665 100644 --- a/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpApiCommunicator.java +++ b/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpApiCommunicator.java @@ -213,7 +213,6 @@ private List> parseNodeList( if(serviceNodesResponse == null || serviceNodesResponse.getData() == null || serviceNodesResponse.getData().isEmpty()) { MetricRecorder.recordNullOrEmptyListNodeResponse(DataStoreType.HTTP, metricId); MetricRecorder.recordNullOrEmptyListNodeResponse(DataStoreType.HTTP, metricId, service.getServiceName()); - log.warn("Received empty node list from http. Response body: {}", response.body()); } if (serviceNodesResponse.valid()) { diff --git a/ranger-id/perf/results/io.appform.ranger.id.IdGeneratorPerfTest.testGenerate.json b/ranger-id/perf/results/io.appform.ranger.id.IdGeneratorPerfTest.testGenerate.json index 5be9813c..5f29a5c5 100644 --- a/ranger-id/perf/results/io.appform.ranger.id.IdGeneratorPerfTest.testGenerate.json +++ b/ranger-id/perf/results/io.appform.ranger.id.IdGeneratorPerfTest.testGenerate.json @@ -4,5 +4,5 @@ "iterations" : 4, "threads" : 1, "forks" : 3, - "mean_ops" : 678336.7125441622 + "mean_ops" : 650814.0256598704 } \ No newline at end of file diff --git a/ranger-id/perf/results/io.appform.ranger.id.IdGeneratorPerfTest.testGenerateBase36.json b/ranger-id/perf/results/io.appform.ranger.id.IdGeneratorPerfTest.testGenerateBase36.json index 08d82286..75ec8538 100644 --- a/ranger-id/perf/results/io.appform.ranger.id.IdGeneratorPerfTest.testGenerateBase36.json +++ b/ranger-id/perf/results/io.appform.ranger.id.IdGeneratorPerfTest.testGenerateBase36.json @@ -4,5 +4,5 @@ "iterations" : 4, "threads" : 1, "forks" : 3, - "mean_ops" : 535260.4479626919 + "mean_ops" : 516175.3456572488 } \ No newline at end of file diff --git a/ranger-server-bundle/src/main/java/io/appform/ranger/server/bundle/resources/RangerResource.java b/ranger-server-bundle/src/main/java/io/appform/ranger/server/bundle/resources/RangerResource.java index d30e72da..fbc1ebb2 100644 --- a/ranger-server-bundle/src/main/java/io/appform/ranger/server/bundle/resources/RangerResource.java +++ b/ranger-server-bundle/src/main/java/io/appform/ranger/server/bundle/resources/RangerResource.java @@ -20,6 +20,7 @@ import io.appform.ranger.core.model.Service; import io.appform.ranger.core.model.ServiceNode; import io.appform.ranger.core.model.ServiceRegistry; +import io.appform.ranger.core.util.MetricRecorder; import io.appform.ranger.http.response.model.GenericResponse; import lombok.extern.slf4j.Slf4j; import lombok.val; @@ -53,12 +54,14 @@ public RangerResource(List> rangerHubs) { @Timed public GenericResponse> getServices( @QueryParam("skipDataFromReplicationSources") @DefaultValue("false") boolean skipDataFromReplicationSources) { + val services = rangerHubs.stream() + .filter(hub -> !skipDataFromReplicationSources || !hub.isReplicationSource()) + .map(RangerHubClient::getRegisteredServices) + .flatMap(Collection::stream) + .collect(Collectors.toSet()); + MetricRecorder.recordServicesReturned(services.size()); return GenericResponse.>builder() - .data(rangerHubs.stream() - .filter(hub -> !skipDataFromReplicationSources || !hub.isReplicationSource()) - .map(RangerHubClient::getRegisteredServices) - .flatMap(Collection::stream) - .collect(Collectors.toSet())) + .data(services) .build(); } @@ -70,18 +73,20 @@ public GenericResponse>> getNodes( @NotNull @NotEmpty @PathParam("serviceName") final String serviceName, @QueryParam("skipDataFromReplicationSources") @DefaultValue("false") boolean skipDataFromReplicationSources) { val service = Service.builder().namespace(namespace).serviceName(serviceName).build(); + val serviceNodes = rangerHubs.stream() + .filter(hub -> !(skipDataFromReplicationSources && hub.isReplicationSource())) + .map(hub -> hub.getAllNodes(service)) + .flatMap(List::stream) + .collect(Collectors.toMap(node -> node.getHost() + ":" + node.getPort(), + Function.identity(), + (oldV, newV) -> + oldV.getLastUpdatedTimeStamp() > newV.getLastUpdatedTimeStamp() + ? oldV + : newV)) + .values(); + MetricRecorder.recordServiceNodesReturned(service.getServiceName(), serviceNodes.size()); return GenericResponse.>>builder() - .data(rangerHubs.stream() - .filter(hub -> !(skipDataFromReplicationSources && hub.isReplicationSource())) - .map(hub -> hub.getAllNodes(service)) - .flatMap(List::stream) - .collect(Collectors.toMap(node -> node.getHost() + ":" + node.getPort(), - Function.identity(), - (oldV, newV) -> - oldV.getLastUpdatedTimeStamp() > newV.getLastUpdatedTimeStamp() - ? oldV - : newV)) - .values()) + .data(serviceNodes) .build(); } } From ac4485f774299e6de8b42a227917cf580ca1fcd8 Mon Sep 17 00:00:00 2001 From: Jitendra Dhawan Date: Tue, 2 Jun 2026 19:30:24 +0530 Subject: [PATCH 20/24] Refactor metrics recording to include DataStoreType in metric names for improved clarity --- .../ranger/core/util/MetricRecorder.java | 32 +++++++++---------- ...RegistryUpdaterMetricsIntegrationTest.java | 12 +++---- .../HealthCheckerMetricsIntegrationTest.java | 6 ++-- ...eNodeDataSourceMetricsIntegrationTest.java | 4 +-- ...rviceDataSourceMetricsIntegrationTest.java | 4 +-- .../common/HttpNodeDataStoreConnector.java | 2 +- ...rviceDataSourceMetricsIntegrationTest.java | 2 +- ...kNodeDataSourceMetricsIntegrationTest.java | 4 +-- .../ZkNodeDataSinkMetricsIntegrationTest.java | 12 +++---- 9 files changed, 39 insertions(+), 39 deletions(-) diff --git a/ranger-core/src/main/java/io/appform/ranger/core/util/MetricRecorder.java b/ranger-core/src/main/java/io/appform/ranger/core/util/MetricRecorder.java index 4e9c7115..9562651d 100644 --- a/ranger-core/src/main/java/io/appform/ranger/core/util/MetricRecorder.java +++ b/ranger-core/src/main/java/io/appform/ranger/core/util/MetricRecorder.java @@ -10,7 +10,7 @@ public class MetricRecorder { private static final String PACKAGE_PREFIX = "io.appform.ranger"; - private static final String ACTIVE = "active"; + public static final String ACTIVE = "active"; private static final String INACTIVE = "inactive"; private static final String NULL_OR_EMPTY_RESPONSE = "nullOrEmptyResponse"; private static final String DATA_STORE_TYPE = "dataStoreType"; @@ -52,55 +52,55 @@ public static void recordZombieNodeFound(String serviceName) { public static void recordNoteDataSourceStatus(DataStoreType dataStoreType, String metricId, boolean active) { if (metricRegistry != null) { - metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_SOURCE, dataStoreType.name(), - metricId, active ? ACTIVE : INACTIVE)).mark(); + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), + DATA_SOURCE, metricId, active ? ACTIVE : INACTIVE)).mark(); } } public static void recordNodeDataRefreshSuccess(DataStoreType dataStoreType, String metricId, long elapsed) { if (metricRegistry != null) { - metricRegistry.timer(MetricRegistry.name(PACKAGE_PREFIX, DATA_SOURCE, dataStoreType.name(), - metricId, NODE_DATA_REFRESH, SUCCESS)).update(elapsed, MILLISECONDS); + metricRegistry.timer(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), + DATA_SOURCE, metricId, NODE_DATA_REFRESH, SUCCESS)).update(elapsed, MILLISECONDS); } } public static void recordNodeDataRefreshFailure(String serviceName, DataStoreType dataStoreType, String metricId, long elapsed) { if (metricRegistry != null) { - metricRegistry.timer(MetricRegistry.name(PACKAGE_PREFIX, DATA_SOURCE, dataStoreType.name(), - metricId, NODE_DATA_REFRESH, FAILURE)).update(elapsed, MILLISECONDS); - metricRegistry.timer(MetricRegistry.name(PACKAGE_PREFIX, DATA_SOURCE, dataStoreType.name(), - metricId, SERVICE_NAME, serviceName, NODE_DATA_REFRESH, FAILURE)) + metricRegistry.timer(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), + DATA_SOURCE, metricId, NODE_DATA_REFRESH, FAILURE)).update(elapsed, MILLISECONDS); + metricRegistry.timer(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), + DATA_SOURCE, metricId, SERVICE_NAME, serviceName, NODE_DATA_REFRESH, FAILURE)) .update(elapsed, MILLISECONDS); } } public static void recordStaleDataRetained(String serviceName, DataStoreType dataStoreType, String metricId) { if (metricRegistry != null) { - metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_SOURCE, dataStoreType.name(), - metricId, STALE_DATA_RETAINED)).mark(); - metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_SOURCE, dataStoreType.name(), - metricId, SERVICE_NAME, serviceName, STALE_DATA_RETAINED)).mark(); + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), + DATA_SOURCE, metricId, STALE_DATA_RETAINED)).mark(); + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), + DATA_SOURCE, metricId, SERVICE_NAME, serviceName, STALE_DATA_RETAINED)).mark(); } } public static void recordNodeDataSinkUpdateStatus(DataStoreType dataStoreType, String metricId, String status) { if (metricRegistry != null) { - metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_SOURCE, dataStoreType.name(), + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), DATA_SOURCE, metricId, NODE_DATA_SINK, UPDATE, status)).mark(); } } public static void recordHealthcheckFailure(DataStoreType dataStoreType, String metricId) { if (metricRegistry != null) { - metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_SOURCE, dataStoreType.name(), + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), DATA_SOURCE, metricId, HEALTHCHECK, FAILURE)).mark(); } } public static void recordHealthcheckStatus(DataStoreType dataStoreType, String metricId, boolean healthy) { if (metricRegistry != null) { - metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_SOURCE, dataStoreType.name(), + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), DATA_SOURCE, metricId, HEALTHCHECK, "status", healthy ? HEALTHY : UNHEALTHY)).mark(); } } diff --git a/ranger-core/src/test/java/io/appform/ranger/core/finder/serviceregistry/ServiceRegistryUpdaterMetricsIntegrationTest.java b/ranger-core/src/test/java/io/appform/ranger/core/finder/serviceregistry/ServiceRegistryUpdaterMetricsIntegrationTest.java index 53829555..b3e886ac 100644 --- a/ranger-core/src/test/java/io/appform/ranger/core/finder/serviceregistry/ServiceRegistryUpdaterMetricsIntegrationTest.java +++ b/ranger-core/src/test/java/io/appform/ranger/core/finder/serviceregistry/ServiceRegistryUpdaterMetricsIntegrationTest.java @@ -84,13 +84,13 @@ void testSuccessfulRefresh_recordsSuccessTimer() { sleep(100); // Allow time for MetricRecorder call after updateNodes // Verify success timer - val timerName = "io.appform.ranger.dataSource.ZK." + METRIC_ID + ".nodeDataRefresh.success"; + val timerName = "io.appform.ranger.dataStoreType.ZK.dataSource." + METRIC_ID + ".nodeDataRefresh.success"; val timer = metricRegistry.getTimers().get(timerName); assertNotNull(timer, "Node data refresh success timer should exist"); assertTrue(timer.getCount() >= 1, "Timer should have at least 1 update"); // No failure timer - val failureTimerName = "io.appform.ranger.dataSource.ZK." + METRIC_ID + ".nodeDataRefresh.failure"; + val failureTimerName = "io.appform.ranger.dataStoreType.ZK.dataSource." + METRIC_ID + ".nodeDataRefresh.failure"; val failureTimer = metricRegistry.getTimers().get(failureTimerName); assertTrue(failureTimer == null || failureTimer.getCount() == 0, "Failure timer should not be recorded on success"); @@ -114,7 +114,7 @@ void testRefreshThrowsException_recordsFailureTimer() { sleep(200); // Verify failure timer - val failureTimerName = "io.appform.ranger.dataSource.HTTP." + METRIC_ID + ".nodeDataRefresh.failure"; + val failureTimerName = "io.appform.ranger.dataStoreType.HTTP.dataSource." + METRIC_ID + ".nodeDataRefresh.failure"; val failureTimer = metricRegistry.getTimers().get(failureTimerName); assertNotNull(failureTimer, "Node data refresh failure timer should exist"); assertTrue(failureTimer.getCount() >= 1, "Failure timer should have at least 1 update"); @@ -146,7 +146,7 @@ void testInactiveDataSource_recordsStaleDataRetained() { sleep(200); // Verify stale data retained meter - val meterName = "io.appform.ranger.dataSource.DROVE." + METRIC_ID + ".staleDataRetained"; + val meterName = "io.appform.ranger.dataStoreType.DROVE.dataSource." + METRIC_ID + ".staleDataRetained"; val meter = metricRegistry.getMeters().get(meterName); assertNotNull(meter, "Stale data retained meter should exist"); assertTrue(meter.getCount() >= 1, "Stale data retained should be recorded at least once"); @@ -188,7 +188,7 @@ void testSuccessfulRefresh_zombieNodesFiltered_recordsZombieMetric() { assertTrue(svcZombieMeter.getCount() >= 1); // Also verify success timer still recorded - val timerName = "io.appform.ranger.dataSource.ZK." + METRIC_ID + ".nodeDataRefresh.success"; + val timerName = "io.appform.ranger.dataStoreType.ZK.dataSource." + METRIC_ID + ".nodeDataRefresh.success"; val timer = metricRegistry.getTimers().get(timerName); assertNotNull(timer); assertTrue(timer.getCount() >= 1); @@ -238,7 +238,7 @@ void testRefreshReturnsNull_noSuccessOrFailureTimer_butStaleRetained() { // When refresh returns null, no success timer should be incremented for that second call // The success timer from the first call should be 1 - val timerName = "io.appform.ranger.dataSource.ZK." + METRIC_ID + ".nodeDataRefresh.success"; + val timerName = "io.appform.ranger.dataStoreType.ZK.dataSource." + METRIC_ID + ".nodeDataRefresh.success"; val beforeTimer = metricRegistry.getTimers().get(timerName); final long beforeCount = beforeTimer == null ? 0L : beforeTimer.getCount(); diff --git a/ranger-core/src/test/java/io/appform/ranger/core/healthcheck/HealthCheckerMetricsIntegrationTest.java b/ranger-core/src/test/java/io/appform/ranger/core/healthcheck/HealthCheckerMetricsIntegrationTest.java index 691093ad..531b2d22 100644 --- a/ranger-core/src/test/java/io/appform/ranger/core/healthcheck/HealthCheckerMetricsIntegrationTest.java +++ b/ranger-core/src/test/java/io/appform/ranger/core/healthcheck/HealthCheckerMetricsIntegrationTest.java @@ -32,9 +32,9 @@ class HealthCheckerMetricsIntegrationTest { private static final String METRIC_ID = "test-hc-metric"; // Metric keys produced by MetricRecorder for DataStoreType.ZK and METRIC_ID: - // recordHealthcheckStatus -> io.appform.ranger.dataSource.ZK.dataSource..healthcheck.status. - // recordHealthcheckFailure -> io.appform.ranger.dataSource.ZK.dataSource..healthcheck.failure - private static final String HC_PREFIX = "io.appform.ranger.dataSource.ZK.dataSource." + METRIC_ID + ".healthcheck"; + // recordHealthcheckStatus -> io.appform.ranger.dataStoreType.ZK.dataSource..healthcheck.status. + // recordHealthcheckFailure -> io.appform.ranger.dataStoreType.ZK.dataSource..healthcheck.failure + private static final String HC_PREFIX = "io.appform.ranger.dataStoreType.ZK.dataSource." + METRIC_ID + ".healthcheck"; private static final String HEALTHY_KEY = HC_PREFIX + ".status.healthy"; private static final String UNHEALTHY_KEY = HC_PREFIX + ".status.unhealthy"; private static final String FAILURE_KEY = HC_PREFIX + ".failure"; diff --git a/ranger-drove/src/test/java/io/appform/ranger/drove/servicefinder/DroveNodeDataSourceMetricsIntegrationTest.java b/ranger-drove/src/test/java/io/appform/ranger/drove/servicefinder/DroveNodeDataSourceMetricsIntegrationTest.java index 5ddae2e4..eaddeb3b 100644 --- a/ranger-drove/src/test/java/io/appform/ranger/drove/servicefinder/DroveNodeDataSourceMetricsIntegrationTest.java +++ b/ranger-drove/src/test/java/io/appform/ranger/drove/servicefinder/DroveNodeDataSourceMetricsIntegrationTest.java @@ -81,7 +81,7 @@ void testIsActive_healthyUpstream_recordsActiveStatus(WireMockRuntimeInfo wm) { assertTrue(active); val activeMeter = metricRegistry.getMeters().get( - METRIC_PREFIX + ".dataSource.DROVE.drove-node-src-1.active"); + METRIC_PREFIX + ".dataStoreType.DROVE.dataSource.drove-node-src-1.active"); assertNotNull(activeMeter, "Active status meter should be recorded"); assertEquals(1, activeMeter.getCount()); } @@ -114,7 +114,7 @@ void testIsActive_unhealthyUpstream_recordsInactiveStatus(WireMockRuntimeInfo wm assertFalse(active); val inactiveMeter = metricRegistry.getMeters().get( - METRIC_PREFIX + ".dataSource.DROVE.drove-node-src-2.inactive"); + METRIC_PREFIX + ".dataStoreType.DROVE.dataSource.drove-node-src-2.inactive"); assertNotNull(inactiveMeter, "Inactive status meter should be recorded"); assertEquals(1, inactiveMeter.getCount()); } diff --git a/ranger-drove/src/test/java/io/appform/ranger/drove/servicefinderhub/DroveServiceDataSourceMetricsIntegrationTest.java b/ranger-drove/src/test/java/io/appform/ranger/drove/servicefinderhub/DroveServiceDataSourceMetricsIntegrationTest.java index c2f22754..b9c4d55b 100644 --- a/ranger-drove/src/test/java/io/appform/ranger/drove/servicefinderhub/DroveServiceDataSourceMetricsIntegrationTest.java +++ b/ranger-drove/src/test/java/io/appform/ranger/drove/servicefinderhub/DroveServiceDataSourceMetricsIntegrationTest.java @@ -139,7 +139,7 @@ void testIsActive_healthy_recordsActiveStatus(WireMockRuntimeInfo wm) { // Verify active status val activeMeter = metricRegistry.getMeters().get( - METRIC_PREFIX + ".dataSource.DROVE.drove-svc-src-3.active"); + METRIC_PREFIX + ".dataStoreType.DROVE.dataSource.drove-svc-src-3.active"); assertNotNull(activeMeter, "Active status meter should be recorded"); assertEquals(1, activeMeter.getCount()); } @@ -168,7 +168,7 @@ void testIsActive_afterFailedCall_recordsInactiveStatus(WireMockRuntimeInfo wm) // Verify inactive status val inactiveMeter = metricRegistry.getMeters().get( - METRIC_PREFIX + ".dataSource.DROVE.drove-svc-src-4.inactive"); + METRIC_PREFIX + ".dataStoreType.DROVE.dataSource.drove-svc-src-4.inactive"); assertNotNull(inactiveMeter, "Inactive status meter should be recorded"); assertEquals(1, inactiveMeter.getCount()); } diff --git a/ranger-http/src/main/java/io/appform/ranger/http/common/HttpNodeDataStoreConnector.java b/ranger-http/src/main/java/io/appform/ranger/http/common/HttpNodeDataStoreConnector.java index 1f945c93..4244ca05 100644 --- a/ranger-http/src/main/java/io/appform/ranger/http/common/HttpNodeDataStoreConnector.java +++ b/ranger-http/src/main/java/io/appform/ranger/http/common/HttpNodeDataStoreConnector.java @@ -62,7 +62,7 @@ protected int defaultPort() { @Override public boolean isActive() { - MetricRecorder.recordNoteDataSourceStatus(DataStoreType.HTTP, config.getId(), true); + MetricRecorder.recordNoteDataSourceStatus(DataStoreType.HTTP, config.getId(),true); return true; } } diff --git a/ranger-http/src/test/java/io/appform/ranger/http/servicefinderhub/HttpServiceDataSourceMetricsIntegrationTest.java b/ranger-http/src/test/java/io/appform/ranger/http/servicefinderhub/HttpServiceDataSourceMetricsIntegrationTest.java index 7b020184..b6721bdc 100644 --- a/ranger-http/src/test/java/io/appform/ranger/http/servicefinderhub/HttpServiceDataSourceMetricsIntegrationTest.java +++ b/ranger-http/src/test/java/io/appform/ranger/http/servicefinderhub/HttpServiceDataSourceMetricsIntegrationTest.java @@ -130,7 +130,7 @@ void testIsActive_recordsDataSourceStatus(WireMockRuntimeInfo wmInfo) { // HttpNodeDataStoreConnector.isActive() records status with config.getId() val statusMeter = metricRegistry.getMeters().get( - METRIC_PREFIX + ".dataSource.HTTP.sds-metric-4.active"); + METRIC_PREFIX + ".dataStoreType.HTTP.dataSource.sds-metric-4.active"); assertNotNull(statusMeter, "Active status meter should be recorded"); assertEquals(1, statusMeter.getCount()); } diff --git a/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/servicefinder/ZkNodeDataSourceMetricsIntegrationTest.java b/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/servicefinder/ZkNodeDataSourceMetricsIntegrationTest.java index e976cafc..fa0defd4 100644 --- a/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/servicefinder/ZkNodeDataSourceMetricsIntegrationTest.java +++ b/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/servicefinder/ZkNodeDataSourceMetricsIntegrationTest.java @@ -92,7 +92,7 @@ void testIsActive_connectedZk_recordsActiveStatus() { // Verify active status metric val activeMeter = metricRegistry.getMeters().get( - METRIC_PREFIX + ".dataSource.ZK.zk-node-src-1.active"); + METRIC_PREFIX + ".dataStoreType.ZK.dataSource.zk-node-src-1.active"); assertNotNull(activeMeter, "Active status meter should be recorded"); assertEquals(1, activeMeter.getCount()); @@ -124,7 +124,7 @@ void testIsActive_disconnectedZk_recordsInactiveStatus() throws Exception { // Verify inactive status metric val inactiveMeter = metricRegistry.getMeters().get( - METRIC_PREFIX + ".dataSource.ZK.zk-node-src-2.inactive"); + METRIC_PREFIX + ".dataStoreType.ZK.dataSource.zk-node-src-2.inactive"); assertNotNull(inactiveMeter, "Inactive status meter should be recorded"); assertEquals(1, inactiveMeter.getCount()); diff --git a/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/serviceprovider/ZkNodeDataSinkMetricsIntegrationTest.java b/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/serviceprovider/ZkNodeDataSinkMetricsIntegrationTest.java index 48a36263..4593d5c7 100644 --- a/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/serviceprovider/ZkNodeDataSinkMetricsIntegrationTest.java +++ b/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/serviceprovider/ZkNodeDataSinkMetricsIntegrationTest.java @@ -99,13 +99,13 @@ void testUpdateState_createNewNode_recordsSuccess() { // Verify success meter val successMeter = metricRegistry.getMeters().get( - METRIC_PREFIX + ".dataSource.ZK.dataSource.zk-sink-1.nodeDataSink.update.success"); + METRIC_PREFIX + ".dataStoreType.ZK.dataSource.zk-sink-1.nodeDataSink.update.success"); assertNotNull(successMeter, "Node data sink update success meter should be recorded"); assertEquals(1, successMeter.getCount()); // No failure val failureMeter = metricRegistry.getMeters().get( - METRIC_PREFIX + ".dataSource.ZK.dataSource.zk-sink-1.nodeDataSink.update.failure"); + METRIC_PREFIX + ".dataStoreType.ZK.dataSource.zk-sink-1.nodeDataSink.update.failure"); assertNull(failureMeter, "No failure meter should be recorded on success"); sink.stop(); @@ -140,7 +140,7 @@ void testUpdateState_updateExistingNode_recordsSuccess() { // Verify 2 success markers val successMeter = metricRegistry.getMeters().get( - METRIC_PREFIX + ".dataSource.ZK.dataSource.zk-sink-2.nodeDataSink.update.success"); + METRIC_PREFIX + ".dataStoreType.ZK.dataSource.zk-sink-2.nodeDataSink.update.success"); assertNotNull(successMeter, "Success meter should exist"); assertEquals(2, successMeter.getCount()); @@ -213,9 +213,9 @@ void testUpdateState_stoppedSink_noMetricsRecorded() { // No success or failure metrics assertNull(metricRegistry.getMeters().get( - METRIC_PREFIX + ".dataSource.ZK.dataSource.zk-sink-4.nodeDataSink.update.success")); + METRIC_PREFIX + ".dataStoreType.ZK.dataSource.zk-sink-4.nodeDataSink.update.success")); assertNull(metricRegistry.getMeters().get( - METRIC_PREFIX + ".dataSource.ZK.dataSource.zk-sink-4.nodeDataSink.update.failure")); + METRIC_PREFIX + ".dataStoreType.ZK.dataSource.zk-sink-4.nodeDataSink.update.failure")); } // ==================== isActive() via ZkNodeDataStoreConnector base class ==================== @@ -231,7 +231,7 @@ void testIsActive_connectedCurator_recordsActiveStatus() { assertTrue(active); val activeMeter = metricRegistry.getMeters().get( - METRIC_PREFIX + ".dataSource.ZK.zk-sink-5.active"); + METRIC_PREFIX + ".dataStoreType.ZK.dataSource.zk-sink-5.active"); assertNotNull(activeMeter, "Active meter should be recorded"); assertEquals(1, activeMeter.getCount()); From e612bff7d19a7ccd23d7741427293be859ce161d Mon Sep 17 00:00:00 2001 From: Jitendra Dhawan Date: Tue, 2 Jun 2026 19:38:03 +0530 Subject: [PATCH 21/24] Refactor HealthChecker to remove DataStoreType and metricId parameters; simplify metric recording --- .../core/healthcheck/HealthChecker.java | 12 +++------ .../BaseServiceProviderBuilder.java | 4 +-- .../ranger/core/util/MetricRecorder.java | 11 ++++---- .../HealthCheckerMetricsIntegrationTest.java | 26 +++++++++---------- 4 files changed, 21 insertions(+), 32 deletions(-) diff --git a/ranger-core/src/main/java/io/appform/ranger/core/healthcheck/HealthChecker.java b/ranger-core/src/main/java/io/appform/ranger/core/healthcheck/HealthChecker.java index 891ee53c..a52e65ad 100644 --- a/ranger-core/src/main/java/io/appform/ranger/core/healthcheck/HealthChecker.java +++ b/ranger-core/src/main/java/io/appform/ranger/core/healthcheck/HealthChecker.java @@ -15,7 +15,6 @@ */ package io.appform.ranger.core.healthcheck; -import io.appform.ranger.core.model.DataStoreType; import io.appform.ranger.core.util.MetricRecorder; import lombok.extern.slf4j.Slf4j; import lombok.val; @@ -29,17 +28,12 @@ @Slf4j public class HealthChecker implements Supplier { - private final DataStoreType dataStoreType; - private final String metricId; private final List healthChecks; private final int staleUpdateThreshold; private HealthcheckStatus lastHealthcheckStatus; private long lastUpdatedTime; - public HealthChecker(DataStoreType dataStoreType, String metricId, - List healthChecks, int staleUpdateThreshold) { - this.dataStoreType = dataStoreType; - this.metricId = metricId; + public HealthChecker(List healthChecks, int staleUpdateThreshold) { this.healthChecks = healthChecks; this.staleUpdateThreshold = staleUpdateThreshold; } @@ -64,13 +58,13 @@ private boolean refreshHealth() { catch (Exception e) { log.error("Error running healthcheck. Setting node to unhealthy", e); healthcheckStatus = HealthcheckStatus.unhealthy; - MetricRecorder.recordHealthcheckFailure(dataStoreType, metricId); + MetricRecorder.recordHealthcheckFailure(); } if (HealthcheckStatus.unhealthy == healthcheckStatus) { break; } } - MetricRecorder.recordHealthcheckStatus(dataStoreType, metricId, HealthcheckStatus.healthy == healthcheckStatus); + MetricRecorder.recordHealthcheckStatus(HealthcheckStatus.healthy == healthcheckStatus); //Trigger update only if state change has happened //Conditions on which update will be triggered //1. First time diff --git a/ranger-core/src/main/java/io/appform/ranger/core/serviceprovider/BaseServiceProviderBuilder.java b/ranger-core/src/main/java/io/appform/ranger/core/serviceprovider/BaseServiceProviderBuilder.java index 24d1d3b6..be564948 100644 --- a/ranger-core/src/main/java/io/appform/ranger/core/serviceprovider/BaseServiceProviderBuilder.java +++ b/ranger-core/src/main/java/io/appform/ranger/core/serviceprovider/BaseServiceProviderBuilder.java @@ -18,7 +18,6 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; import io.appform.ranger.core.healthcheck.HealthChecker; import io.appform.ranger.core.healthcheck.Healthcheck; import io.appform.ranger.core.healthcheck.HealthcheckResult; @@ -211,8 +210,7 @@ protected final ServiceProvider buildProvider() { val healthcheckUpdateSignalGenerator = new ScheduledSignal<>( service, - new HealthChecker(usableNodeDataSource.getDataStoreType(), metricId, healthchecks, - staleUpdateThresholdMs), + new HealthChecker(healthchecks, staleUpdateThresholdMs), Collections.emptyList(), healthUpdateIntervalMs ); diff --git a/ranger-core/src/main/java/io/appform/ranger/core/util/MetricRecorder.java b/ranger-core/src/main/java/io/appform/ranger/core/util/MetricRecorder.java index 9562651d..fcfb28ad 100644 --- a/ranger-core/src/main/java/io/appform/ranger/core/util/MetricRecorder.java +++ b/ranger-core/src/main/java/io/appform/ranger/core/util/MetricRecorder.java @@ -36,6 +36,7 @@ public class MetricRecorder { public static final String DESERIALIZATION = "deserialization"; public static final String STALE_DATA_RETAINED = "staleDataRetained"; public static final String ZK_READ = "zkRead"; + public static final String HEALTH_CHECKER = "healthChecker"; private static MetricRegistry metricRegistry = new MetricRegistry(); @@ -91,17 +92,15 @@ public static void recordNodeDataSinkUpdateStatus(DataStoreType dataStoreType, S } } - public static void recordHealthcheckFailure(DataStoreType dataStoreType, String metricId) { + public static void recordHealthcheckFailure() { if (metricRegistry != null) { - metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), - DATA_SOURCE, metricId, HEALTHCHECK, FAILURE)).mark(); + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, HEALTH_CHECKER, FAILURE)).mark(); } } - public static void recordHealthcheckStatus(DataStoreType dataStoreType, String metricId, boolean healthy) { + public static void recordHealthcheckStatus(boolean healthy) { if (metricRegistry != null) { - metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), - DATA_SOURCE, metricId, HEALTHCHECK, "status", healthy ? HEALTHY : UNHEALTHY)).mark(); + metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, HEALTH_CHECKER,"status", healthy ? HEALTHY : UNHEALTHY)).mark(); } } diff --git a/ranger-core/src/test/java/io/appform/ranger/core/healthcheck/HealthCheckerMetricsIntegrationTest.java b/ranger-core/src/test/java/io/appform/ranger/core/healthcheck/HealthCheckerMetricsIntegrationTest.java index 531b2d22..9e94bf3a 100644 --- a/ranger-core/src/test/java/io/appform/ranger/core/healthcheck/HealthCheckerMetricsIntegrationTest.java +++ b/ranger-core/src/test/java/io/appform/ranger/core/healthcheck/HealthCheckerMetricsIntegrationTest.java @@ -16,7 +16,6 @@ package io.appform.ranger.core.healthcheck; import com.codahale.metrics.MetricRegistry; -import io.appform.ranger.core.model.DataStoreType; import io.appform.ranger.core.util.MetricRecorder; import lombok.val; import org.junit.jupiter.api.BeforeEach; @@ -29,12 +28,11 @@ class HealthCheckerMetricsIntegrationTest { private MetricRegistry metricRegistry; - private static final String METRIC_ID = "test-hc-metric"; - // Metric keys produced by MetricRecorder for DataStoreType.ZK and METRIC_ID: - // recordHealthcheckStatus -> io.appform.ranger.dataStoreType.ZK.dataSource..healthcheck.status. - // recordHealthcheckFailure -> io.appform.ranger.dataStoreType.ZK.dataSource..healthcheck.failure - private static final String HC_PREFIX = "io.appform.ranger.dataStoreType.ZK.dataSource." + METRIC_ID + ".healthcheck"; + // Metric keys produced by MetricRecorder (no DataStoreType/metricId context): + // recordHealthcheckStatus -> io.appform.ranger.healthChecker.status. + // recordHealthcheckFailure -> io.appform.ranger.healthChecker.failure + private static final String HC_PREFIX = "io.appform.ranger.healthChecker"; private static final String HEALTHY_KEY = HC_PREFIX + ".status.healthy"; private static final String UNHEALTHY_KEY = HC_PREFIX + ".status.unhealthy"; private static final String FAILURE_KEY = HC_PREFIX + ".failure"; @@ -47,7 +45,7 @@ void setUp() { @Test void testHealthyCheck_recordsHealthyMetric() { - val healthChecker = new HealthChecker(DataStoreType.ZK, METRIC_ID, + val healthChecker = new HealthChecker( List.of(() -> HealthcheckStatus.healthy), 10000); val result = healthChecker.get(); @@ -68,7 +66,7 @@ void testHealthyCheck_recordsHealthyMetric() { @Test void testUnhealthyCheck_recordsUnhealthyMetric() { - val healthChecker = new HealthChecker(DataStoreType.ZK, METRIC_ID, + val healthChecker = new HealthChecker( List.of(() -> HealthcheckStatus.unhealthy), 10000); val result = healthChecker.get(); @@ -87,7 +85,7 @@ void testUnhealthyCheck_recordsUnhealthyMetric() { @Test void testExceptionInHealthcheck_recordsFailureAndUnhealthyMetrics() { - val healthChecker = new HealthChecker(DataStoreType.ZK, METRIC_ID, List.of(() -> { + val healthChecker = new HealthChecker(List.of(() -> { throw new RuntimeException("Healthcheck error"); }), 10000); @@ -109,7 +107,7 @@ void testExceptionInHealthcheck_recordsFailureAndUnhealthyMetrics() { @Test void testMultipleHealthchecks_firstUnhealthy_shortCircuits() { - val healthChecker = new HealthChecker(DataStoreType.ZK, METRIC_ID, List.of( + val healthChecker = new HealthChecker(List.of( () -> HealthcheckStatus.unhealthy, () -> HealthcheckStatus.healthy // Should not be reached ), 10000); @@ -126,7 +124,7 @@ void testMultipleHealthchecks_firstUnhealthy_shortCircuits() { @Test void testMultipleHealthchecks_allHealthy() { - val healthChecker = new HealthChecker(DataStoreType.ZK, METRIC_ID, List.of( + val healthChecker = new HealthChecker(List.of( () -> HealthcheckStatus.healthy, () -> HealthcheckStatus.healthy, () -> HealthcheckStatus.healthy @@ -144,7 +142,7 @@ void testMultipleHealthchecks_allHealthy() { @Test void testRepeatedCalls_metricsAccumulate() { - val healthChecker = new HealthChecker(DataStoreType.ZK, METRIC_ID, + val healthChecker = new HealthChecker( List.of(() -> HealthcheckStatus.healthy), 0); // staleUpdateThreshold=0 => always returns result healthChecker.get(); @@ -160,7 +158,7 @@ void testRepeatedCalls_metricsAccumulate() { void testHealthStatusTransition_bothMetricsRecorded() { // Start healthy, then transition to unhealthy val statusHolder = new HealthcheckStatus[]{HealthcheckStatus.healthy}; - val healthChecker = new HealthChecker(DataStoreType.ZK, METRIC_ID, List.of(() -> statusHolder[0]), 0); + val healthChecker = new HealthChecker(List.of(() -> statusHolder[0]), 0); // First call: healthy val result1 = healthChecker.get(); @@ -184,7 +182,7 @@ void testHealthStatusTransition_bothMetricsRecorded() { @Test void testExceptionInSecondHealthcheck_recordsFailure() { - val healthChecker = new HealthChecker(DataStoreType.ZK, METRIC_ID, List.of( + val healthChecker = new HealthChecker(List.of( () -> HealthcheckStatus.healthy, // First passes () -> { throw new RuntimeException("Second fails"); } // Second throws ), 10000); From 610615403d1a3a116fe13b3e7c770abf1dfd656f Mon Sep 17 00:00:00 2001 From: Jitendra Dhawan Date: Tue, 2 Jun 2026 20:42:19 +0530 Subject: [PATCH 22/24] Address code review comments --- .../client/AbstractRangerHubClient.java | 8 +- .../ranger/client/stubs/RangerTestHub.java | 2 +- .../stubs/TestServiceFinderFactory.java | 2 +- .../TestSimpleUnshardedServiceFinder.java | 4 +- .../client/utils/RangerHubTestUtils.java | 4 +- .../core/finder/BaseServiceFinderBuilder.java | 12 +- .../ServiceRegistryUpdater.java | 6 +- .../ranger/core/model/NodeDataSink.java | 2 +- .../ranger/core/model/NodeDataSource.java | 2 +- .../BaseServiceProviderBuilder.java | 12 +- .../ranger/core/util/MetricRecorder.java | 105 +++++++++--------- ...RegistryUpdaterMetricsIntegrationTest.java | 101 +++++++++-------- .../core/finderhub/ServiceFinderHubTest.java | 23 ++-- .../HealthCheckerMetricsIntegrationTest.java | 2 +- .../serviceprovider/ServiceProviderTest.java | 8 +- .../bundle/ServiceDiscoveryBundle.java | 5 +- .../bundle/ServiceDiscoveryBundle.java | 5 +- .../drove/AbstractRangerDroveHubClient.java | 2 +- .../client/drove/SimpleRangerDroveClient.java | 2 +- .../drove/ShardedRangerDroveClientTest.java | 2 +- .../drove/UnshardedRangerDroveClientTest.java | 2 +- .../drove/common/DroveApiCommunicator.java | 20 ++-- .../common/DroveCachingCommunicator.java | 8 +- .../common/DroveNodeDataStoreConnector.java | 6 +- .../drove/common/DroveOkHttpTransport.java | 9 +- .../servicefinder/DroveNodeDataSource.java | 6 +- .../DroveShardedServiceFinderBuilder.java | 2 +- .../DroveUnshardedServiceFinderBuilider.java | 2 +- .../DroveServiceDataSource.java | 6 +- .../DroveShardedServiceFinderFactory.java | 2 +- .../DroveUnshardedServiceFinderFactory.java | 2 +- ...ApiCommunicatorMetricsIntegrationTest.java | 4 +- ...eNodeDataSourceMetricsIntegrationTest.java | 4 +- .../DroveShardedServiceFinderBuilderTest.java | 2 +- ...rviceDataSourceMetricsIntegrationTest.java | 4 +- .../http/AbstractRangerHttpHubClient.java | 4 +- .../client/http/SimpleRangerHttpClient.java | 2 +- .../http/ShardedRangerHttpClientTest.java | 2 +- .../http/UnshardedRangerHttpClientTest.java | 2 +- .../servicefinder/HttpApiCommunicator.java | 28 ++--- .../servicefinder/HttpNodeDataSource.java | 12 +- .../HttpShardedServiceFinderBuilder.java | 4 +- .../HttpUnshardedServiceFinderBuilider.java | 4 +- .../HttpServiceDataSource.java | 10 +- .../HttpShardedServiceFinderFactory.java | 2 +- .../HttpUnshardedServiceFinderFactory.java | 2 +- .../serviceprovider/HttpNodeDataSink.java | 28 +++-- .../HttpShardedServiceProviderBuilder.java | 4 +- ...ApiCommunicatorMetricsIntegrationTest.java | 4 +- .../HttpShardedServiceFinderBuilderTest.java | 2 +- ...rviceDataSourceMetricsIntegrationTest.java | 4 +- ...HttpShardedServiceProviderBuilderTest.java | 2 +- .../server/bundle/RangerHubServerBundle.java | 6 +- .../server/bundle/RangerHubServerBundle.java | 6 +- .../server/bundle/RangerHubServerBundle.java | 6 +- .../client/zk/AbstractRangerZKHubClient.java | 4 +- .../client/zk/ShardedRangerZKHubClient.java | 2 +- .../client/zk/SimpleRangerZKClient.java | 7 +- .../client/zk/UnshardedRangerZKHubClient.java | 2 +- .../client/zk/BaseRangerZKClientTest.java | 2 +- .../client/zk/ShardedZKRangerClientTest.java | 2 +- .../client/zk/SimpleRangerZKClientTest.java | 2 +- .../zk/UnshardedZKRangerClientTest.java | 2 +- .../common/ZkNodeDataStoreConnector.java | 8 +- .../servicefinder/ZkNodeDataSource.java | 33 +++--- .../ZkSimpleShardedServiceFinderBuilder.java | 4 +- ...ZkSimpleUnshardedServiceFinderBuilder.java | 4 +- .../ZKUnshardedServiceFinderFactory.java | 8 +- .../servicefinderhub/ZkServiceDataSource.java | 37 +++--- .../ZkShardedServiceFinderFactory.java | 8 +- .../serviceprovider/ZkNodeDataSink.java | 16 +-- .../ZkServiceProviderBuilder.java | 4 +- .../ServiceProviderIntegrationTest.java | 5 +- .../model/CustomShardSelectorTest.java | 4 +- .../model/ServiceNoProviderTest.java | 4 +- .../model/ServiceProviderExtCuratorTest.java | 4 +- .../model/ServiceProviderHealthcheckTest.java | 4 +- .../zookeeper/model/ServiceProviderTest.java | 8 +- .../model/SimpleServiceProviderTest.java | 4 +- .../zookeeper/servicehub/ServiceHubTest.java | 4 +- .../BaseServiceProviderBuilderTest.java | 4 +- 81 files changed, 361 insertions(+), 347 deletions(-) diff --git a/ranger-client/src/main/java/io/appform/ranger/client/AbstractRangerHubClient.java b/ranger-client/src/main/java/io/appform/ranger/client/AbstractRangerHubClient.java index 0028508a..66d838a3 100644 --- a/ranger-client/src/main/java/io/appform/ranger/client/AbstractRangerHubClient.java +++ b/ranger-client/src/main/java/io/appform/ranger/client/AbstractRangerHubClient.java @@ -37,7 +37,7 @@ @SuperBuilder public abstract class AbstractRangerHubClient, D extends Deserializer> implements RangerHubClient { - private final String metricId; + private final String upstreamId; private final String namespace; private final ObjectMapper mapper; private final D deserializer; @@ -61,7 +61,7 @@ public abstract class AbstractRangerHubClient, D @Override public void start() { - requireNonNull(metricId, "metricId can't be null"); + requireNonNull(upstreamId, "upstreamId can't be null"); requireNonNull(mapper, "Mapper can't be null"); requireNonNull(namespace, "namespace can't be null"); requireNonNull(deserializer, "deserializer can't be null"); @@ -90,7 +90,7 @@ public void start() { this.excludedServices = Objects.requireNonNullElseGet(this.excludedServices, Set::of); if(null == this.serviceDataSource){ - this.serviceDataSource = getDefaultDataSource(metricId); + this.serviceDataSource = getDefaultDataSource(upstreamId); } this.hub = buildHub(); @@ -195,7 +195,7 @@ public CompletableFuture addService(Service service) { } - protected abstract ServiceDataSource getDefaultDataSource(String metricId); + protected abstract ServiceDataSource getDefaultDataSource(String upstreamId); protected abstract ServiceFinderFactory getFinderFactory(); diff --git a/ranger-client/src/test/java/io/appform/ranger/client/stubs/RangerTestHub.java b/ranger-client/src/test/java/io/appform/ranger/client/stubs/RangerTestHub.java index 25970cbd..01b0c8b1 100644 --- a/ranger-client/src/test/java/io/appform/ranger/client/stubs/RangerTestHub.java +++ b/ranger-client/src/test/java/io/appform/ranger/client/stubs/RangerTestHub.java @@ -52,7 +52,7 @@ protected void postBuild(ServiceFinderHub> buildFinder(Service service) { val finder = new TestSimpleUnshardedServiceFinder() - .withMetricId("test-metric") + .withUpstreamId("test-metric") .withNamespace(service.getNamespace()) .withServiceName(service.getServiceName()) .withDeserializer(new Deserializer<>() { diff --git a/ranger-client/src/test/java/io/appform/ranger/client/stubs/TestSimpleUnshardedServiceFinder.java b/ranger-client/src/test/java/io/appform/ranger/client/stubs/TestSimpleUnshardedServiceFinder.java index 01ba1648..872dd535 100644 --- a/ranger-client/src/test/java/io/appform/ranger/client/stubs/TestSimpleUnshardedServiceFinder.java +++ b/ranger-client/src/test/java/io/appform/ranger/client/stubs/TestSimpleUnshardedServiceFinder.java @@ -35,14 +35,14 @@ public SimpleUnshardedServiceFinder build() { } @Override - protected NodeDataSource> dataSource(String metricId, Service service) { + protected NodeDataSource> dataSource(String upstreamId, Service service) { return new TestDataSource(); } static class TestDataSource implements NodeDataSource>{ @Override - public String getMetricId() { + public String getUpstreamId() { return "testDataSource"; } diff --git a/ranger-client/src/test/java/io/appform/ranger/client/utils/RangerHubTestUtils.java b/ranger-client/src/test/java/io/appform/ranger/client/utils/RangerHubTestUtils.java index 2f037a82..923729dd 100644 --- a/ranger-client/src/test/java/io/appform/ranger/client/utils/RangerHubTestUtils.java +++ b/ranger-client/src/test/java/io/appform/ranger/client/utils/RangerHubTestUtils.java @@ -39,7 +39,7 @@ public static RangerTestHub getTestHub(){ .nodeRefreshTimeMs(1000) .initialCriteria(new TestCriteria()) .deserializer(new TestDeserializer<>()) - .metricId("test-metric") + .upstreamId("test-metric") .build(); } @@ -52,7 +52,7 @@ public static RangerTestHub getTestHubWithDataSource(){ .useDefaultDataSource(false) .serviceDataSource(new StaticDataSource(Set.of(RangerHubTestUtils.service))) .deserializer(new TestDeserializer<>()) - .metricId("test-metric") + .upstreamId("test-metric") .build(); } diff --git a/ranger-core/src/main/java/io/appform/ranger/core/finder/BaseServiceFinderBuilder.java b/ranger-core/src/main/java/io/appform/ranger/core/finder/BaseServiceFinderBuilder.java index 84b265bd..90176edc 100644 --- a/ranger-core/src/main/java/io/appform/ranger/core/finder/BaseServiceFinderBuilder.java +++ b/ranger-core/src/main/java/io/appform/ranger/core/finder/BaseServiceFinderBuilder.java @@ -47,7 +47,7 @@ public abstract class BaseServiceFinderBuilder B extends BaseServiceFinderBuilder, D extends Deserializer> { - protected String metricId; + protected String upstreamId; protected String namespace; protected String serviceName; protected int nodeRefreshIntervalMs; @@ -59,8 +59,8 @@ public abstract class BaseServiceFinderBuilder protected final List> startSignalHandlers = new ArrayList<>(); protected final List> stopSignalHandlers = new ArrayList<>(); - public B withMetricId(final String metricId) { - this.metricId = metricId; + public B withUpstreamId(final String upstreamId) { + this.upstreamId = upstreamId; return (B)this; } @@ -142,7 +142,7 @@ public B withStopSignalHandlers(List> stopSignalHandlers) { public abstract F build(); protected F buildFinder() { - requireNonNull(metricId); + requireNonNull(upstreamId); requireNonNull(namespace); requireNonNull(serviceName); requireNonNull(deserializer); @@ -156,7 +156,7 @@ protected F buildFinder() { val finder = buildFinder(service, shardSelector, nodeSelector); val registry = finder.getServiceRegistry(); val signalGenerators = new ArrayList>(); - val nodeDataSource = dataSource(metricId, service); + val nodeDataSource = dataSource(upstreamId, service); signalGenerators.add(new ScheduledRegistryUpdateSignal<>(service, nodeRefreshIntervalMs)); additionalRefreshSignals.addAll(implementationSpecificRefreshSignals(service, nodeDataSource)); @@ -185,7 +185,7 @@ protected List> implementationSpecificRefreshSignals(Service service, return Collections.emptyList(); } - protected abstract NodeDataSource dataSource(String metricId, Service service); + protected abstract NodeDataSource dataSource(String upstreamId, Service service); protected abstract F buildFinder( Service service, diff --git a/ranger-core/src/main/java/io/appform/ranger/core/finder/serviceregistry/ServiceRegistryUpdater.java b/ranger-core/src/main/java/io/appform/ranger/core/finder/serviceregistry/ServiceRegistryUpdater.java index 1df2fa7c..63a1db87 100644 --- a/ranger-core/src/main/java/io/appform/ranger/core/finder/serviceregistry/ServiceRegistryUpdater.java +++ b/ranger-core/src/main/java/io/appform/ranger/core/finder/serviceregistry/ServiceRegistryUpdater.java @@ -151,7 +151,7 @@ private void updateRegistry() throws InterruptedException { //Remove all stale nodes before updating. This is done centrally to ensure some data sources //don't skip this check. Some control is still provided so that they can overload. serviceRegistry.updateNodes(FinderUtils.filterValidNodes(serviceRegistry.getService(), nodeList, livenessCheckMaxAge)); - MetricRecorder.recordNodeDataRefreshSuccess(nodeDataSource.getDataStoreType(), nodeDataSource.getMetricId(), + MetricRecorder.recordNodeDataRefreshSuccess(nodeDataSource.getDataStoreType(), nodeDataSource.getUpstreamId(), stopwatch.elapsed(TimeUnit.MILLISECONDS)); } else { @@ -165,7 +165,7 @@ private void updateRegistry() throws InterruptedException { e.getMessage()); callFailed = true; MetricRecorder.recordNodeDataRefreshFailure(serviceRegistry.getService().getServiceName(), - nodeDataSource.getDataStoreType(), nodeDataSource.getMetricId(), + nodeDataSource.getDataStoreType(), nodeDataSource.getUpstreamId(), stopwatch.elapsed(TimeUnit.MILLISECONDS)); } finally { stopwatch.stop(); @@ -182,7 +182,7 @@ private void updateRegistry() throws InterruptedException { .map(node -> node.setLastUpdatedTimeStamp(currTime)) .toList()); MetricRecorder.recordStaleDataRetained(serviceRegistry.getService().getServiceName(), - nodeDataSource.getDataStoreType(), nodeDataSource.getMetricId()); + nodeDataSource.getDataStoreType(), nodeDataSource.getUpstreamId()); } } diff --git a/ranger-core/src/main/java/io/appform/ranger/core/model/NodeDataSink.java b/ranger-core/src/main/java/io/appform/ranger/core/model/NodeDataSink.java index 1a9cde8e..1fb367c2 100644 --- a/ranger-core/src/main/java/io/appform/ranger/core/model/NodeDataSink.java +++ b/ranger-core/src/main/java/io/appform/ranger/core/model/NodeDataSink.java @@ -22,7 +22,7 @@ public interface NodeDataSink> extends NodeDataStoreC DataStoreType getDataStoreType(); - String getMetricId(); + String getUpstreamId(); void updateState(S serializer, ServiceNode serviceNode); } diff --git a/ranger-core/src/main/java/io/appform/ranger/core/model/NodeDataSource.java b/ranger-core/src/main/java/io/appform/ranger/core/model/NodeDataSource.java index 14d5e93d..3aedad4c 100644 --- a/ranger-core/src/main/java/io/appform/ranger/core/model/NodeDataSource.java +++ b/ranger-core/src/main/java/io/appform/ranger/core/model/NodeDataSource.java @@ -26,7 +26,7 @@ @SuppressWarnings("unused") public interface NodeDataSource> extends NodeDataStoreConnector { - String getMetricId(); + String getUpstreamId(); DataStoreType getDataStoreType(); diff --git a/ranger-core/src/main/java/io/appform/ranger/core/serviceprovider/BaseServiceProviderBuilder.java b/ranger-core/src/main/java/io/appform/ranger/core/serviceprovider/BaseServiceProviderBuilder.java index be564948..046de4a3 100644 --- a/ranger-core/src/main/java/io/appform/ranger/core/serviceprovider/BaseServiceProviderBuilder.java +++ b/ranger-core/src/main/java/io/appform/ranger/core/serviceprovider/BaseServiceProviderBuilder.java @@ -49,7 +49,7 @@ @SuppressWarnings({"unchecked", "unused", "UnusedReturnValue"}) public abstract class BaseServiceProviderBuilder, S extends Serializer> { - protected String metricId; + protected String upstreamId; protected String namespace; protected String serviceName; protected S serializer; @@ -69,8 +69,8 @@ public abstract class BaseServiceProviderBuilder> isolatedMonitors = new ArrayList<>(); - public B withMetricId(final String metricId) { - this.metricId = metricId; + public B withUpstreamId(final String upstreamId) { + this.upstreamId = upstreamId; return (B)this; } @@ -179,7 +179,7 @@ public B healthUpdateHandler(final HealthUpdateHandler healthUpdateHandler) { } protected final ServiceProvider buildProvider() { - requireNonNull(metricId); + requireNonNull(upstreamId); requireNonNull(namespace); requireNonNull(serviceName); requireNonNull(serializer); @@ -205,7 +205,7 @@ protected final ServiceProvider buildProvider() { healthchecks.add(serviceHealthAggregator); val service = Service.builder().namespace(namespace).serviceName(serviceName).build(); - val usableNodeDataSource = dataSink(metricId, service); + val usableNodeDataSource = dataSink(upstreamId, service); val healthcheckUpdateSignalGenerator = new ScheduledSignal<>( @@ -251,5 +251,5 @@ protected final ServiceProvider buildProvider() { public abstract ServiceProvider build(); - protected abstract NodeDataSink dataSink(String metricId, final Service service); + protected abstract NodeDataSink dataSink(String upstreamId, final Service service); } diff --git a/ranger-core/src/main/java/io/appform/ranger/core/util/MetricRecorder.java b/ranger-core/src/main/java/io/appform/ranger/core/util/MetricRecorder.java index fcfb28ad..2e827bb8 100644 --- a/ranger-core/src/main/java/io/appform/ranger/core/util/MetricRecorder.java +++ b/ranger-core/src/main/java/io/appform/ranger/core/util/MetricRecorder.java @@ -29,7 +29,6 @@ public class MetricRecorder { public static final String LIST_NODES = "listNodes"; public static final String SUCCESS = "success"; public static final String FAILURE = "failure"; - public static final String HEALTHCHECK = "healthcheck"; public static final String NODE_DATA_SINK = "nodeDataSink"; public static final String REGISTER_SERVICE = "registerService"; public static final String SERIALIZATION = "serialization"; @@ -38,7 +37,7 @@ public class MetricRecorder { public static final String ZK_READ = "zkRead"; public static final String HEALTH_CHECKER = "healthChecker"; - private static MetricRegistry metricRegistry = new MetricRegistry(); + private static MetricRegistry metricRegistry; public static void initialize(MetricRegistry registry) { metricRegistry = registry; @@ -51,44 +50,44 @@ public static void recordZombieNodeFound(String serviceName) { } } - public static void recordNoteDataSourceStatus(DataStoreType dataStoreType, String metricId, boolean active) { + public static void recordNoteDataSourceStatus(DataStoreType dataStoreType, String upstreamId, boolean active) { if (metricRegistry != null) { metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), - DATA_SOURCE, metricId, active ? ACTIVE : INACTIVE)).mark(); + DATA_SOURCE, upstreamId, active ? ACTIVE : INACTIVE)).mark(); } } - public static void recordNodeDataRefreshSuccess(DataStoreType dataStoreType, String metricId, long elapsed) { + public static void recordNodeDataRefreshSuccess(DataStoreType dataStoreType, String upstreamId, long elapsed) { if (metricRegistry != null) { metricRegistry.timer(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), - DATA_SOURCE, metricId, NODE_DATA_REFRESH, SUCCESS)).update(elapsed, MILLISECONDS); + DATA_SOURCE, upstreamId, NODE_DATA_REFRESH, SUCCESS)).update(elapsed, MILLISECONDS); } } public static void recordNodeDataRefreshFailure(String serviceName, DataStoreType dataStoreType, - String metricId, long elapsed) { + String upstreamId, long elapsed) { if (metricRegistry != null) { metricRegistry.timer(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), - DATA_SOURCE, metricId, NODE_DATA_REFRESH, FAILURE)).update(elapsed, MILLISECONDS); + DATA_SOURCE, upstreamId, NODE_DATA_REFRESH, FAILURE)).update(elapsed, MILLISECONDS); metricRegistry.timer(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), - DATA_SOURCE, metricId, SERVICE_NAME, serviceName, NODE_DATA_REFRESH, FAILURE)) + DATA_SOURCE, upstreamId, SERVICE_NAME, serviceName, NODE_DATA_REFRESH, FAILURE)) .update(elapsed, MILLISECONDS); } } - public static void recordStaleDataRetained(String serviceName, DataStoreType dataStoreType, String metricId) { + public static void recordStaleDataRetained(String serviceName, DataStoreType dataStoreType, String upstreamId) { if (metricRegistry != null) { metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), - DATA_SOURCE, metricId, STALE_DATA_RETAINED)).mark(); + DATA_SOURCE, upstreamId, STALE_DATA_RETAINED)).mark(); metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), - DATA_SOURCE, metricId, SERVICE_NAME, serviceName, STALE_DATA_RETAINED)).mark(); + DATA_SOURCE, upstreamId, SERVICE_NAME, serviceName, STALE_DATA_RETAINED)).mark(); } } - public static void recordNodeDataSinkUpdateStatus(DataStoreType dataStoreType, String metricId, String status) { + public static void recordNodeDataSinkUpdateStatus(DataStoreType dataStoreType, String upstreamId, String status) { if (metricRegistry != null) { metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), - DATA_SOURCE, metricId, NODE_DATA_SINK, UPDATE, status)).mark(); + DATA_SOURCE, upstreamId, NODE_DATA_SINK, UPDATE, status)).mark(); } } @@ -104,137 +103,137 @@ public static void recordHealthcheckStatus(boolean healthy) { } } - public static void recordServicesFetchStatus(DataStoreType dataStoreType, String metricId, String success) { + public static void recordServicesFetchStatus(DataStoreType dataStoreType, String upstreamId, String success) { if (metricRegistry != null) { metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), - DATA_SOURCE, metricId, SERVICES_LIST, "fetch", success)).mark(); + DATA_SOURCE, upstreamId, SERVICES_LIST, "fetch", success)).mark(); } } - public static void recordRemoteCallStatusCode(DataStoreType dataStoreType, String metricId, + public static void recordRemoteCallStatusCode(DataStoreType dataStoreType, String upstreamId, String remoteCall, int statusCode) { if (metricRegistry != null) { metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), - DATA_SOURCE, metricId, HTTP_CALL, remoteCall, "responseStatus", Integer.toString(statusCode))).mark(); + DATA_SOURCE, upstreamId, HTTP_CALL, remoteCall, "responseStatus", Integer.toString(statusCode))).mark(); } } - public static void recordCacheUpdateOnDroveEvent(DataStoreType dataStoreType, String metricId, + public static void recordCacheUpdateOnDroveEvent(DataStoreType dataStoreType, String upstreamId, String eventName, String serviceName) { if (metricRegistry != null) { metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), - DATA_SOURCE, metricId, "cacheUpdateOnDroveEvent", eventName, SERVICE_NAME, serviceName)).mark(); + DATA_SOURCE, upstreamId, "cacheUpdateOnDroveEvent", eventName, SERVICE_NAME, serviceName)).mark(); } } - public static void recordServicesParseFailure(DataStoreType dataStoreType, String metricId) { + public static void recordServicesParseFailure(DataStoreType dataStoreType, String upstreamId) { if (metricRegistry != null) { metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), - DATA_SOURCE, metricId, HTTP_CALL, SERVICES_LIST, RESPONSE_PARSE_FAILURE)).mark(); + DATA_SOURCE, upstreamId, HTTP_CALL, SERVICES_LIST, RESPONSE_PARSE_FAILURE)).mark(); } } - public static void recordListNodesParseFailure(DataStoreType dataStoreType, String metricId) { + public static void recordListNodesParseFailure(DataStoreType dataStoreType, String upstreamId) { if (metricRegistry != null) { metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), - DATA_SOURCE, metricId, HTTP_CALL, LIST_NODES, RESPONSE_PARSE_FAILURE)).mark(); + DATA_SOURCE, upstreamId, HTTP_CALL, LIST_NODES, RESPONSE_PARSE_FAILURE)).mark(); } } - public static void recordListNodesParseFailure(DataStoreType dataStoreType, String metricId, String serviceName) { + public static void recordListNodesParseFailure(DataStoreType dataStoreType, String upstreamId, String serviceName) { if (metricRegistry != null) { metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), - DATA_SOURCE, metricId, HTTP_CALL, LIST_NODES, SERVICE_NAME, serviceName, RESPONSE_PARSE_FAILURE)).mark(); + DATA_SOURCE, upstreamId, HTTP_CALL, LIST_NODES, SERVICE_NAME, serviceName, RESPONSE_PARSE_FAILURE)).mark(); } } - public static void recordZookeeperReadUnknownFailure(DataStoreType dataStoreType, String metricId, + public static void recordZookeeperReadUnknownFailure(DataStoreType dataStoreType, String upstreamId, String operation, String exceptionName) { if (metricRegistry != null) { metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), - DATA_SOURCE, metricId, ZK_READ, operation, UNKNOWN_FAILURE)).mark(); + DATA_SOURCE, upstreamId, ZK_READ, operation, UNKNOWN_FAILURE)).mark(); metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), - DATA_SOURCE, metricId, ZK_READ, operation, UNKNOWN_FAILURE, exceptionName)).mark(); + DATA_SOURCE, upstreamId, ZK_READ, operation, UNKNOWN_FAILURE, exceptionName)).mark(); } } - public static void recordRemoteCallUnknownFailure(DataStoreType dataStoreType, String metricId, + public static void recordRemoteCallUnknownFailure(DataStoreType dataStoreType, String upstreamId, String httpMethodName, String exceptionName) { if (metricRegistry != null) { metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), - DATA_SOURCE, metricId, HTTP_CALL, httpMethodName, UNKNOWN_FAILURE)).mark(); + DATA_SOURCE, upstreamId, HTTP_CALL, httpMethodName, UNKNOWN_FAILURE)).mark(); metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), - DATA_SOURCE, metricId, HTTP_CALL, httpMethodName, UNKNOWN_FAILURE, exceptionName)).mark(); + DATA_SOURCE, upstreamId, HTTP_CALL, httpMethodName, UNKNOWN_FAILURE, exceptionName)).mark(); } } - public static void recordRemoteCallUnknownFailure(DataStoreType dataStoreType, String metricId, + public static void recordRemoteCallUnknownFailure(DataStoreType dataStoreType, String upstreamId, String httpMethodName, String exceptionName, String serviceName) { - recordRemoteCallUnknownFailure(dataStoreType, metricId, httpMethodName, exceptionName); + recordRemoteCallUnknownFailure(dataStoreType, upstreamId, httpMethodName, exceptionName); if (metricRegistry != null) { metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), - DATA_SOURCE, metricId, HTTP_CALL, httpMethodName, SERVICE_NAME, serviceName, + DATA_SOURCE, upstreamId, HTTP_CALL, httpMethodName, SERVICE_NAME, serviceName, UNKNOWN_FAILURE)).mark(); metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), - DATA_SOURCE, metricId, HTTP_CALL, httpMethodName, SERVICE_NAME, serviceName, + DATA_SOURCE, upstreamId, HTTP_CALL, httpMethodName, SERVICE_NAME, serviceName, UNKNOWN_FAILURE, exceptionName)).mark(); } } - public static void recordNullOrEmptyServicesListResponse(DataStoreType dataStoreType, String metricId) { + public static void recordNullOrEmptyServicesListResponse(DataStoreType dataStoreType, String upstreamId) { if (metricRegistry != null) { metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), - DATA_SOURCE, metricId, HTTP_CALL, SERVICES_LIST, NULL_OR_EMPTY_RESPONSE)).mark(); + DATA_SOURCE, upstreamId, HTTP_CALL, SERVICES_LIST, NULL_OR_EMPTY_RESPONSE)).mark(); } } - public static void recordNullOrEmptyListNodeResponse(DataStoreType dataStoreType, String metricId) { + public static void recordNullOrEmptyListNodeResponse(DataStoreType dataStoreType, String upstreamId) { if (metricRegistry != null) { metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), - DATA_SOURCE, metricId, HTTP_CALL, LIST_NODES, NULL_OR_EMPTY_RESPONSE)).mark(); + DATA_SOURCE, upstreamId, HTTP_CALL, LIST_NODES, NULL_OR_EMPTY_RESPONSE)).mark(); } } - public static void recordNullOrEmptyListNodeResponse(DataStoreType dataStoreType, String metricId, + public static void recordNullOrEmptyListNodeResponse(DataStoreType dataStoreType, String upstreamId, String serviceName) { if (metricRegistry != null) { metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), - DATA_SOURCE, metricId, HTTP_CALL, LIST_NODES, SERVICE_NAME, serviceName, NULL_OR_EMPTY_RESPONSE)).mark(); + DATA_SOURCE, upstreamId, HTTP_CALL, LIST_NODES, SERVICE_NAME, serviceName, NULL_OR_EMPTY_RESPONSE)).mark(); } } - public static void recordNodeDataSinkUnknownFailure(DataStoreType dataStoreType, String metricId, + public static void recordNodeDataSinkUnknownFailure(DataStoreType dataStoreType, String upstreamId, String serviceName, String exceptionName) { if (metricRegistry != null) { metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), - DATA_SOURCE, metricId, NODE_DATA_SINK, UNKNOWN_FAILURE)).mark(); + DATA_SOURCE, upstreamId, NODE_DATA_SINK, UNKNOWN_FAILURE)).mark(); metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), - DATA_SOURCE, metricId, NODE_DATA_SINK, SERVICE_NAME, serviceName, UNKNOWN_FAILURE)).mark(); + DATA_SOURCE, upstreamId, NODE_DATA_SINK, SERVICE_NAME, serviceName, UNKNOWN_FAILURE)).mark(); metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), - DATA_SOURCE, metricId, NODE_DATA_SINK, SERVICE_NAME, serviceName, UNKNOWN_FAILURE, exceptionName)).mark(); + DATA_SOURCE, upstreamId, NODE_DATA_SINK, SERVICE_NAME, serviceName, UNKNOWN_FAILURE, exceptionName)).mark(); } } - public static void recordNodeDataSinkSerDeFailure(DataStoreType dataStoreType, String metricId, + public static void recordNodeDataSinkSerDeFailure(DataStoreType dataStoreType, String upstreamId, String serDe, String serviceName, String exceptionName) { if (metricRegistry != null) { metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), - DATA_SOURCE, metricId, NODE_DATA_SINK, serDe, FAILURE)).mark(); + DATA_SOURCE, upstreamId, NODE_DATA_SINK, serDe, FAILURE)).mark(); metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), - DATA_SOURCE, metricId, NODE_DATA_SINK, serDe, SERVICE_NAME, serviceName, FAILURE)).mark(); + DATA_SOURCE, upstreamId, NODE_DATA_SINK, serDe, SERVICE_NAME, serviceName, FAILURE)).mark(); metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), - DATA_SOURCE, metricId, NODE_DATA_SINK, serDe, SERVICE_NAME, serviceName, FAILURE, exceptionName)).mark(); + DATA_SOURCE, upstreamId, NODE_DATA_SINK, serDe, SERVICE_NAME, serviceName, FAILURE, exceptionName)).mark(); } } - public static void recordNullOrEmptyRegisterServiceResponse(DataStoreType dataStoreType, String metricId, + public static void recordNullOrEmptyRegisterServiceResponse(DataStoreType dataStoreType, String upstreamId, String serviceName) { if (metricRegistry != null) { metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), - DATA_SOURCE, metricId, HTTP_CALL, REGISTER_SERVICE, NULL_OR_EMPTY_RESPONSE)).mark(); + DATA_SOURCE, upstreamId, HTTP_CALL, REGISTER_SERVICE, NULL_OR_EMPTY_RESPONSE)).mark(); metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), - DATA_SOURCE, metricId, HTTP_CALL, REGISTER_SERVICE, SERVICE_NAME, serviceName, NULL_OR_EMPTY_RESPONSE)) + DATA_SOURCE, upstreamId, HTTP_CALL, REGISTER_SERVICE, SERVICE_NAME, serviceName, NULL_OR_EMPTY_RESPONSE)) .mark(); } } diff --git a/ranger-core/src/test/java/io/appform/ranger/core/finder/serviceregistry/ServiceRegistryUpdaterMetricsIntegrationTest.java b/ranger-core/src/test/java/io/appform/ranger/core/finder/serviceregistry/ServiceRegistryUpdaterMetricsIntegrationTest.java index b3e886ac..0298859b 100644 --- a/ranger-core/src/test/java/io/appform/ranger/core/finder/serviceregistry/ServiceRegistryUpdaterMetricsIntegrationTest.java +++ b/ranger-core/src/test/java/io/appform/ranger/core/finder/serviceregistry/ServiceRegistryUpdaterMetricsIntegrationTest.java @@ -22,20 +22,15 @@ import io.appform.ranger.core.units.TestNodeData; import io.appform.ranger.core.util.MetricRecorder; import lombok.val; -import org.awaitility.Awaitility; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import java.time.Duration; -import java.time.temporal.TemporalUnit; import java.util.Collections; import java.util.List; import java.util.Optional; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; -import java.util.function.Consumer; -import java.util.function.Supplier; import static org.awaitility.Awaitility.await; import static org.junit.jupiter.api.Assertions.*; @@ -81,13 +76,17 @@ void testSuccessfulRefresh_recordsSuccessTimer() { // Wait for initial update and a brief period for metric recording to complete awaitRefresh(registry); - sleep(100); // Allow time for MetricRecorder call after updateNodes - // Verify success timer + // Wait for success timer to be recorded using Awaitility val timerName = "io.appform.ranger.dataStoreType.ZK.dataSource." + METRIC_ID + ".nodeDataRefresh.success"; - val timer = metricRegistry.getTimers().get(timerName); - assertNotNull(timer, "Node data refresh success timer should exist"); - assertTrue(timer.getCount() >= 1, "Timer should have at least 1 update"); + await() + .atMost(Duration.ofSeconds(5)) + .pollInterval(Duration.ofMillis(50)) + .untilAsserted(() -> { + val timer = metricRegistry.getTimers().get(timerName); + assertNotNull(timer, "Node data refresh success timer should exist"); + assertTrue(timer.getCount() >= 1, "Timer should have at least 1 update"); + }); // No failure timer val failureTimerName = "io.appform.ranger.dataStoreType.ZK.dataSource." + METRIC_ID + ".nodeDataRefresh.failure"; @@ -110,14 +109,16 @@ void testRefreshThrowsException_recordsFailureTimer() { // Expected: initial update fails } - // Give some time for the updater thread to process - sleep(200); - - // Verify failure timer + // Wait for failure timer to be recorded using Awaitility val failureTimerName = "io.appform.ranger.dataStoreType.HTTP.dataSource." + METRIC_ID + ".nodeDataRefresh.failure"; - val failureTimer = metricRegistry.getTimers().get(failureTimerName); - assertNotNull(failureTimer, "Node data refresh failure timer should exist"); - assertTrue(failureTimer.getCount() >= 1, "Failure timer should have at least 1 update"); + await() + .atMost(Duration.ofSeconds(5)) + .pollInterval(Duration.ofMillis(50)) + .untilAsserted(() -> { + val failureTimer = metricRegistry.getTimers().get(failureTimerName); + assertNotNull(failureTimer, "Node data refresh failure timer should exist"); + assertTrue(failureTimer.getCount() >= 1, "Failure timer should have at least 1 update"); + }); } @Test @@ -143,13 +144,17 @@ void testInactiveDataSource_recordsStaleDataRetained() { // Now deactivate the data source and trigger another update dataSource.setActive(false); signal.fire(); - sleep(200); - // Verify stale data retained meter + // Wait for stale data retained meter to be recorded using Awaitility val meterName = "io.appform.ranger.dataStoreType.DROVE.dataSource." + METRIC_ID + ".staleDataRetained"; - val meter = metricRegistry.getMeters().get(meterName); - assertNotNull(meter, "Stale data retained meter should exist"); - assertTrue(meter.getCount() >= 1, "Stale data retained should be recorded at least once"); + await() + .atMost(Duration.ofSeconds(5)) + .pollInterval(Duration.ofMillis(50)) + .untilAsserted(() -> { + val meter = metricRegistry.getMeters().get(meterName); + assertNotNull(meter, "Stale data retained meter should exist"); + assertTrue(meter.getCount() >= 1, "Stale data retained should be recorded at least once"); + }); } @Test @@ -231,21 +236,32 @@ void testRefreshReturnsNull_noSuccessOrFailureTimer_butStaleRetained() { updater.start(); awaitRefresh(registry); - // Now set to null and trigger update - dataSource.setNodeList(null); - signal.fire(); - sleep(200); - - // When refresh returns null, no success timer should be incremented for that second call - // The success timer from the first call should be 1 + // Get the initial count after first successful refresh val timerName = "io.appform.ranger.dataStoreType.ZK.dataSource." + METRIC_ID + ".nodeDataRefresh.success"; + + // Wait for the first success timer to be recorded + await() + .atMost(Duration.ofSeconds(5)) + .pollInterval(Duration.ofMillis(50)) + .untilAsserted(() -> { + val timer = metricRegistry.getTimers().get(timerName); + assertNotNull(timer, "Success timer should exist after first refresh"); + assertTrue(timer.getCount() >= 1, "Should have at least one success"); + }); + val beforeTimer = metricRegistry.getTimers().get(timerName); - final long beforeCount = beforeTimer == null ? 0L : beforeTimer.getCount(); + final long beforeCount = beforeTimer.getCount(); // Now set to null and trigger update dataSource.setNodeList(null); signal.fire(); - sleep(200); + + // Wait a bit to ensure the signal processing would have completed + // Use a shorter wait since we're just ensuring the async operation completes + await() + .atMost(Duration.ofSeconds(2)) + .pollDelay(Duration.ofMillis(100)) + .until(() -> true); val afterTimer = metricRegistry.getTimers().get(timerName); final long afterCount = afterTimer == null ? 0L : afterTimer.getCount(); @@ -257,15 +273,10 @@ void testRefreshReturnsNull_noSuccessOrFailureTimer_butStaleRetained() { // ==================== Test helpers ==================== private void awaitRefresh(ServiceRegistry registry) { - val start = System.currentTimeMillis(); - while (!registry.isRefreshed() && (System.currentTimeMillis() - start) < 5000) { - sleep(50); - } - assertTrue(registry.isRefreshed(), "Registry should be refreshed within timeout"); - } - - private void sleep(long ms) { - await().pollDelay(Duration.ofMillis(ms)).until(() -> true); + await() + .atMost(Duration.ofSeconds(5)) + .pollInterval(Duration.ofMillis(50)) + .untilAsserted(() -> assertTrue(registry.isRefreshed(), "Registry should be refreshed")); } // ==================== Test implementations ==================== @@ -274,15 +285,15 @@ static class TestDeserializer implements Deserializer { } static class TestNodeDataSource implements NodeDataSource { - private final String metricId; + private final String upstreamId; private final DataStoreType dataStoreType; private final AtomicBoolean active; private volatile List> nodeList; private final boolean throwOnRefresh; - TestNodeDataSource(String metricId, DataStoreType dataStoreType, boolean active, + TestNodeDataSource(String upstreamId, DataStoreType dataStoreType, boolean active, List> nodeList, boolean throwOnRefresh) { - this.metricId = metricId; + this.upstreamId = upstreamId; this.dataStoreType = dataStoreType; this.active = new AtomicBoolean(active); this.nodeList = nodeList; @@ -298,8 +309,8 @@ void setNodeList(List> nodeList) { } @Override - public String getMetricId() { - return metricId; + public String getUpstreamId() { + return upstreamId; } @Override diff --git a/ranger-core/src/test/java/io/appform/ranger/core/finderhub/ServiceFinderHubTest.java b/ranger-core/src/test/java/io/appform/ranger/core/finderhub/ServiceFinderHubTest.java index 291a25ea..bce345ae 100644 --- a/ranger-core/src/test/java/io/appform/ranger/core/finderhub/ServiceFinderHubTest.java +++ b/ranger-core/src/test/java/io/appform/ranger/core/finderhub/ServiceFinderHubTest.java @@ -17,7 +17,6 @@ package io.appform.ranger.core.finderhub; -import com.google.common.collect.Lists; import io.appform.ranger.core.exceptions.CommunicationException; import io.appform.ranger.core.finder.BaseServiceFinderBuilder; import io.appform.ranger.core.finder.ServiceFinder; @@ -47,7 +46,7 @@ class ServiceFinderHubTest { new DynamicDataSource(List.of(new Service("NS", "PRE_REGISTERED_SERVICE"))), service -> new TestServiceFinderBuilder() - .withMetricId("test-metric") + .withUpstreamId("test-metric") .withNamespace(service.getNamespace()) .withServiceName(service.getServiceName()) .withDeserializer(new Deserializer() { @@ -94,7 +93,7 @@ void testTimeoutOnHubStartup() { void testDelayedServiceAddition() { val delayedHub = new ServiceFinderHub<>(new DynamicDataSource(List.of(new Service("NS", "SERVICE"))), service -> new TestServiceFinderBuilder() - .withMetricId("test-metric") + .withUpstreamId("test-metric") .withNamespace(service.getNamespace()) .withServiceName(service.getServiceName()) .withDeserializer(new Deserializer() {}) @@ -103,7 +102,7 @@ void testDelayedServiceAddition() { Assertions.assertThrows(IllegalStateException.class, delayedHub::start); val serviceFinderHub = new ServiceFinderHub<>(new DynamicDataSource(List.of(new Service("NS", "SERVICE"))), service -> new TestServiceFinderBuilder() - .withMetricId("test-metric") + .withUpstreamId("test-metric") .withNamespace(service.getNamespace()) .withServiceName(service.getServiceName()) .withDeserializer(new Deserializer() {}) @@ -117,7 +116,7 @@ void testDelayedServiceAddition() { @Test void testDynamicServiceAdditionWithNonDynamicDataSource() { val serviceFinderHub = new ServiceFinderHub<>(new StaticDataSource(new HashSet<>()), service -> new TestServiceFinderBuilder() - .withMetricId("test-metric") + .withUpstreamId("test-metric") .withNamespace(service.getNamespace()) .withServiceName(service.getServiceName()) .withDeserializer(new Deserializer() { @@ -139,7 +138,7 @@ void testWeightedNodeSelectionWithVaryingWeights() { new DynamicDataSource(List.of(new Service("NS", "PRE_REGISTERED_SERVICE"))), service -> new TestServiceFinderBuilder() - .withMetricId("test-metric") + .withUpstreamId("test-metric") .withNamespace(service.getNamespace()) .withServiceName(service.getServiceName()) .withNodeSelector(new WeightedRandomServiceNodeSelector<>( @@ -152,7 +151,7 @@ void testWeightedNodeSelectionWithVaryingWeights() { }) .withDataSource(new NodeDataSource<>() { @Override - public String getMetricId() { + public String getUpstreamId() { return "testVaryingWeights"; } @@ -244,7 +243,7 @@ void testWeightedNodeSelectionWithVaryingNodeAge() { new DynamicDataSource(List.of(new Service("NS", "PRE_REGISTERED_SERVICE"))), service -> new TestServiceFinderBuilder() - .withMetricId("test-metric") + .withUpstreamId("test-metric") .withNamespace(service.getNamespace()) .withServiceName(service.getServiceName()) .withNodeSelector(new WeightedRandomServiceNodeSelector<>( @@ -257,7 +256,7 @@ void testWeightedNodeSelectionWithVaryingNodeAge() { }) .withDataSource(new NodeDataSource<>() { @Override - public String getMetricId() { + public String getUpstreamId() { return "testVaryingNodeAge"; } @@ -349,7 +348,7 @@ public class TestServiceFinderFactory implements ServiceFinderFactory> buildFinder(Service service) { val finder = new TestServiceFinderBuilder() - .withMetricId("test-metric") + .withUpstreamId("test-metric") .withNamespace(service.getNamespace()) .withServiceName(service.getServiceName()) .withDeserializer(new Deserializer() {}) @@ -386,7 +385,7 @@ public ServiceFinder> build( } @Override - protected NodeDataSource> dataSource(String metricId, Service service) { + protected NodeDataSource> dataSource(String upstreamId, Service service) { return testNodeDataSource; } @@ -412,7 +411,7 @@ public TestServiceFinderBuilder withDataSource( private static class TestNodeDataSource implements NodeDataSource> { @Override - public String getMetricId() { + public String getUpstreamId() { return "testNodeDataSource"; } diff --git a/ranger-core/src/test/java/io/appform/ranger/core/healthcheck/HealthCheckerMetricsIntegrationTest.java b/ranger-core/src/test/java/io/appform/ranger/core/healthcheck/HealthCheckerMetricsIntegrationTest.java index 9e94bf3a..0639f38e 100644 --- a/ranger-core/src/test/java/io/appform/ranger/core/healthcheck/HealthCheckerMetricsIntegrationTest.java +++ b/ranger-core/src/test/java/io/appform/ranger/core/healthcheck/HealthCheckerMetricsIntegrationTest.java @@ -29,7 +29,7 @@ class HealthCheckerMetricsIntegrationTest { private MetricRegistry metricRegistry; - // Metric keys produced by MetricRecorder (no DataStoreType/metricId context): + // Metric keys produced by MetricRecorder (no DataStoreType/upstreamId context): // recordHealthcheckStatus -> io.appform.ranger.healthChecker.status. // recordHealthcheckFailure -> io.appform.ranger.healthChecker.failure private static final String HC_PREFIX = "io.appform.ranger.healthChecker"; diff --git a/ranger-core/src/test/java/io/appform/ranger/core/serviceprovider/ServiceProviderTest.java b/ranger-core/src/test/java/io/appform/ranger/core/serviceprovider/ServiceProviderTest.java index 9d9d4387..0b1bfab8 100644 --- a/ranger-core/src/test/java/io/appform/ranger/core/serviceprovider/ServiceProviderTest.java +++ b/ranger-core/src/test/java/io/appform/ranger/core/serviceprovider/ServiceProviderTest.java @@ -64,7 +64,7 @@ public DataStoreType getDataStoreType() { } @Override - public String getMetricId() { + public String getUpstreamId() { return "test-metric"; } @@ -103,7 +103,7 @@ public ServiceProvider> build() { } @Override - protected NodeDataSink> dataSink(String metricId, Service service) { + protected NodeDataSink> dataSink(String upstreamId, Service service) { return new TestNodeDataSink<>(); } } @@ -123,7 +123,7 @@ void testInvalidServiceProviderNoHealthCheck() { final HealthUpdateHandler healthUpdateHandler = new LastUpdatedHandler() .setNext(new HealthStatusHandler()); Assertions.assertThrowsExactly(IllegalArgumentException.class, () -> new TestServiceProviderBuilder<>() - .withMetricId("test-metric") + .withUpstreamId("test-metric") .withServiceName("test-service") .withNamespace("test") .withHostname("localhost-1") @@ -150,7 +150,7 @@ void testBuildServiceProvider() { final HealthUpdateHandler healthUpdateHandler = new LastUpdatedHandler() .setNext(new HealthStatusHandler()); val testProvider = new TestServiceProviderBuilder<>() - .withMetricId("test-metric") + .withUpstreamId("test-metric") .withServiceName("test-service") .withNamespace("test") .withHostname("localhost-1") diff --git a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundle.java b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundle.java index 24f57be6..9570da63 100644 --- a/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundle.java +++ b/ranger-discovery-bundle/src/main/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundle.java @@ -20,7 +20,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; import io.appform.ranger.client.RangerClient; import io.appform.ranger.client.zk.SimpleRangerZKClient; import io.appform.ranger.common.server.ShardInfo; @@ -260,7 +259,7 @@ private RangerClient> buildDiscove ShardSelector> shardSelector, final ServiceNodeSelector nodeSelector) { return SimpleRangerZKClient.builder() - .metricId(DEFAULT_DATA_SINK_ID) + .upstreamId(DEFAULT_DATA_SINK_ID) .curatorFramework(curator) .namespace(namespace) .serviceName(serviceName) @@ -306,7 +305,7 @@ private ServiceProvider> buildService .setNext(new RoutingWeightHandler<>(getWeightSupplier().get())) .setNext(new StartupTimeHandler<>()); val serviceProviderBuilder = ServiceProviderBuilders.shardedServiceProviderBuilder() - .withMetricId(DEFAULT_DATA_SINK_ID) + .withUpstreamId(DEFAULT_DATA_SINK_ID) .withCuratorFramework(curator) .withNamespace(namespace) .withServiceName(serviceName) diff --git a/ranger-discovery-dw5-bundle/src/main/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundle.java b/ranger-discovery-dw5-bundle/src/main/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundle.java index 5523d8b2..9861e407 100644 --- a/ranger-discovery-dw5-bundle/src/main/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundle.java +++ b/ranger-discovery-dw5-bundle/src/main/java/io/appform/ranger/discovery/bundle/ServiceDiscoveryBundle.java @@ -20,7 +20,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; import io.appform.ranger.client.RangerClient; import io.appform.ranger.client.zk.SimpleRangerZKClient; import io.appform.ranger.common.server.ShardInfo; @@ -259,7 +258,7 @@ private RangerClient> buildDiscove ShardSelector> shardSelector, final ServiceNodeSelector nodeSelector) { return SimpleRangerZKClient.builder() - .metricId(DEFAULT_DATA_SINK_ID) + .upstreamId(DEFAULT_DATA_SINK_ID) .curatorFramework(curator) .namespace(namespace) .serviceName(serviceName) @@ -305,7 +304,7 @@ private ServiceProvider> buildService .setNext(new RoutingWeightHandler<>(getWeightSupplier().get())) .setNext(new StartupTimeHandler<>()); val serviceProviderBuilder = ServiceProviderBuilders.shardedServiceProviderBuilder() - .withMetricId(DEFAULT_DATA_SINK_ID) + .withUpstreamId(DEFAULT_DATA_SINK_ID) .withCuratorFramework(curator) .withNamespace(namespace) .withServiceName(serviceName) diff --git a/ranger-drove-client/src/main/java/io/appform/ranger/client/drove/AbstractRangerDroveHubClient.java b/ranger-drove-client/src/main/java/io/appform/ranger/client/drove/AbstractRangerDroveHubClient.java index 0c8cd508..fd350094 100644 --- a/ranger-drove-client/src/main/java/io/appform/ranger/client/drove/AbstractRangerDroveHubClient.java +++ b/ranger-drove-client/src/main/java/io/appform/ranger/client/drove/AbstractRangerDroveHubClient.java @@ -45,7 +45,7 @@ public abstract class AbstractRangerDroveHubClient nodeSelector = new RandomServiceNodeSelector<>(); @Override - protected ServiceDataSource getDefaultDataSource(String metricId) { + protected ServiceDataSource getDefaultDataSource(String upstreamId) { return new DroveServiceDataSource<>(clientConfig, getMapper(), getNamespace(), droveCommunicator); } diff --git a/ranger-drove-client/src/main/java/io/appform/ranger/client/drove/SimpleRangerDroveClient.java b/ranger-drove-client/src/main/java/io/appform/ranger/client/drove/SimpleRangerDroveClient.java index 6135c879..8b1ace0c 100644 --- a/ranger-drove-client/src/main/java/io/appform/ranger/client/drove/SimpleRangerDroveClient.java +++ b/ranger-drove-client/src/main/java/io/appform/ranger/client/drove/SimpleRangerDroveClient.java @@ -58,7 +58,7 @@ public void start() { requireNonNull(deserializer, "deserializer can't be null"); this.serviceFinder = DroveServiceFinderBuilders.droveUnshardedServiceFinderBuilider() - .withMetricId(clientConfig.getId()) + .withUpstreamId(clientConfig.getId()) .withClientConfig(clientConfig) .withServiceName(serviceName) .withNamespace(namespace) diff --git a/ranger-drove-client/src/test/java/io/appform/ranger/client/drove/ShardedRangerDroveClientTest.java b/ranger-drove-client/src/test/java/io/appform/ranger/client/drove/ShardedRangerDroveClientTest.java index faba7ad5..fc0a5fab 100644 --- a/ranger-drove-client/src/test/java/io/appform/ranger/client/drove/ShardedRangerDroveClientTest.java +++ b/ranger-drove-client/src/test/java/io/appform/ranger/client/drove/ShardedRangerDroveClientTest.java @@ -43,7 +43,7 @@ public TestNodeData translate(ExposedAppInfo appInfo, ExposedAppInfo.ExposedHost .namespace(namespace) .mapper(getObjectMapper()) .nodeRefreshTimeMs(1000) - .metricId("test-metric") + .upstreamId("test-metric") .build(); client.start(); val service = RangerTestUtils.getService(namespace, "TEST_APP"); diff --git a/ranger-drove-client/src/test/java/io/appform/ranger/client/drove/UnshardedRangerDroveClientTest.java b/ranger-drove-client/src/test/java/io/appform/ranger/client/drove/UnshardedRangerDroveClientTest.java index a30949cd..952f2a29 100644 --- a/ranger-drove-client/src/test/java/io/appform/ranger/client/drove/UnshardedRangerDroveClientTest.java +++ b/ranger-drove-client/src/test/java/io/appform/ranger/client/drove/UnshardedRangerDroveClientTest.java @@ -42,7 +42,7 @@ public TestNodeData translate(ExposedAppInfo appInfo, ExposedAppInfo.ExposedHost }) .mapper(getObjectMapper()) .nodeRefreshTimeMs(1000) - .metricId("test-metric") + .upstreamId("test-metric") .build(); client.start(); val service = RangerTestUtils.getService(namespace, "TEST_APP"); diff --git a/ranger-drove/src/main/java/io/appform/ranger/drove/common/DroveApiCommunicator.java b/ranger-drove/src/main/java/io/appform/ranger/drove/common/DroveApiCommunicator.java index 46b87064..728c7e6a 100644 --- a/ranger-drove/src/main/java/io/appform/ranger/drove/common/DroveApiCommunicator.java +++ b/ranger-drove/src/main/java/io/appform/ranger/drove/common/DroveApiCommunicator.java @@ -51,7 +51,7 @@ */ @Slf4j public class DroveApiCommunicator implements DroveCommunicator { - private final String metricId; + private final String upstreamId; private final String namespace; private final DroveUpstreamConfig config; private final DroveClient droveClient; @@ -67,7 +67,7 @@ public DroveApiCommunicator( this.config = config; this.droveClient = droveClient; this.mapper = mapper; - this.metricId = config.getId(); + this.upstreamId = config.getId(); resetter.scheduleWithFixedDelay(() -> upstreamAvailable.set(true), 0, 60, TimeUnit.SECONDS); } @@ -103,7 +103,7 @@ public List defaultValue() { @Override public List handle(DroveClient.Response response) throws Exception { - MetricRecorder.recordRemoteCallStatusCode(DataStoreType.DROVE, metricId, SERVICES_LIST, response.statusCode()); + MetricRecorder.recordRemoteCallStatusCode(DataStoreType.DROVE, upstreamId, SERVICES_LIST, response.statusCode()); if (response.statusCode() != HttpStatus.SC_OK) { throw new DroveCommunicationException("Error communicating to drove: " + response); } @@ -136,13 +136,13 @@ private ApiResponse> parseServicesResponse(DroveClient.R }); if(apiResponse == null || apiResponse.getData() == null || apiResponse.getData().isEmpty()) { - MetricRecorder.recordNullOrEmptyServicesListResponse(DataStoreType.DROVE, metricId); + MetricRecorder.recordNullOrEmptyServicesListResponse(DataStoreType.DROVE, upstreamId); log.warn("Received empty services list from drove. Response body: {}", response.body()); } return apiResponse; } catch (JsonProcessingException e) { - MetricRecorder.recordServicesParseFailure(DataStoreType.DROVE, metricId); + MetricRecorder.recordServicesParseFailure(DataStoreType.DROVE, upstreamId); log.error("Error parsing response from drove: {}. Response body: {}", e.getMessage(), response.body(), e); throw new DroveCommunicationException("Error parsing response from drove: " + e.getMessage()); } @@ -169,7 +169,7 @@ public Map> defaultValue() { @Override public Map> handle(DroveClient.Response response) { - MetricRecorder.recordRemoteCallStatusCode(DataStoreType.DROVE, metricId, LIST_NODES, response.statusCode()); + MetricRecorder.recordRemoteCallStatusCode(DataStoreType.DROVE, upstreamId, LIST_NODES, response.statusCode()); if (response.statusCode() != HttpStatus.SC_OK) { throwDroveCommError(response); } @@ -195,15 +195,15 @@ private ApiResponse> parseListNodesResponse(Iterable MetricRecorder.recordNullOrEmptyListNodeResponse(DataStoreType.DROVE, metricId, service.getServiceName())); + MetricRecorder.recordNullOrEmptyListNodeResponse(DataStoreType.DROVE, upstreamId); + services.forEach(service -> MetricRecorder.recordNullOrEmptyListNodeResponse(DataStoreType.DROVE, upstreamId, service.getServiceName())); log.warn("Received empty node list from drove. Response body: {}", response.body()); } return apiResponse; } catch (JsonProcessingException e) { - MetricRecorder.recordListNodesParseFailure(DataStoreType.DROVE, metricId); - services.forEach(service -> MetricRecorder.recordListNodesParseFailure(DataStoreType.DROVE, metricId, service.getServiceName())); + MetricRecorder.recordListNodesParseFailure(DataStoreType.DROVE, upstreamId); + services.forEach(service -> MetricRecorder.recordListNodesParseFailure(DataStoreType.DROVE, upstreamId, service.getServiceName())); log.error("Error parsing response from drove: {}. Response body: {}", e.getMessage(), response.body(), e); throw new DroveCommunicationException("Error parsing response from drove: " + e.getMessage()); } diff --git a/ranger-drove/src/main/java/io/appform/ranger/drove/common/DroveCachingCommunicator.java b/ranger-drove/src/main/java/io/appform/ranger/drove/common/DroveCachingCommunicator.java index a67f6207..7c7e8378 100644 --- a/ranger-drove/src/main/java/io/appform/ranger/drove/common/DroveCachingCommunicator.java +++ b/ranger-drove/src/main/java/io/appform/ranger/drove/common/DroveCachingCommunicator.java @@ -51,7 +51,7 @@ @Slf4j public class DroveCachingCommunicator implements DroveCommunicator { - private final String metricId; + private final String upstreamId; private final DroveCommunicator root; private final DroveRemoteEventListener listener; //Zombie check is 60 secs .. so this provides about 10 secs @@ -64,7 +64,7 @@ public DroveCachingCommunicator( DroveUpstreamConfig config, DroveClient droveClient, ObjectMapper mapper) { - this.metricId = config.getId(); + this.upstreamId = config.getId(); this.root = root; val offsetStore = new DroveEventPollingOffsetInMemoryStore(); offsetStore.setLastOffset(System.currentTimeMillis()); //Only interested in new events @@ -142,7 +142,7 @@ public Service visit(DroveAppStateChangeEvent appStateChanged) { val appName = appStateChanged.getMetadata().get(AppEventDataTag.APP_NAME); log.info("Received app state change event for app: {}", appName); val service = new Service(namespace, appName.toString()); - MetricRecorder.recordCacheUpdateOnDroveEvent(DataStoreType.DROVE, metricId, appStateChanged.getType().name(), service.getServiceName()); + MetricRecorder.recordCacheUpdateOnDroveEvent(DataStoreType.DROVE, upstreamId, appStateChanged.getType().name(), service.getServiceName()); return service; } @@ -151,7 +151,7 @@ public Service visit(DroveInstanceStateChangeEvent instanceStateChanged) { val appName = instanceStateChanged.getMetadata().get(AppInstanceEventDataTag.APP_NAME); log.info("Received instance state change event for app: {}", appName); val service = new Service(namespace, appName.toString()); - MetricRecorder.recordCacheUpdateOnDroveEvent(DataStoreType.DROVE, metricId, instanceStateChanged.getType().name(), service.getServiceName()); + MetricRecorder.recordCacheUpdateOnDroveEvent(DataStoreType.DROVE, upstreamId, instanceStateChanged.getType().name(), service.getServiceName()); return service; } })) diff --git a/ranger-drove/src/main/java/io/appform/ranger/drove/common/DroveNodeDataStoreConnector.java b/ranger-drove/src/main/java/io/appform/ranger/drove/common/DroveNodeDataStoreConnector.java index c15ea8ca..13f22da9 100644 --- a/ranger-drove/src/main/java/io/appform/ranger/drove/common/DroveNodeDataStoreConnector.java +++ b/ranger-drove/src/main/java/io/appform/ranger/drove/common/DroveNodeDataStoreConnector.java @@ -29,7 +29,7 @@ @Slf4j public class DroveNodeDataStoreConnector implements NodeDataStoreConnector { - protected final String metricId; + protected final String upstreamId; protected final DroveUpstreamConfig config; protected final ObjectMapper mapper; protected final DroveCommunicator droveClient; @@ -38,7 +38,7 @@ public DroveNodeDataStoreConnector( final DroveUpstreamConfig config, final ObjectMapper mapper, final DroveCommunicator droveClient) { - this.metricId = config.getId(); + this.upstreamId = config.getId(); this.config = config; this.mapper = mapper; this.droveClient = droveClient; @@ -66,7 +66,7 @@ public void stop() { @Override public boolean isActive() { var healthy = droveClient.healthy(); - MetricRecorder.recordNoteDataSourceStatus(DataStoreType.DROVE, metricId, healthy); + MetricRecorder.recordNoteDataSourceStatus(DataStoreType.DROVE, upstreamId, healthy); return healthy; } diff --git a/ranger-drove/src/main/java/io/appform/ranger/drove/common/DroveOkHttpTransport.java b/ranger-drove/src/main/java/io/appform/ranger/drove/common/DroveOkHttpTransport.java index 8f857a90..03a02141 100644 --- a/ranger-drove/src/main/java/io/appform/ranger/drove/common/DroveOkHttpTransport.java +++ b/ranger-drove/src/main/java/io/appform/ranger/drove/common/DroveOkHttpTransport.java @@ -26,7 +26,6 @@ import okhttp3.Headers; import okhttp3.OkHttpClient; import okhttp3.Request; -import okhttp3.internal.http.HttpMethod; import org.apache.http.client.methods.HttpGet; import java.net.URI; @@ -39,11 +38,11 @@ @Slf4j public class DroveOkHttpTransport implements DroveHttpTransport { - private final String metricId; + private final String upstreamId; private final OkHttpClient httpClient; - public DroveOkHttpTransport(String metricId, final OkHttpClient httpClient) { - this.metricId = metricId; + public DroveOkHttpTransport(String upstreamId, final OkHttpClient httpClient) { + this.upstreamId = upstreamId; this.httpClient = httpClient; log.info("Okhttp based transport initialized"); } @@ -75,7 +74,7 @@ public T get( return responseHandler.handle(droveResponse); } catch (Exception e) { - MetricRecorder.recordRemoteCallUnknownFailure(DataStoreType.DROVE, metricId, HttpGet.METHOD_NAME, e.getClass().getSimpleName()); + MetricRecorder.recordRemoteCallUnknownFailure(DataStoreType.DROVE, upstreamId, HttpGet.METHOD_NAME, e.getClass().getSimpleName()); log.error("Error calling drove: {}. Error: {}", e.getMessage(), e.getClass().getSimpleName()); throw new DroveCommunicationException(e.getMessage()); } diff --git a/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinder/DroveNodeDataSource.java b/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinder/DroveNodeDataSource.java index 17111810..58cc585b 100644 --- a/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinder/DroveNodeDataSource.java +++ b/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinder/DroveNodeDataSource.java @@ -53,8 +53,8 @@ public DroveNodeDataSource( } @Override - public String getMetricId() { - return metricId; + public String getUpstreamId() { + return upstreamId; } @Override @@ -80,7 +80,7 @@ public Optional>> refresh(D deserializer) { @Override public boolean isActive() { var healthy = droveClient.healthy(); - MetricRecorder.recordNoteDataSourceStatus(DataStoreType.DROVE, metricId, healthy); + MetricRecorder.recordNoteDataSourceStatus(DataStoreType.DROVE, upstreamId, healthy); return healthy; } diff --git a/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinder/DroveShardedServiceFinderBuilder.java b/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinder/DroveShardedServiceFinderBuilder.java index 472f617c..6aa3c605 100644 --- a/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinder/DroveShardedServiceFinderBuilder.java +++ b/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinder/DroveShardedServiceFinderBuilder.java @@ -59,7 +59,7 @@ public SimpleShardedServiceFinder build() { } @Override - protected NodeDataSource> dataSource(String metricId, Service service) { + protected NodeDataSource> dataSource(String upstreamId, Service service) { return new DroveNodeDataSource<>(service, clientConfig, mapper, Objects.requireNonNullElseGet(droveClient, () -> RangerDroveUtils.buildDroveClient( diff --git a/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinder/DroveUnshardedServiceFinderBuilider.java b/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinder/DroveUnshardedServiceFinderBuilider.java index e8173583..b324cd1c 100644 --- a/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinder/DroveUnshardedServiceFinderBuilider.java +++ b/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinder/DroveUnshardedServiceFinderBuilider.java @@ -56,7 +56,7 @@ public SimpleUnshardedServiceFinder build() { } @Override - protected NodeDataSource> dataSource(String metricId, Service service) { + protected NodeDataSource> dataSource(String upstreamId, Service service) { return new DroveNodeDataSource<>( service, clientConfig, diff --git a/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinderhub/DroveServiceDataSource.java b/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinderhub/DroveServiceDataSource.java index 2b9c9365..ec87c8e9 100644 --- a/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinderhub/DroveServiceDataSource.java +++ b/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinderhub/DroveServiceDataSource.java @@ -54,11 +54,11 @@ public Collection services() { .stream() .map(serviceName -> new Service(namespace, serviceName)) .toList(); - MetricRecorder.recordServicesFetchStatus(DataStoreType.DROVE, metricId, SUCCESS); + MetricRecorder.recordServicesFetchStatus(DataStoreType.DROVE, upstreamId, SUCCESS); return result; } catch (Exception e) { - log.error("Error fetching services from drove data source id: {}, namespace: {}", metricId, namespace, e); - MetricRecorder.recordServicesFetchStatus(DataStoreType.DROVE, metricId, FAILURE); + log.error("Error fetching services from drove data source id: {}, namespace: {}", upstreamId, namespace, e); + MetricRecorder.recordServicesFetchStatus(DataStoreType.DROVE, upstreamId, FAILURE); throw e; } } diff --git a/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinderhub/DroveShardedServiceFinderFactory.java b/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinderhub/DroveShardedServiceFinderFactory.java index 0f8520b5..487d4569 100644 --- a/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinderhub/DroveShardedServiceFinderFactory.java +++ b/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinderhub/DroveShardedServiceFinderFactory.java @@ -61,7 +61,7 @@ public DroveShardedServiceFinderFactory( @Override public ServiceFinder> buildFinder(Service service) { val serviceFinder = new DroveShardedServiceFinderBuilder() - .withMetricId(clientConfig.getId()) + .withUpstreamId(clientConfig.getId()) .withClientConfig(clientConfig) .withDroveClient(droveClient) .withObjectMapper(mapper) diff --git a/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinderhub/DroveUnshardedServiceFinderFactory.java b/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinderhub/DroveUnshardedServiceFinderFactory.java index cafd4370..3b73913b 100644 --- a/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinderhub/DroveUnshardedServiceFinderFactory.java +++ b/ranger-drove/src/main/java/io/appform/ranger/drove/servicefinderhub/DroveUnshardedServiceFinderFactory.java @@ -61,7 +61,7 @@ public DroveUnshardedServiceFinderFactory( @Override public ServiceFinder> buildFinder(Service service) { val serviceFinder = new DroveUnshardedServiceFinderBuilider() - .withMetricId(clientConfig.getId()) + .withUpstreamId(clientConfig.getId()) .withClientConfig(clientConfig) .withDroveCommunicator(droveCommunicator) .withObjectMapper(mapper) diff --git a/ranger-drove/src/test/java/io/appform/ranger/drove/common/DroveApiCommunicatorMetricsIntegrationTest.java b/ranger-drove/src/test/java/io/appform/ranger/drove/common/DroveApiCommunicatorMetricsIntegrationTest.java index 170c56f8..8fc6e05d 100644 --- a/ranger-drove/src/test/java/io/appform/ranger/drove/common/DroveApiCommunicatorMetricsIntegrationTest.java +++ b/ranger-drove/src/test/java/io/appform/ranger/drove/common/DroveApiCommunicatorMetricsIntegrationTest.java @@ -355,11 +355,11 @@ void testListNodes_multiServiceParseFailure_aggregateMetricPushedOnce(WireMockRu // ==================== Helper ==================== - private DroveCommunicator buildClient(WireMockRuntimeInfo wm, String metricId) { + private DroveCommunicator buildClient(WireMockRuntimeInfo wm, String upstreamId) { return RangerDroveUtils.buildDroveClient( "testns", DroveUpstreamConfig.builder() - .id(metricId) + .id(upstreamId) .endpoints(List.of("http://localhost:" + wm.getHttpPort())) .username("guest") .password("guest") diff --git a/ranger-drove/src/test/java/io/appform/ranger/drove/servicefinder/DroveNodeDataSourceMetricsIntegrationTest.java b/ranger-drove/src/test/java/io/appform/ranger/drove/servicefinder/DroveNodeDataSourceMetricsIntegrationTest.java index eaddeb3b..d4ea9c30 100644 --- a/ranger-drove/src/test/java/io/appform/ranger/drove/servicefinder/DroveNodeDataSourceMetricsIntegrationTest.java +++ b/ranger-drove/src/test/java/io/appform/ranger/drove/servicefinder/DroveNodeDataSourceMetricsIntegrationTest.java @@ -185,9 +185,9 @@ void testRefresh_communicationFailure_returnsEmpty(WireMockRuntimeInfo wm) { // ==================== Helpers ==================== - private DroveUpstreamConfig buildConfig(WireMockRuntimeInfo wm, String metricId) { + private DroveUpstreamConfig buildConfig(WireMockRuntimeInfo wm, String upstreamId) { return DroveUpstreamConfig.builder() - .id(metricId) + .id(upstreamId) .endpoints(List.of("http://localhost:" + wm.getHttpPort())) .username("guest") .password("guest") diff --git a/ranger-drove/src/test/java/io/appform/ranger/drove/servicefinder/DroveShardedServiceFinderBuilderTest.java b/ranger-drove/src/test/java/io/appform/ranger/drove/servicefinder/DroveShardedServiceFinderBuilderTest.java index cd5cadb7..babc9857 100644 --- a/ranger-drove/src/test/java/io/appform/ranger/drove/servicefinder/DroveShardedServiceFinderBuilderTest.java +++ b/ranger-drove/src/test/java/io/appform/ranger/drove/servicefinder/DroveShardedServiceFinderBuilderTest.java @@ -89,7 +89,7 @@ public NodeData translate(ExposedAppInfo appInfo, ExposedAppInfo.ExposedHost hos .withShardSelector((criteria, registry) -> registry.nodeList()) .withNodeRefreshIntervalMs(1000) .withDroveClient(droveClient) - .withMetricId("test-metric") + .withUpstreamId("test-metric") .build(); finder.start(); await().atMost(Duration.ofSeconds(30)) diff --git a/ranger-drove/src/test/java/io/appform/ranger/drove/servicefinderhub/DroveServiceDataSourceMetricsIntegrationTest.java b/ranger-drove/src/test/java/io/appform/ranger/drove/servicefinderhub/DroveServiceDataSourceMetricsIntegrationTest.java index b9c4d55b..982e167a 100644 --- a/ranger-drove/src/test/java/io/appform/ranger/drove/servicefinderhub/DroveServiceDataSourceMetricsIntegrationTest.java +++ b/ranger-drove/src/test/java/io/appform/ranger/drove/servicefinderhub/DroveServiceDataSourceMetricsIntegrationTest.java @@ -176,9 +176,9 @@ void testIsActive_afterFailedCall_recordsInactiveStatus(WireMockRuntimeInfo wm) // ==================== Helpers ==================== - private DroveUpstreamConfig buildConfig(WireMockRuntimeInfo wm, String metricId) { + private DroveUpstreamConfig buildConfig(WireMockRuntimeInfo wm, String upstreamId) { return DroveUpstreamConfig.builder() - .id(metricId) + .id(upstreamId) .endpoints(List.of("http://localhost:" + wm.getHttpPort())) .username("guest") .password("guest") diff --git a/ranger-http-client/src/main/java/io/appform/ranger/client/http/AbstractRangerHttpHubClient.java b/ranger-http-client/src/main/java/io/appform/ranger/client/http/AbstractRangerHttpHubClient.java index f078e287..aff83c23 100644 --- a/ranger-http-client/src/main/java/io/appform/ranger/client/http/AbstractRangerHttpHubClient.java +++ b/ranger-http-client/src/main/java/io/appform/ranger/client/http/AbstractRangerHttpHubClient.java @@ -49,8 +49,8 @@ public abstract class AbstractRangerHttpHubClient nodeSelector = new RandomServiceNodeSelector<>(); @Override - protected ServiceDataSource getDefaultDataSource(String metricId) { - return new HttpServiceDataSource<>(metricId, clientConfig, + protected ServiceDataSource getDefaultDataSource(String upstreamId) { + return new HttpServiceDataSource<>(upstreamId, clientConfig, Objects.requireNonNullElseGet(getHttpClient(), () -> RangerHttpUtils.httpClient( clientConfig, diff --git a/ranger-http-client/src/main/java/io/appform/ranger/client/http/SimpleRangerHttpClient.java b/ranger-http-client/src/main/java/io/appform/ranger/client/http/SimpleRangerHttpClient.java index 8c839d76..440f4f23 100644 --- a/ranger-http-client/src/main/java/io/appform/ranger/client/http/SimpleRangerHttpClient.java +++ b/ranger-http-client/src/main/java/io/appform/ranger/client/http/SimpleRangerHttpClient.java @@ -57,7 +57,7 @@ public void start() { requireNonNull(deserializer, "deserializer can't be null"); this.serviceFinder = HttpServiceFinderBuilders.httpUnshardedServiceFinderBuilider() - .withMetricId(clientConfig.getId()) + .withUpstreamId(clientConfig.getId()) .withClientConfig(clientConfig) .withServiceName(serviceName) .withNamespace(namespace) diff --git a/ranger-http-client/src/test/java/io/appform/ranger/client/http/ShardedRangerHttpClientTest.java b/ranger-http-client/src/test/java/io/appform/ranger/client/http/ShardedRangerHttpClientTest.java index 26b981df..0c420c95 100644 --- a/ranger-http-client/src/test/java/io/appform/ranger/client/http/ShardedRangerHttpClientTest.java +++ b/ranger-http-client/src/test/java/io/appform/ranger/client/http/ShardedRangerHttpClientTest.java @@ -34,7 +34,7 @@ void testShardedHttpHubClient(){ .deserializer(this::read) .mapper(getObjectMapper()) .nodeRefreshTimeMs(1000) - .metricId("test-metric") + .upstreamId("test-metric") .build(); client.start(); val service = RangerTestUtils.getService("test-n", "test-s"); diff --git a/ranger-http-client/src/test/java/io/appform/ranger/client/http/UnshardedRangerHttpClientTest.java b/ranger-http-client/src/test/java/io/appform/ranger/client/http/UnshardedRangerHttpClientTest.java index 0b734f0e..03749275 100644 --- a/ranger-http-client/src/test/java/io/appform/ranger/client/http/UnshardedRangerHttpClientTest.java +++ b/ranger-http-client/src/test/java/io/appform/ranger/client/http/UnshardedRangerHttpClientTest.java @@ -34,7 +34,7 @@ void testUnshardedRangerHubClient(){ .deserializer(this::read) .mapper(getObjectMapper()) .nodeRefreshTimeMs(1000) - .metricId("test-metric") + .upstreamId("test-metric") .build(); client.start(); val service = RangerTestUtils.getService("test-n", "test-s"); diff --git a/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpApiCommunicator.java b/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpApiCommunicator.java index 39833665..efaabbd2 100644 --- a/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpApiCommunicator.java +++ b/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpApiCommunicator.java @@ -50,7 +50,7 @@ public class HttpApiCommunicator implements HttpCommunicator { private final AtomicBoolean upstreamAvailable = new AtomicBoolean(true); private final ScheduledExecutorService resetter = Executors.newSingleThreadScheduledExecutor(); - private final String metricId; + private final String upstreamId; @Getter private final OkHttpClient httpClient; private final HttpClientConfig config; @@ -58,7 +58,7 @@ public class HttpApiCommunicator implements HttpCommunicator { public HttpApiCommunicator(OkHttpClient httpClient, HttpClientConfig config, ObjectMapper mapper) { Objects.requireNonNull(mapper, "mapper has not been set for node data"); - this.metricId = config.getId(); + this.upstreamId = config.getId(); this.httpClient = httpClient; this.config = config; this.mapper = mapper; @@ -86,7 +86,7 @@ public Set services() { .build(); try (val response = httpClient.newCall(request).execute()) { - MetricRecorder.recordRemoteCallStatusCode(DataStoreType.HTTP, metricId, SERVICES_LIST, response.code()); + MetricRecorder.recordRemoteCallStatusCode(DataStoreType.HTTP, upstreamId, SERVICES_LIST, response.code()); if (response.isSuccessful()) { return parseServices(response, httpUrl); } @@ -96,7 +96,7 @@ public Set services() { } } catch (Exception e) { - MetricRecorder.recordRemoteCallUnknownFailure(DataStoreType.HTTP, metricId, SERVICES_LIST, e.getClass().getSimpleName()); + MetricRecorder.recordRemoteCallUnknownFailure(DataStoreType.HTTP, upstreamId, SERVICES_LIST, e.getClass().getSimpleName()); throw new HttpCommunicationException( "Error parsing the response from server for url: " + httpUrl + " with exception " + e.getClass().getSimpleName() + ": " + e.getMessage()); @@ -125,7 +125,7 @@ public List> listNodes( .build(); try (val response = httpClient.newCall(request).execute()) { - MetricRecorder.recordRemoteCallStatusCode(DataStoreType.HTTP, metricId, LIST_NODES, response.code()); + MetricRecorder.recordRemoteCallStatusCode(DataStoreType.HTTP, upstreamId, LIST_NODES, response.code()); if (response.isSuccessful()) { return parseNodeList(service, deserializer, response, httpUrl); } @@ -134,7 +134,7 @@ public List> listNodes( } } catch (Exception e) { - MetricRecorder.recordRemoteCallUnknownFailure(DataStoreType.HTTP, metricId, LIST_NODES, e.getClass().getSimpleName(), service.getServiceName()); + MetricRecorder.recordRemoteCallUnknownFailure(DataStoreType.HTTP, upstreamId, LIST_NODES, e.getClass().getSimpleName(), service.getServiceName()); throw new HttpCommunicationException("Error getting node data from the http endpoint: " + httpUrl + ". Error: " + e.getMessage()); } @@ -173,7 +173,7 @@ private Set parseServices(Response response, HttpUrl httpUrl) { val serviceDataSourceResponse = mapper.readValue(bytes, ServiceDataSourceResponse.class); if(serviceDataSourceResponse == null || serviceDataSourceResponse.getData() == null || serviceDataSourceResponse.getData().isEmpty()) { - MetricRecorder.recordNullOrEmptyServicesListResponse(DataStoreType.HTTP, metricId); + MetricRecorder.recordNullOrEmptyServicesListResponse(DataStoreType.HTTP, upstreamId); log.warn("Received empty services list from http. Response body: {}", response.body()); } @@ -188,7 +188,7 @@ private Set parseServices(Response response, HttpUrl httpUrl) { } catch (HttpCommunicationException httpCommunicationException){ throw httpCommunicationException; } catch (Exception e) { - MetricRecorder.recordServicesParseFailure(DataStoreType.HTTP, metricId); + MetricRecorder.recordServicesParseFailure(DataStoreType.HTTP, upstreamId); throw new HttpCommunicationException( "Error reading data from server. Url: " + httpUrl + "Error: " + e.getMessage()); } @@ -202,8 +202,8 @@ private List> parseNodeList( try (val body = response.body()) { if (null == body) { log.warn("HTTP call to {} returned empty body", httpUrl); - MetricRecorder.recordNullOrEmptyListNodeResponse(DataStoreType.HTTP, metricId); - MetricRecorder.recordNullOrEmptyListNodeResponse(DataStoreType.HTTP, metricId, service.getServiceName()); + MetricRecorder.recordNullOrEmptyListNodeResponse(DataStoreType.HTTP, upstreamId); + MetricRecorder.recordNullOrEmptyListNodeResponse(DataStoreType.HTTP, upstreamId, service.getServiceName()); throw new HttpCommunicationException("Empty response received for call to " + httpUrl); } else { @@ -211,8 +211,8 @@ private List> parseNodeList( val serviceNodesResponse = deserializer.deserialize(bytes); if(serviceNodesResponse == null || serviceNodesResponse.getData() == null || serviceNodesResponse.getData().isEmpty()) { - MetricRecorder.recordNullOrEmptyListNodeResponse(DataStoreType.HTTP, metricId); - MetricRecorder.recordNullOrEmptyListNodeResponse(DataStoreType.HTTP, metricId, service.getServiceName()); + MetricRecorder.recordNullOrEmptyListNodeResponse(DataStoreType.HTTP, upstreamId); + MetricRecorder.recordNullOrEmptyListNodeResponse(DataStoreType.HTTP, upstreamId, service.getServiceName()); } if (serviceNodesResponse.valid()) { @@ -226,8 +226,8 @@ private List> parseNodeList( } catch (HttpCommunicationException httpCommunicationException){ throw httpCommunicationException; } catch (Exception e) { - MetricRecorder.recordListNodesParseFailure(DataStoreType.HTTP, metricId); - MetricRecorder.recordListNodesParseFailure(DataStoreType.HTTP, metricId, service.getServiceName()); + MetricRecorder.recordListNodesParseFailure(DataStoreType.HTTP, upstreamId); + MetricRecorder.recordListNodesParseFailure(DataStoreType.HTTP, upstreamId, service.getServiceName()); throw new HttpCommunicationException( "Error parsing node data from server. Url: " + httpUrl + "Error: " + e.getMessage()); } diff --git a/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpNodeDataSource.java b/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpNodeDataSource.java index 7b171c54..fbf30844 100644 --- a/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpNodeDataSource.java +++ b/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpNodeDataSource.java @@ -40,26 +40,26 @@ @Slf4j public class HttpNodeDataSource> extends HttpNodeDataStoreConnector implements NodeDataSource { - private final String metricId; + private final String upstreamId; private final Service service; private final AtomicBoolean upstreamAvailable = new AtomicBoolean(true); private final ScheduledExecutorService resetter = Executors.newSingleThreadScheduledExecutor(); public HttpNodeDataSource( - final String metricId, + final String upstreamId, final Service service, final HttpClientConfig config, final HttpCommunicator httpCommunicator) { super(config, httpCommunicator); Objects.requireNonNull(config, "client config has not been set for node data"); Objects.requireNonNull(httpCommunicator, "http communicator has not been set for node data"); - this.metricId = metricId; + this.upstreamId = upstreamId; this.service = service; resetter.scheduleWithFixedDelay(() -> upstreamAvailable.set(true), 0, 60, TimeUnit.SECONDS); } - public String getMetricId() { - return metricId; + public String getUpstreamId() { + return upstreamId; } @Override @@ -75,7 +75,7 @@ public Optional>> refresh(D deserializer) { @Override public boolean isActive() { var httpUpstreamAvailable = upstreamAvailable.get(); - MetricRecorder.recordNoteDataSourceStatus(DataStoreType.HTTP, metricId, httpUpstreamAvailable); + MetricRecorder.recordNoteDataSourceStatus(DataStoreType.HTTP, upstreamId, httpUpstreamAvailable); return httpUpstreamAvailable; } } diff --git a/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpShardedServiceFinderBuilder.java b/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpShardedServiceFinderBuilder.java index 70f995f5..d269dfa7 100644 --- a/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpShardedServiceFinderBuilder.java +++ b/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpShardedServiceFinderBuilder.java @@ -56,8 +56,8 @@ public SimpleShardedServiceFinder build() { } @Override - protected NodeDataSource> dataSource(String metricId, Service service) { - return new HttpNodeDataSource<>(metricId, service, clientConfig, + protected NodeDataSource> dataSource(String upstreamId, Service service) { + return new HttpNodeDataSource<>(upstreamId, service, clientConfig, Objects.requireNonNullElseGet(httpCommunicator, () -> RangerHttpUtils.httpClient(clientConfig, mapper))); } diff --git a/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpUnshardedServiceFinderBuilider.java b/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpUnshardedServiceFinderBuilider.java index 3954c5a0..625c7cc5 100644 --- a/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpUnshardedServiceFinderBuilider.java +++ b/ranger-http/src/main/java/io/appform/ranger/http/servicefinder/HttpUnshardedServiceFinderBuilider.java @@ -54,8 +54,8 @@ public SimpleUnshardedServiceFinder build() { } @Override - protected NodeDataSource> dataSource(String metricId, Service service) { - return new HttpNodeDataSource<>(metricId, service, clientConfig, + protected NodeDataSource> dataSource(String upstreamId, Service service) { + return new HttpNodeDataSource<>(upstreamId, service, clientConfig, Objects.requireNonNullElseGet(httpClient, () -> RangerHttpUtils.httpClient(clientConfig, mapper))); } diff --git a/ranger-http/src/main/java/io/appform/ranger/http/servicefinderhub/HttpServiceDataSource.java b/ranger-http/src/main/java/io/appform/ranger/http/servicefinderhub/HttpServiceDataSource.java index 3181694f..71eec43f 100644 --- a/ranger-http/src/main/java/io/appform/ranger/http/servicefinderhub/HttpServiceDataSource.java +++ b/ranger-http/src/main/java/io/appform/ranger/http/servicefinderhub/HttpServiceDataSource.java @@ -35,11 +35,11 @@ @Slf4j public class HttpServiceDataSource extends HttpNodeDataStoreConnector implements ServiceDataSource { - private final String metricId; + private final String upstreamId; - public HttpServiceDataSource(String metricId, HttpClientConfig config, HttpCommunicator httpClient) { + public HttpServiceDataSource(String upstreamId, HttpClientConfig config, HttpCommunicator httpClient) { super(config, httpClient); - this.metricId = metricId; + this.upstreamId = upstreamId; } @Override @@ -47,11 +47,11 @@ public Collection services() { Objects.requireNonNull(config, "client config has not been set for node data"); try { val result = httpCommunicator.services(); - MetricRecorder.recordServicesFetchStatus(DataStoreType.HTTP, metricId, SUCCESS); + MetricRecorder.recordServicesFetchStatus(DataStoreType.HTTP, upstreamId, SUCCESS); return result; } catch (Exception e) { - MetricRecorder.recordServicesFetchStatus(DataStoreType.HTTP, metricId, FAILURE); + MetricRecorder.recordServicesFetchStatus(DataStoreType.HTTP, upstreamId, FAILURE); throw e; } } diff --git a/ranger-http/src/main/java/io/appform/ranger/http/servicefinderhub/HttpShardedServiceFinderFactory.java b/ranger-http/src/main/java/io/appform/ranger/http/servicefinderhub/HttpShardedServiceFinderFactory.java index 3e1c042d..88ea5e4a 100644 --- a/ranger-http/src/main/java/io/appform/ranger/http/servicefinderhub/HttpShardedServiceFinderFactory.java +++ b/ranger-http/src/main/java/io/appform/ranger/http/servicefinderhub/HttpShardedServiceFinderFactory.java @@ -61,7 +61,7 @@ public HttpShardedServiceFinderFactory( @Override public ServiceFinder> buildFinder(Service service) { val serviceFinder = new HttpShardedServiceFinderBuilder() - .withMetricId(clientConfig.getId()) + .withUpstreamId(clientConfig.getId()) .withClientConfig(clientConfig) .withObjectMapper(mapper) .withHttpCommunicator(httpClient) diff --git a/ranger-http/src/main/java/io/appform/ranger/http/servicefinderhub/HttpUnshardedServiceFinderFactory.java b/ranger-http/src/main/java/io/appform/ranger/http/servicefinderhub/HttpUnshardedServiceFinderFactory.java index 3bdd5a73..5cd5a400 100644 --- a/ranger-http/src/main/java/io/appform/ranger/http/servicefinderhub/HttpUnshardedServiceFinderFactory.java +++ b/ranger-http/src/main/java/io/appform/ranger/http/servicefinderhub/HttpUnshardedServiceFinderFactory.java @@ -60,7 +60,7 @@ public HttpUnshardedServiceFinderFactory( @Override public ServiceFinder> buildFinder(Service service) { val serviceFinder = new HttpUnshardedServiceFinderBuilider() - .withMetricId(clientConfig.getId()) + .withUpstreamId(clientConfig.getId()) .withClientConfig(clientConfig) .withObjectMapper(mapper) .withHttpClient(httpClient) diff --git a/ranger-http/src/main/java/io/appform/ranger/http/serviceprovider/HttpNodeDataSink.java b/ranger-http/src/main/java/io/appform/ranger/http/serviceprovider/HttpNodeDataSink.java index 19f93194..2830358d 100644 --- a/ranger-http/src/main/java/io/appform/ranger/http/serviceprovider/HttpNodeDataSink.java +++ b/ranger-http/src/main/java/io/appform/ranger/http/serviceprovider/HttpNodeDataSink.java @@ -45,13 +45,13 @@ @Slf4j public class HttpNodeDataSink> extends HttpNodeDataStoreConnector implements NodeDataSink { - private final String metricId; + private final String upstreamId; private final Service service; private final ObjectMapper mapper; - public HttpNodeDataSink(String metricId, Service service, HttpClientConfig config, ObjectMapper mapper, HttpCommunicator httpClient) { + public HttpNodeDataSink(String upstreamId, Service service, HttpClientConfig config, ObjectMapper mapper, HttpCommunicator httpClient) { super(config, httpClient); - this.metricId = metricId; + this.upstreamId = upstreamId; this.service = service; this.mapper = mapper; } @@ -62,8 +62,8 @@ public DataStoreType getDataStoreType() { } @Override - public String getMetricId() { - return metricId; + public String getUpstreamId() { + return upstreamId; } @Override @@ -89,18 +89,22 @@ public void updateState(S serializer, ServiceNode serviceNode) { val serviceRegistrationResponse = registerService(service.getServiceName(), httpUrl, requestBody).orElse(null); if(null == serviceRegistrationResponse || !serviceRegistrationResponse.valid()){ log.warn("Http call to {} returned a failure response {}", httpUrl, serviceRegistrationResponse); - MetricRecorder.recordNullOrEmptyRegisterServiceResponse(DataStoreType.HTTP, metricId, service.getServiceName()); - MetricRecorder.recordNodeDataSinkUpdateStatus(DataStoreType.HTTP, metricId, FAILURE); + recordNullOrEmptyRegisterServiceResponse(); Exceptions.illegalState("Error updating state on the server for node data: " + httpUrl); } - MetricRecorder.recordNodeDataSinkUpdateStatus(DataStoreType.HTTP, metricId, SUCCESS); + MetricRecorder.recordNodeDataSinkUpdateStatus(DataStoreType.HTTP, upstreamId, SUCCESS); + } + + private void recordNullOrEmptyRegisterServiceResponse() { + MetricRecorder.recordNullOrEmptyRegisterServiceResponse(DataStoreType.HTTP, upstreamId, service.getServiceName()); + MetricRecorder.recordNodeDataSinkUpdateStatus(DataStoreType.HTTP, upstreamId, FAILURE); } private > byte[] getSerializedData(String serviceName, S serializer, ServiceNode serviceNode) { try { return serializer.serialize(serviceNode); } catch (Exception e) { - MetricRecorder.recordNodeDataSinkSerDeFailure(DataStoreType.HTTP, metricId, MetricRecorder.SERIALIZATION, serviceName, e.getClass().getSimpleName()); + MetricRecorder.recordNodeDataSinkSerDeFailure(DataStoreType.HTTP, upstreamId, MetricRecorder.SERIALIZATION, serviceName, e.getClass().getSimpleName()); log.error("Error serializing data for service {} with node {} with exception", serviceName, serviceNode, e); throw e; } @@ -112,7 +116,7 @@ private Optional> registerService(String serviceN .post(requestBody) .build(); try (val response = httpCommunicator.getHttpClient().newCall(request).execute()) { - MetricRecorder.recordRemoteCallStatusCode(DataStoreType.HTTP, metricId, REGISTER_SERVICE, response.code()); + MetricRecorder.recordRemoteCallStatusCode(DataStoreType.HTTP, upstreamId, REGISTER_SERVICE, response.code()); if (response.isSuccessful()) { try (val body = response.body()) { if (null == body) { @@ -128,7 +132,7 @@ private Optional> registerService(String serviceN } } catch (IOException e) { - MetricRecorder.recordNodeDataSinkUnknownFailure(DataStoreType.HTTP, metricId, serviceName, e.getClass().getSimpleName()); + MetricRecorder.recordNodeDataSinkUnknownFailure(DataStoreType.HTTP, upstreamId, serviceName, e.getClass().getSimpleName()); log.error("Error updating state on the server with httpUrl {} with exception {} ", httpUrl, e); } return Optional.empty(); @@ -141,7 +145,7 @@ private ServiceRegistrationResponse parseRegisterServiceResponse(String servi }); } catch (IOException e) { - MetricRecorder.recordNodeDataSinkSerDeFailure(DataStoreType.HTTP, metricId, DESERIALIZATION, serviceName, e.getClass().getSimpleName()); + MetricRecorder.recordNodeDataSinkSerDeFailure(DataStoreType.HTTP, upstreamId, DESERIALIZATION, serviceName, e.getClass().getSimpleName()); throw e; } } diff --git a/ranger-http/src/main/java/io/appform/ranger/http/serviceprovider/HttpShardedServiceProviderBuilder.java b/ranger-http/src/main/java/io/appform/ranger/http/serviceprovider/HttpShardedServiceProviderBuilder.java index 504434d3..6b6868ad 100644 --- a/ranger-http/src/main/java/io/appform/ranger/http/serviceprovider/HttpShardedServiceProviderBuilder.java +++ b/ranger-http/src/main/java/io/appform/ranger/http/serviceprovider/HttpShardedServiceProviderBuilder.java @@ -56,8 +56,8 @@ public ServiceProvider> build() { } @Override - protected NodeDataSink> dataSink(String metricId, Service service) { - return new HttpNodeDataSink<>(metricId, service, clientConfig, mapper, + protected NodeDataSink> dataSink(String upstreamId, Service service) { + return new HttpNodeDataSink<>(upstreamId, service, clientConfig, mapper, Objects.requireNonNullElseGet(httpClient, () -> RangerHttpUtils.httpClient(clientConfig, mapper))); } diff --git a/ranger-http/src/test/java/io/appform/ranger/http/servicefinder/HttpApiCommunicatorMetricsIntegrationTest.java b/ranger-http/src/test/java/io/appform/ranger/http/servicefinder/HttpApiCommunicatorMetricsIntegrationTest.java index f492e6e0..e7275279 100644 --- a/ranger-http/src/test/java/io/appform/ranger/http/servicefinder/HttpApiCommunicatorMetricsIntegrationTest.java +++ b/ranger-http/src/test/java/io/appform/ranger/http/servicefinder/HttpApiCommunicatorMetricsIntegrationTest.java @@ -357,9 +357,9 @@ void testListNodes_connectionRefused_recordsUnknownFailureWithServiceName() { // ==================== Helper ==================== - private HttpClientConfig buildConfig(WireMockRuntimeInfo wmInfo, String metricId) { + private HttpClientConfig buildConfig(WireMockRuntimeInfo wmInfo, String upstreamId) { return HttpClientConfig.builder() - .id(metricId) + .id(upstreamId) .host("127.0.0.1") .port(wmInfo.getHttpPort()) .connectionTimeoutMs(30_000) diff --git a/ranger-http/src/test/java/io/appform/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java b/ranger-http/src/test/java/io/appform/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java index 6f882785..9fc42a80 100644 --- a/ranger-http/src/test/java/io/appform/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java +++ b/ranger-http/src/test/java/io/appform/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java @@ -88,7 +88,7 @@ void testFinder(WireMockRuntimeInfo wireMockRuntimeInfo) throws Exception { }) .withShardSelector((criteria, registry) -> registry.nodeList()) .withNodeRefreshIntervalMs(1000) - .withMetricId("test-metric") + .withUpstreamId("test-metric") .build(); finder.start(); RangerTestUtils.sleepUntilFinderStarts(finder); diff --git a/ranger-http/src/test/java/io/appform/ranger/http/servicefinderhub/HttpServiceDataSourceMetricsIntegrationTest.java b/ranger-http/src/test/java/io/appform/ranger/http/servicefinderhub/HttpServiceDataSourceMetricsIntegrationTest.java index b6721bdc..724daebc 100644 --- a/ranger-http/src/test/java/io/appform/ranger/http/servicefinderhub/HttpServiceDataSourceMetricsIntegrationTest.java +++ b/ranger-http/src/test/java/io/appform/ranger/http/servicefinderhub/HttpServiceDataSourceMetricsIntegrationTest.java @@ -135,9 +135,9 @@ void testIsActive_recordsDataSourceStatus(WireMockRuntimeInfo wmInfo) { assertEquals(1, statusMeter.getCount()); } - private HttpClientConfig buildConfig(WireMockRuntimeInfo wmInfo, String metricId) { + private HttpClientConfig buildConfig(WireMockRuntimeInfo wmInfo, String upstreamId) { return HttpClientConfig.builder() - .id(metricId) + .id(upstreamId) .host("127.0.0.1") .port(wmInfo.getHttpPort()) .connectionTimeoutMs(30_000) diff --git a/ranger-http/src/test/java/io/appform/ranger/http/serviceprovider/HttpShardedServiceProviderBuilderTest.java b/ranger-http/src/test/java/io/appform/ranger/http/serviceprovider/HttpShardedServiceProviderBuilderTest.java index a89b45a8..a0ec57c7 100644 --- a/ranger-http/src/test/java/io/appform/ranger/http/serviceprovider/HttpShardedServiceProviderBuilderTest.java +++ b/ranger-http/src/test/java/io/appform/ranger/http/serviceprovider/HttpShardedServiceProviderBuilderTest.java @@ -89,7 +89,7 @@ void testProvider(WireMockRuntimeInfo wireMockRuntimeInfo) throws Exception { .withNodeData(farmNodeData) .withSerializer(node -> requestBytes) .healthUpdateHandler(healthUpdateHandler) - .withMetricId("test-metric") + .withUpstreamId("test-metric") .build(); serviceProvider.start(); Assertions.assertNotNull(serviceProvider); diff --git a/ranger-hub-server-bundle-core/ranger-hub-server-bundle/src/main/java/io/appform/ranger/hub/server/bundle/RangerHubServerBundle.java b/ranger-hub-server-bundle-core/ranger-hub-server-bundle/src/main/java/io/appform/ranger/hub/server/bundle/RangerHubServerBundle.java index 5808600b..b10d71f5 100644 --- a/ranger-hub-server-bundle-core/ranger-hub-server-bundle/src/main/java/io/appform/ranger/hub/server/bundle/RangerHubServerBundle.java +++ b/ranger-hub-server-bundle-core/ranger-hub-server-bundle/src/main/java/io/appform/ranger/hub/server/bundle/RangerHubServerBundle.java @@ -100,7 +100,7 @@ private RangerHubClient> addCurat .build(); curatorFrameworks.add(curatorFramework); return UnshardedRangerZKHubClient.builder() - .metricId(zkConfiguration.getId()) + .upstreamId(zkConfiguration.getId()) .namespace(namespace) .connectionString(zookeeper) .curatorFramework(curatorFramework) @@ -126,7 +126,7 @@ private RangerHubClient> addCurat private RangerHubClient> getHttpHubClient( HttpClientConfig httpClientConfig, RangerHttpUpstreamConfiguration httpConfiguration) { return UnshardedRangerHttpHubClient.builder() - .metricId(httpClientConfig.getId()) + .upstreamId(httpClientConfig.getId()) .namespace(namespace) .mapper(getMapper()) .clientConfig(httpClientConfig) @@ -155,7 +155,7 @@ private RangerHubClient> getDrove DroveUpstreamConfig.DEFAULT_REGION_TAG_NAME); val droveCommunicator = RangerDroveUtils.buildDroveClient(namespace, droveConfig, getMapper()); return UnshardedRangerDroveHubClient.builder() - .metricId(droveConfig.getId()) + .upstreamId(droveConfig.getId()) .namespace(namespace) .mapper(getMapper()) .clientConfig(droveConfig) diff --git a/ranger-hub-server-bundle/src/main/java/io/appform/ranger/hub/server/bundle/RangerHubServerBundle.java b/ranger-hub-server-bundle/src/main/java/io/appform/ranger/hub/server/bundle/RangerHubServerBundle.java index 54ebb52f..c1b3bcae 100644 --- a/ranger-hub-server-bundle/src/main/java/io/appform/ranger/hub/server/bundle/RangerHubServerBundle.java +++ b/ranger-hub-server-bundle/src/main/java/io/appform/ranger/hub/server/bundle/RangerHubServerBundle.java @@ -111,7 +111,7 @@ private RangerHubClient> addCurat .build(); curatorFrameworks.add(curatorFramework); return UnshardedRangerZKHubClient.builder() - .metricId(zkConfiguration.getId()) + .upstreamId(zkConfiguration.getId()) .namespace(namespace) .connectionString(zookeeper) .curatorFramework(curatorFramework) @@ -138,7 +138,7 @@ private RangerHubClient> addCurat private RangerHubClient> getHttpHubClient( HttpClientConfig httpClientConfig, RangerHttpUpstreamConfiguration httpConfiguration) { return UnshardedRangerHttpHubClient.builder() - .metricId(httpClientConfig.getId()) + .upstreamId(httpClientConfig.getId()) .namespace(namespace) .mapper(getMapper()) .clientConfig(httpClientConfig) @@ -168,7 +168,7 @@ private RangerHubClient> getDrove DroveUpstreamConfig.DEFAULT_REGION_TAG_NAME); val droveCommunicator = RangerDroveUtils.buildDroveClient(namespace, droveConfig, getMapper()); return UnshardedRangerDroveHubClient.builder() - .metricId(droveConfig.getId()) + .upstreamId(droveConfig.getId()) .namespace(namespace) .mapper(getMapper()) .clientConfig(droveConfig) diff --git a/ranger-hub-server-dw5-bundle/src/main/java/io/appform/ranger/hub/server/bundle/RangerHubServerBundle.java b/ranger-hub-server-dw5-bundle/src/main/java/io/appform/ranger/hub/server/bundle/RangerHubServerBundle.java index 7a54486b..7f445899 100644 --- a/ranger-hub-server-dw5-bundle/src/main/java/io/appform/ranger/hub/server/bundle/RangerHubServerBundle.java +++ b/ranger-hub-server-dw5-bundle/src/main/java/io/appform/ranger/hub/server/bundle/RangerHubServerBundle.java @@ -111,7 +111,7 @@ private RangerHubClient> addCurat .build(); curatorFrameworks.add(curatorFramework); return UnshardedRangerZKHubClient.builder() - .metricId(zkConfiguration.getId()) + .upstreamId(zkConfiguration.getId()) .namespace(namespace) .connectionString(zookeeper) .curatorFramework(curatorFramework) @@ -138,7 +138,7 @@ private RangerHubClient> addCurat private RangerHubClient> getHttpHubClient( HttpClientConfig httpClientConfig, RangerHttpUpstreamConfiguration httpConfiguration) { return UnshardedRangerHttpHubClient.builder() - .metricId(httpClientConfig.getId()) + .upstreamId(httpClientConfig.getId()) .namespace(namespace) .mapper(getMapper()) .clientConfig(httpClientConfig) @@ -168,7 +168,7 @@ private RangerHubClient> getDrove DroveUpstreamConfig.DEFAULT_REGION_TAG_NAME); val droveCommunicator = RangerDroveUtils.buildDroveClient(namespace, droveConfig, getMapper()); return UnshardedRangerDroveHubClient.builder() - .metricId(droveConfig.getId()) + .upstreamId(droveConfig.getId()) .namespace(namespace) .mapper(getMapper()) .clientConfig(droveConfig) diff --git a/ranger-zk-client/src/main/java/io/appform/ranger/client/zk/AbstractRangerZKHubClient.java b/ranger-zk-client/src/main/java/io/appform/ranger/client/zk/AbstractRangerZKHubClient.java index 68364789..311aa31c 100644 --- a/ranger-zk-client/src/main/java/io/appform/ranger/client/zk/AbstractRangerZKHubClient.java +++ b/ranger-zk-client/src/main/java/io/appform/ranger/client/zk/AbstractRangerZKHubClient.java @@ -53,8 +53,8 @@ protected ServiceFinderHub buildHub() { } @Override - protected ServiceDataSource getDefaultDataSource(String metricId) { - return new ZkServiceDataSource(metricId, getNamespace(), connectionString, curatorFramework); + protected ServiceDataSource getDefaultDataSource(String upstreamId) { + return new ZkServiceDataSource(upstreamId, getNamespace(), connectionString, curatorFramework); } } diff --git a/ranger-zk-client/src/main/java/io/appform/ranger/client/zk/ShardedRangerZKHubClient.java b/ranger-zk-client/src/main/java/io/appform/ranger/client/zk/ShardedRangerZKHubClient.java index aaa88288..e32053ff 100644 --- a/ranger-zk-client/src/main/java/io/appform/ranger/client/zk/ShardedRangerZKHubClient.java +++ b/ranger-zk-client/src/main/java/io/appform/ranger/client/zk/ShardedRangerZKHubClient.java @@ -41,7 +41,7 @@ public class ShardedRangerZKHubClient @Override protected ServiceFinderFactory> getFinderFactory() { return ZkShardedServiceFinderFactory.builder() - .metricId(getMetricId()) + .upstreamId(getUpstreamId()) .curatorFramework(getCuratorFramework()) .connectionString(getConnectionString()) .nodeRefreshIntervalMs(getNodeRefreshTimeMs()) diff --git a/ranger-zk-client/src/main/java/io/appform/ranger/client/zk/SimpleRangerZKClient.java b/ranger-zk-client/src/main/java/io/appform/ranger/client/zk/SimpleRangerZKClient.java index 1b8610f2..e4cb2904 100644 --- a/ranger-zk-client/src/main/java/io/appform/ranger/client/zk/SimpleRangerZKClient.java +++ b/ranger-zk-client/src/main/java/io/appform/ranger/client/zk/SimpleRangerZKClient.java @@ -18,7 +18,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import io.appform.ranger.client.AbstractRangerClient; import io.appform.ranger.core.finder.SimpleShardedServiceFinder; -import io.appform.ranger.core.finder.nodeselector.RandomServiceNodeSelector; import io.appform.ranger.core.finder.serviceregistry.MapBasedServiceRegistry; import io.appform.ranger.core.finder.shardselector.MatchingShardSelector; import io.appform.ranger.core.model.HubConstants; @@ -41,7 +40,7 @@ @SuperBuilder public class SimpleRangerZKClient extends AbstractRangerClient> { - private String metricId; + private String upstreamId; private final String serviceName; private final String namespace; private final ObjectMapper mapper; @@ -59,7 +58,7 @@ public class SimpleRangerZKClient extends AbstractRangerClientshardedFinderBuilder() - .withMetricId(metricId) + .withUpstreamId(upstreamId) .withCuratorFramework(curatorFramework) .withNamespace(namespace) .withServiceName(serviceName) diff --git a/ranger-zk-client/src/main/java/io/appform/ranger/client/zk/UnshardedRangerZKHubClient.java b/ranger-zk-client/src/main/java/io/appform/ranger/client/zk/UnshardedRangerZKHubClient.java index 7e34a320..4367c3ab 100644 --- a/ranger-zk-client/src/main/java/io/appform/ranger/client/zk/UnshardedRangerZKHubClient.java +++ b/ranger-zk-client/src/main/java/io/appform/ranger/client/zk/UnshardedRangerZKHubClient.java @@ -41,7 +41,7 @@ public class UnshardedRangerZKHubClient @Override protected ServiceFinderFactory> getFinderFactory() { return ZKUnshardedServiceFinderFactory.builder() - .metricId(getMetricId()) + .upstreamId(getUpstreamId()) .curatorFramework(getCuratorFramework()) .connectionString(getConnectionString()) .nodeRefreshIntervalMs(getNodeRefreshTimeMs()) diff --git a/ranger-zk-client/src/test/java/io/appform/ranger/client/zk/BaseRangerZKClientTest.java b/ranger-zk-client/src/test/java/io/appform/ranger/client/zk/BaseRangerZKClientTest.java index c81d0b0c..f22252cd 100644 --- a/ranger-zk-client/src/test/java/io/appform/ranger/client/zk/BaseRangerZKClientTest.java +++ b/ranger-zk-client/src/test/java/io/appform/ranger/client/zk/BaseRangerZKClientTest.java @@ -110,7 +110,7 @@ protected void initilizeProvider(){ final HealthUpdateHandler healthUpdateHandler = new LastUpdatedHandler() .setNext(new HealthStatusHandler()); provider = ServiceProviderBuilders.shardedServiceProviderBuilder() - .withMetricId("test-metric") + .withUpstreamId("test-metric") .withHostname("localhost") .withPort(1080) .withNamespace("test-n") diff --git a/ranger-zk-client/src/test/java/io/appform/ranger/client/zk/ShardedZKRangerClientTest.java b/ranger-zk-client/src/test/java/io/appform/ranger/client/zk/ShardedZKRangerClientTest.java index c016d782..85b6bd99 100644 --- a/ranger-zk-client/src/test/java/io/appform/ranger/client/zk/ShardedZKRangerClientTest.java +++ b/ranger-zk-client/src/test/java/io/appform/ranger/client/zk/ShardedZKRangerClientTest.java @@ -28,7 +28,7 @@ class ShardedZKRangerClientTest extends BaseRangerZKClientTest { @Test void testShardedHub(){ val zkHubClient = ShardedRangerZKHubClient.builder() - .metricId("testzk") + .upstreamId("testzk") .namespace("test-n") .connectionString(getTestingCluster().getConnectString()) .curatorFramework(getCuratorFramework()) diff --git a/ranger-zk-client/src/test/java/io/appform/ranger/client/zk/SimpleRangerZKClientTest.java b/ranger-zk-client/src/test/java/io/appform/ranger/client/zk/SimpleRangerZKClientTest.java index 40bcf201..d1e096cb 100644 --- a/ranger-zk-client/src/test/java/io/appform/ranger/client/zk/SimpleRangerZKClientTest.java +++ b/ranger-zk-client/src/test/java/io/appform/ranger/client/zk/SimpleRangerZKClientTest.java @@ -31,7 +31,7 @@ void testBaseClient(){ .serviceName("s1") .disableWatchers(true) .mapper(getObjectMapper()) - .metricId("test-metric") + .upstreamId("test-metric") .build(); client.start(); Assertions.assertNotNull( client.getNode().orElse(null)); diff --git a/ranger-zk-client/src/test/java/io/appform/ranger/client/zk/UnshardedZKRangerClientTest.java b/ranger-zk-client/src/test/java/io/appform/ranger/client/zk/UnshardedZKRangerClientTest.java index ad78e8b6..e9aeefb4 100644 --- a/ranger-zk-client/src/test/java/io/appform/ranger/client/zk/UnshardedZKRangerClientTest.java +++ b/ranger-zk-client/src/test/java/io/appform/ranger/client/zk/UnshardedZKRangerClientTest.java @@ -35,7 +35,7 @@ void testShardedHub(){ .mapper(getObjectMapper()) .deserializer(this::read) .nodeRefreshTimeMs(1000) - .metricId("test-metric") + .upstreamId("test-metric") .build(); zkHubClient.start(); val service = RangerTestUtils.getService("test-n", "s1"); diff --git a/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/common/ZkNodeDataStoreConnector.java b/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/common/ZkNodeDataStoreConnector.java index db5f7fd4..41eb55fb 100644 --- a/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/common/ZkNodeDataStoreConnector.java +++ b/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/common/ZkNodeDataStoreConnector.java @@ -39,7 +39,7 @@ @Slf4j public class ZkNodeDataStoreConnector implements NodeDataStoreConnector { - protected final String metricId; + protected final String upstreamId; @Getter(AccessLevel.PROTECTED) protected final Service service; @Getter(AccessLevel.PROTECTED) @@ -64,10 +64,10 @@ public class ZkNodeDataStoreConnector implements NodeDataStoreConnector { .build(); protected ZkNodeDataStoreConnector( - String metricId, final Service service, + String upstreamId, final Service service, final CuratorFramework curatorFramework, final ZkStoreType storeType) { - this.metricId = metricId; + this.upstreamId = upstreamId; this.service = service; this.curatorFramework = curatorFramework; this.storeType = storeType; @@ -168,7 +168,7 @@ public void stop() { public boolean isActive() { var zkConnectionActive = curatorFramework != null && curatorFramework.getZookeeperClient() != null && curatorFramework.getZookeeperClient().isConnected(); - MetricRecorder.recordNoteDataSourceStatus(DataStoreType.ZK, metricId, zkConnectionActive); + MetricRecorder.recordNoteDataSourceStatus(DataStoreType.ZK, upstreamId, zkConnectionActive); return zkConnectionActive; } diff --git a/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinder/ZkNodeDataSource.java b/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinder/ZkNodeDataSource.java index 41bdbb42..f349d863 100644 --- a/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinder/ZkNodeDataSource.java +++ b/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinder/ZkNodeDataSource.java @@ -45,14 +45,14 @@ public class ZkNodeDataSource> extends ZkNodeDataStoreConnector implements NodeDataSource { public ZkNodeDataSource( - String metricId, Service service, + String upstreamId, Service service, CuratorFramework curatorFramework) { - super(metricId, service, curatorFramework, ZkStoreType.SOURCE); + super(upstreamId, service, curatorFramework, ZkStoreType.SOURCE); } @Override - public String getMetricId() { - return metricId; + public String getUpstreamId() { + return upstreamId; } @Override @@ -90,8 +90,7 @@ private Optional>> checkForUpdateOnZookeeper(D deserializer) List> nodes = new ArrayList<>(children.size()); log.debug("Found {} nodes for [{}]", children.size(), serviceName); if(children.isEmpty()){ - MetricRecorder.recordNullOrEmptyListNodeResponse(DataStoreType.ZK, metricId); - MetricRecorder.recordNullOrEmptyListNodeResponse(DataStoreType.ZK, metricId, serviceName); + recordNullOrEmptyResponse(serviceName); } for (val child : children) { byte[] data = readChild(serviceName, parentPath, child).orElse(null); @@ -104,15 +103,14 @@ private Optional>> checkForUpdateOnZookeeper(D deserializer) return Optional.of(nodes); } catch (NoNodeException e) { - MetricRecorder.recordNullOrEmptyListNodeResponse(DataStoreType.ZK, metricId); - MetricRecorder.recordNullOrEmptyListNodeResponse(DataStoreType.ZK, metricId, serviceName); + recordNullOrEmptyResponse(serviceName); log.error( "No ZK container node found for service: {}. Will return empty list for now. Please doublecheck service name", service.getServiceName()); return Optional.of(Collections.emptyList()); } catch (Exception e) { - MetricRecorder.recordZookeeperReadUnknownFailure(DataStoreType.ZK, metricId, LIST_NODES, e.getClass().getSimpleName()); + MetricRecorder.recordZookeeperReadUnknownFailure(DataStoreType.ZK, upstreamId, LIST_NODES, e.getClass().getSimpleName()); log.error("Error getting node data from zookeeper: ", e); throw new ZkCommunicationException("Error getting node data from zookeeper: exception %s , message: %s" .formatted(e.getClass().getSimpleName(), e.getMessage())); @@ -123,8 +121,8 @@ private > ServiceNode parseServiceNode try { return deserializer.deserialize(data); } catch (Exception e) { - MetricRecorder.recordListNodesParseFailure(DataStoreType.ZK, metricId); - MetricRecorder.recordListNodesParseFailure(DataStoreType.ZK, metricId, serviceName); + MetricRecorder.recordListNodesParseFailure(DataStoreType.ZK, upstreamId); + MetricRecorder.recordListNodesParseFailure(DataStoreType.ZK, upstreamId, serviceName); log.error("Error deserializing node data : {} for service name: {} ", new String(data), serviceName, e); throw e; } @@ -136,21 +134,24 @@ private Optional readChild(String serviceName, String parentPath, String return Optional.ofNullable(curatorFramework.getData().forPath(path)); } catch (KeeperException.NoNodeException e) { - MetricRecorder.recordNullOrEmptyListNodeResponse(DataStoreType.ZK, metricId); - MetricRecorder.recordNullOrEmptyListNodeResponse(DataStoreType.ZK, metricId, serviceName); + recordNullOrEmptyResponse(serviceName); log.warn("Node not found for path {}", path); return Optional.empty(); } catch (KeeperException e) { - MetricRecorder.recordNullOrEmptyListNodeResponse(DataStoreType.ZK, metricId); - MetricRecorder.recordNullOrEmptyListNodeResponse(DataStoreType.ZK, metricId, serviceName); + recordNullOrEmptyResponse(serviceName); log.error("Could not get data for node: {}", path, e); return Optional.empty(); } catch (Exception e){ - MetricRecorder.recordZookeeperReadUnknownFailure(DataStoreType.ZK, metricId, LIST_NODES, e.getClass().getSimpleName()); + MetricRecorder.recordZookeeperReadUnknownFailure(DataStoreType.ZK, upstreamId, LIST_NODES, e.getClass().getSimpleName()); log.error("Could not read child for node: {}", path, e); throw e; } } + private void recordNullOrEmptyResponse(String serviceName) { + MetricRecorder.recordNullOrEmptyListNodeResponse(DataStoreType.ZK, upstreamId); + MetricRecorder.recordNullOrEmptyListNodeResponse(DataStoreType.ZK, upstreamId, serviceName); + } + } diff --git a/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinder/ZkSimpleShardedServiceFinderBuilder.java b/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinder/ZkSimpleShardedServiceFinderBuilder.java index 293fcb5d..5fd83ded 100644 --- a/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinder/ZkSimpleShardedServiceFinderBuilder.java +++ b/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinder/ZkSimpleShardedServiceFinderBuilder.java @@ -69,8 +69,8 @@ public SimpleShardedServiceFinder build() { } @Override - protected NodeDataSource> dataSource(String metricId, Service service) { - return new ZkNodeDataSource<>(metricId, service, curatorFramework); + protected NodeDataSource> dataSource(String upstreamId, Service service) { + return new ZkNodeDataSource<>(upstreamId, service, curatorFramework); } @Override diff --git a/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinder/ZkSimpleUnshardedServiceFinderBuilder.java b/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinder/ZkSimpleUnshardedServiceFinderBuilder.java index e4abc7c9..b2ca6d6e 100644 --- a/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinder/ZkSimpleUnshardedServiceFinderBuilder.java +++ b/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinder/ZkSimpleUnshardedServiceFinderBuilder.java @@ -69,8 +69,8 @@ public SimpleUnshardedServiceFinder build() { @Override protected NodeDataSource> dataSource( - String metricId, Service service) { - return new ZkNodeDataSource<>(metricId, service, curatorFramework); + String upstreamId, Service service) { + return new ZkNodeDataSource<>(upstreamId, service, curatorFramework); } diff --git a/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinderhub/ZKUnshardedServiceFinderFactory.java b/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinderhub/ZKUnshardedServiceFinderFactory.java index d46f9994..6e27a355 100644 --- a/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinderhub/ZKUnshardedServiceFinderFactory.java +++ b/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinderhub/ZKUnshardedServiceFinderFactory.java @@ -37,7 +37,7 @@ public class ZKUnshardedServiceFinderFactory implements ServiceFinderFactory< private final ZkNodeDataDeserializer deserializer; private final ShardSelector> shardSelector; private final ServiceNodeSelector nodeSelector; - private final String metricId; + private final String upstreamId; @Builder public ZKUnshardedServiceFinderFactory( @@ -48,7 +48,7 @@ public ZKUnshardedServiceFinderFactory( ZkNodeDataDeserializer deserializer, ShardSelector> shardSelector, ServiceNodeSelector nodeSelector, - String metricId) { + String upstreamId) { this.curatorFramework = curatorFramework; this.connectionString = connectionString; this.nodeRefreshIntervalMs = nodeRefreshIntervalMs; @@ -56,13 +56,13 @@ public ZKUnshardedServiceFinderFactory( this.deserializer = deserializer; this.shardSelector = shardSelector; this.nodeSelector = nodeSelector; - this.metricId = metricId; + this.upstreamId = upstreamId; } @Override public SimpleUnshardedServiceFinder buildFinder(Service service) { val finder = new ZkSimpleUnshardedServiceFinderBuilder() - .withMetricId(metricId) + .withUpstreamId(upstreamId) .withDeserializer(deserializer) .withNamespace(service.getNamespace()) .withServiceName(service.getServiceName()) diff --git a/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinderhub/ZkServiceDataSource.java b/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinderhub/ZkServiceDataSource.java index 4894ee10..8243e827 100644 --- a/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinderhub/ZkServiceDataSource.java +++ b/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinderhub/ZkServiceDataSource.java @@ -30,6 +30,8 @@ import java.util.Collection; import java.util.Collections; +import java.util.List; +import java.util.Set; import java.util.stream.Collectors; import static io.appform.ranger.core.util.MetricRecorder.*; @@ -41,17 +43,17 @@ @Slf4j public class ZkServiceDataSource implements ServiceDataSource { - private final String metricId; + private final String upstreamId; private final String namespace; private final String connectionString; private CuratorFramework curatorFramework; private boolean curatorProvided; - public ZkServiceDataSource(String metricId, + public ZkServiceDataSource(String upstreamId, String namespace, String connectionString, CuratorFramework curatorFramework){ - this.metricId = metricId; + this.upstreamId = upstreamId; this.namespace = namespace; this.connectionString = connectionString; this.curatorFramework = curatorFramework; @@ -63,26 +65,29 @@ public Collection services() { try { val children = curatorFramework.getChildren() .forPath(PathBuilder.REGISTERED_SERVICES_PATH); - - if(children == null || children.isEmpty()) { - MetricRecorder.recordNullOrEmptyServicesListResponse(DataStoreType.ZK, metricId); - log.warn("No services found for namespace: {} in zk data source with metric id: {}", namespace, metricId); - } - val result = null == children - ? Collections.emptySet() - : children.stream() - .map(child -> Service.builder().namespace(namespace).serviceName(child).build()) - .collect(Collectors.toSet()); - MetricRecorder.recordServicesFetchStatus(DataStoreType.ZK, metricId, SUCCESS); + val result = getServices(children); + MetricRecorder.recordServicesFetchStatus(DataStoreType.ZK, upstreamId, SUCCESS); return result; } catch (Exception e) { - MetricRecorder.recordZookeeperReadUnknownFailure(DataStoreType.ZK, metricId, SERVICES_LIST, e.getClass().getSimpleName()); - MetricRecorder.recordServicesFetchStatus(DataStoreType.ZK, metricId, FAILURE); + MetricRecorder.recordZookeeperReadUnknownFailure(DataStoreType.ZK, upstreamId, SERVICES_LIST, e.getClass().getSimpleName()); + MetricRecorder.recordServicesFetchStatus(DataStoreType.ZK, upstreamId, FAILURE); throw e; } } + private Set getServices(List children) { + if(children == null || children.isEmpty()) { + MetricRecorder.recordNullOrEmptyServicesListResponse(DataStoreType.ZK, upstreamId); + log.warn("No services found for namespace: {} in zk data source with metric id: {}", namespace, upstreamId); + return Collections.emptySet(); + } else { + return children.stream() + .map(child -> Service.builder().namespace(namespace).serviceName(child).build()) + .collect(Collectors.toSet()); + } + } + @Override public void start() { if(null == curatorFramework){ diff --git a/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinderhub/ZkShardedServiceFinderFactory.java b/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinderhub/ZkShardedServiceFinderFactory.java index e05faed8..6ed8020d 100644 --- a/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinderhub/ZkShardedServiceFinderFactory.java +++ b/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/servicefinderhub/ZkShardedServiceFinderFactory.java @@ -38,7 +38,7 @@ public class ZkShardedServiceFinderFactory implements ServiceFinderFactory deserializer; private final ShardSelector> shardSelector; private final ServiceNodeSelector nodeSelector; - private final String metricId; + private final String upstreamId; @Builder public ZkShardedServiceFinderFactory( @@ -49,7 +49,7 @@ public ZkShardedServiceFinderFactory( ZkNodeDataDeserializer deserializer, ShardSelector> shardSelector, ServiceNodeSelector nodeSelector, - String metricId) { + String upstreamId) { this.curatorFramework = curatorFramework; this.connectionString = connectionString; this.nodeRefreshIntervalMs = nodeRefreshIntervalMs; @@ -57,13 +57,13 @@ public ZkShardedServiceFinderFactory( this.deserializer = deserializer; this.shardSelector = shardSelector; this.nodeSelector = nodeSelector; - this.metricId = metricId; + this.upstreamId = upstreamId; } @Override public SimpleShardedServiceFinder buildFinder(Service service) { val finder = new ZkSimpleShardedServiceFinderBuilder() - .withMetricId(metricId) + .withUpstreamId(upstreamId) .withDeserializer(deserializer) .withNamespace(service.getNamespace()) .withServiceName(service.getServiceName()) diff --git a/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/serviceprovider/ZkNodeDataSink.java b/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/serviceprovider/ZkNodeDataSink.java index 1c351b52..320dba76 100644 --- a/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/serviceprovider/ZkNodeDataSink.java +++ b/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/serviceprovider/ZkNodeDataSink.java @@ -42,9 +42,9 @@ @Slf4j public class ZkNodeDataSink> extends ZkNodeDataStoreConnector implements NodeDataSink { public ZkNodeDataSink( - String metricId, Service service, + String upstreamId, Service service, CuratorFramework curatorFramework) { - super(metricId, service, curatorFramework, ZkStoreType.SINK); + super(upstreamId, service, curatorFramework, ZkStoreType.SINK); } @Override @@ -53,8 +53,8 @@ public DataStoreType getDataStoreType() { } @Override - public String getMetricId() { - return metricId; + public String getUpstreamId() { + return upstreamId; } @Override @@ -75,11 +75,11 @@ public void updateState(S serializer, ServiceNode serviceNode) { val serviceData = getSerializedData(service.getServiceName(), serializer, serviceNode); curatorFramework.setData().forPath(path, serviceData); } - MetricRecorder.recordNodeDataSinkUpdateStatus(DataStoreType.ZK, metricId, SUCCESS); + MetricRecorder.recordNodeDataSinkUpdateStatus(getDataStoreType(), upstreamId, SUCCESS); } catch (Exception e) { log.error("Error updating node data at path " + path, e); - MetricRecorder.recordNodeDataSinkUpdateStatus(DataStoreType.ZK, metricId, FAILURE); + MetricRecorder.recordNodeDataSinkUpdateStatus(getDataStoreType(), upstreamId, FAILURE); Exceptions.illegalState(e); } } @@ -88,7 +88,7 @@ private > byte[] getSerializedData(String s try { return serializer.serialize(serviceNode); } catch (Exception e) { - MetricRecorder.recordNodeDataSinkSerDeFailure(DataStoreType.ZK, metricId, MetricRecorder.SERIALIZATION, serviceName, e.getClass().getSimpleName()); + MetricRecorder.recordNodeDataSinkSerDeFailure(getDataStoreType(), upstreamId, MetricRecorder.SERIALIZATION, serviceName, e.getClass().getSimpleName()); throw e; } } @@ -110,7 +110,7 @@ private synchronized void createPath( log.warn("Node already exists.. Race condition?", e); } catch (Exception e) { - MetricRecorder.recordNodeDataSinkUnknownFailure(DataStoreType.ZK, metricId, service.getServiceName(), e.getClass().getSimpleName()); + MetricRecorder.recordNodeDataSinkUnknownFailure(getDataStoreType(), upstreamId, service.getServiceName(), e.getClass().getSimpleName()); val message = String.format( "Could not create node for %s after 60 retries (1 min). " + "This service will not be discoverable. Retry after some time.", service.getServiceName()); diff --git a/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/serviceprovider/ZkServiceProviderBuilder.java b/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/serviceprovider/ZkServiceProviderBuilder.java index cbbfaa25..157c7364 100644 --- a/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/serviceprovider/ZkServiceProviderBuilder.java +++ b/ranger-zookeeper/src/main/java/io/appform/ranger/zookeeper/serviceprovider/ZkServiceProviderBuilder.java @@ -61,7 +61,7 @@ public ServiceProvider> build() { } @Override - protected NodeDataSink> dataSink(String metricId, Service service) { - return new ZkNodeDataSink<>(metricId, service, curatorFramework); + protected NodeDataSink> dataSink(String upstreamId, Service service) { + return new ZkNodeDataSink<>(upstreamId, service, curatorFramework); } } diff --git a/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/healthservice/ServiceProviderIntegrationTest.java b/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/healthservice/ServiceProviderIntegrationTest.java index ea5b6990..741d2ddf 100644 --- a/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/healthservice/ServiceProviderIntegrationTest.java +++ b/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/healthservice/ServiceProviderIntegrationTest.java @@ -24,7 +24,6 @@ import io.appform.ranger.core.healthcheck.updater.HealthStatusHandler; import io.appform.ranger.core.healthcheck.updater.HealthUpdateHandler; import io.appform.ranger.core.healthcheck.updater.LastUpdatedHandler; -import io.appform.ranger.core.healthcheck.updater.StartupTimeHandler; import io.appform.ranger.core.healthservice.TimeEntity; import io.appform.ranger.core.healthservice.monitor.sample.RotationStatusMonitor; import io.appform.ranger.core.model.ServiceNode; @@ -70,7 +69,7 @@ public void startTestCluster() throws Exception { registerService("localhost-4", 9003, 2, anotherFile); serviceFinder = ServiceFinderBuilders.shardedFinderBuilder() - .withMetricId("test-metric") + .withUpstreamId("test-metric") .withConnectionString(testingCluster.getConnectString()) .withNamespace("test") .withServiceName("test-service") @@ -136,7 +135,7 @@ private void registerService(String host, int port, int shardId, File file) { final HealthUpdateHandler healthUpdateHandler = new LastUpdatedHandler() .setNext(new HealthStatusHandler()); val serviceProvider = ServiceProviderBuilders.shardedServiceProviderBuilder() - .withMetricId("test-metric") + .withUpstreamId("test-metric") .withConnectionString(testingCluster.getConnectString()) .withNamespace("test") .withServiceName("test-service") diff --git a/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/model/CustomShardSelectorTest.java b/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/model/CustomShardSelectorTest.java index 3e0c5c08..eaf3712e 100644 --- a/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/model/CustomShardSelectorTest.java +++ b/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/model/CustomShardSelectorTest.java @@ -100,7 +100,7 @@ public List> nodes(Predicate criteria, @Test void testBasicDiscovery() { val serviceFinder = ServiceFinderBuilders.shardedFinderBuilder() - .withMetricId("test-metric") + .withUpstreamId("test-metric") .withConnectionString(testingCluster.getConnectString()) .withNamespace("test") .withServiceName("test-service") @@ -136,7 +136,7 @@ private void registerService(String host, int port, int a, int b) { final HealthUpdateHandler healthUpdateHandler = new LastUpdatedHandler() .setNext(new HealthStatusHandler()); val serviceProvider = ServiceProviderBuilders.shardedServiceProviderBuilder() - .withMetricId("test-metric") + .withUpstreamId("test-metric") .withConnectionString(testingCluster.getConnectString()) .withNamespace("test") .withServiceName("test-service") diff --git a/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/model/ServiceNoProviderTest.java b/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/model/ServiceNoProviderTest.java index 3fd94327..d585a701 100644 --- a/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/model/ServiceNoProviderTest.java +++ b/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/model/ServiceNoProviderTest.java @@ -54,7 +54,7 @@ public void stopTestCluster() throws Exception { @Test void testBasicDiscovery() { val serviceFinder = ServiceFinderBuilders.shardedFinderBuilder() - .withMetricId("test-metric") + .withUpstreamId("test-metric") .withConnectionString(testingCluster.getConnectString()) .withNamespace("test") .withServiceName("test-service") @@ -80,7 +80,7 @@ void testBasicDiscovery() { @Test void testBasicDiscoveryRR() { val serviceFinder = ServiceFinderBuilders.shardedFinderBuilder() - .withMetricId("test-metric") + .withUpstreamId("test-metric") .withConnectionString(testingCluster.getConnectString()) .withNamespace("test") .withServiceName("test-service") diff --git a/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/model/ServiceProviderExtCuratorTest.java b/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/model/ServiceProviderExtCuratorTest.java index 4ac38bd4..876b8f3d 100644 --- a/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/model/ServiceProviderExtCuratorTest.java +++ b/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/model/ServiceProviderExtCuratorTest.java @@ -80,7 +80,7 @@ public void stopTestCluster() throws Exception { @Test void testBasicDiscovery() { val serviceFinder = ServiceFinderBuilders.shardedFinderBuilder() - .withMetricId("test-metric") + .withUpstreamId("test-metric") .withCuratorFramework(curatorFramework) .withNamespace("test") .withServiceName("test-service") @@ -123,7 +123,7 @@ private void registerService(String host, int port, int shardId) { final HealthUpdateHandler healthUpdateHandler = new LastUpdatedHandler() .setNext(new HealthStatusHandler()); val serviceProvider = ServiceProviderBuilders.shardedServiceProviderBuilder() - .withMetricId("test-metric") + .withUpstreamId("test-metric") .withCuratorFramework(curatorFramework) .withNamespace("test") .withServiceName("test-service") diff --git a/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/model/ServiceProviderHealthcheckTest.java b/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/model/ServiceProviderHealthcheckTest.java index 9384f9cd..3e1a6348 100644 --- a/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/model/ServiceProviderHealthcheckTest.java +++ b/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/model/ServiceProviderHealthcheckTest.java @@ -66,7 +66,7 @@ public void stopTestCluster() throws Exception { @Test void testBasicDiscovery() { val serviceFinder = ServiceFinderBuilders.shardedFinderBuilder() - .withMetricId("test-metric") + .withUpstreamId("test-metric") .withConnectionString(testingCluster.getConnectString()) .withNamespace("test") .withServiceName("test-service") @@ -142,7 +142,7 @@ public void start() { final HealthUpdateHandler healthUpdateHandler = new LastUpdatedHandler() .setNext(new HealthStatusHandler()); val serviceProvider = ServiceProviderBuilders.shardedServiceProviderBuilder() - .withMetricId("test-metric") + .withUpstreamId("test-metric") .withConnectionString(connectionString) .withNamespace("test") .withServiceName("test-service") diff --git a/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/model/ServiceProviderTest.java b/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/model/ServiceProviderTest.java index b23e1de7..2fb893ce 100644 --- a/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/model/ServiceProviderTest.java +++ b/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/model/ServiceProviderTest.java @@ -73,7 +73,7 @@ public void stopTestCluster() throws Exception { @Test void testBasicDiscovery() { val serviceFinder = ServiceFinderBuilders.shardedFinderBuilder() - .withMetricId("test-metric") + .withUpstreamId("test-metric") .withConnectionString(testingCluster.getConnectString()) .withNamespace("test") .withServiceName("test-service") @@ -115,7 +115,7 @@ void testBasicDiscovery() { void testBasicDiscoveryRR() { val serviceFinder = ServiceFinderBuilders.shardedFinderBuilder() - .withMetricId("test-metric") + .withUpstreamId("test-metric") .withConnectionString(testingCluster.getConnectString()) .withNamespace("test") .withServiceName("test-service") @@ -160,7 +160,7 @@ void testBasicDiscoveryRR() { void testVisibility() { val serviceFinder = ServiceFinderBuilders. shardedFinderBuilder() - .withMetricId("test-metric") + .withUpstreamId("test-metric") .withConnectionString(testingCluster.getConnectString()) .withNamespace("test") .withServiceName("test-service") @@ -189,7 +189,7 @@ private void registerService(String host, int port, int shardId) { final HealthUpdateHandler healthUpdateHandler = new LastUpdatedHandler() .setNext(new HealthStatusHandler<>()); val serviceProvider = ServiceProviderBuilders.shardedServiceProviderBuilder() - .withMetricId("test-metric") + .withUpstreamId("test-metric") .withConnectionString(testingCluster.getConnectString()) .withNamespace("test") .withServiceName("test-service") diff --git a/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/model/SimpleServiceProviderTest.java b/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/model/SimpleServiceProviderTest.java index 27fcddc3..7a564c15 100644 --- a/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/model/SimpleServiceProviderTest.java +++ b/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/model/SimpleServiceProviderTest.java @@ -75,7 +75,7 @@ public boolean equals(Object obj) { @Test void testBasicDiscovery() { val serviceFinder = ServiceFinderBuilders.unshardedFinderBuilder() - .withMetricId("test-metric") + .withUpstreamId("test-metric") .withConnectionString(testingCluster.getConnectString()) .withNamespace("test") .withServiceName("test-service") @@ -111,7 +111,7 @@ private void registerService(String host, int port) { final HealthUpdateHandler healthUpdateHandler = new LastUpdatedHandler() .setNext(new HealthStatusHandler()); val serviceProvider = ServiceProviderBuilders.unshardedServiceProviderBuilder() - .withMetricId("test-metric") + .withUpstreamId("test-metric") .withConnectionString(testingCluster.getConnectString()) .withNamespace("test") .withServiceName("test-service") diff --git a/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/servicehub/ServiceHubTest.java b/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/servicehub/ServiceHubTest.java index 6511f5aa..7b3d9289 100644 --- a/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/servicehub/ServiceHubTest.java +++ b/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/servicehub/ServiceHubTest.java @@ -93,7 +93,7 @@ void testHub() { final HealthUpdateHandler healthUpdateHandler = new LastUpdatedHandler() .setNext(new HealthStatusHandler<>()); val provider1 = ServiceProviderBuilders.shardedServiceProviderBuilder() - .withMetricId("test-metric") + .withUpstreamId("test-metric") .withHostname("localhost") .withPort(1080) .withNamespace(NAMESPACE) @@ -115,7 +115,7 @@ void testHub() { .withRefreshFrequencyMs(1000) .withServiceDataSource(new ZkServiceDataSource(null, "test", testingCluster.getConnectString(), curatorFramework)) .withServiceFinderFactory(ZkShardedServiceFinderFactory.builder() - .metricId("test-metric") + .upstreamId("test-metric") .curatorFramework(curatorFramework) .deserializer(this::read) .build()) diff --git a/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/serviceprovider/BaseServiceProviderBuilderTest.java b/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/serviceprovider/BaseServiceProviderBuilderTest.java index eaa17865..d0db7bc1 100644 --- a/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/serviceprovider/BaseServiceProviderBuilderTest.java +++ b/ranger-zookeeper/src/test/java/io/appform/ranger/zookeeper/serviceprovider/BaseServiceProviderBuilderTest.java @@ -63,7 +63,7 @@ void testServiceProviderBuilder() { .setNext(new HealthStatusHandler<>()); try { val serviceProvider = ServiceProviderBuilders.unshardedServiceProviderBuilder() - .withMetricId("test-metric") + .withUpstreamId("test-metric") .withConnectionString(testingCluster.getConnectString()) .withNamespace("test") .withServiceName("test-service") @@ -87,7 +87,7 @@ void testServiceProviderBuilder() { Assertions.assertTrue(exception instanceof IllegalArgumentException); val serviceProvider = ServiceProviderBuilders.unshardedServiceProviderBuilder() - .withMetricId("test-metric") + .withUpstreamId("test-metric") .withConnectionString(testingCluster.getConnectString()) .withNamespace("test") .withServiceName("test-service") From 8cb04275f5a941a582b62efbbba3dd4b1882b443 Mon Sep 17 00:00:00 2001 From: Jitendra Dhawan Date: Tue, 2 Jun 2026 21:08:07 +0530 Subject: [PATCH 23/24] Add node count metrics for service registry updates and stale data retention --- .../ServiceRegistryUpdater.java | 21 +- .../ranger/core/util/MetricRecorder.java | 20 +- ...RegistryUpdaterMetricsIntegrationTest.java | 210 ++++++++++++++++++ 3 files changed, 243 insertions(+), 8 deletions(-) diff --git a/ranger-core/src/main/java/io/appform/ranger/core/finder/serviceregistry/ServiceRegistryUpdater.java b/ranger-core/src/main/java/io/appform/ranger/core/finder/serviceregistry/ServiceRegistryUpdater.java index 63a1db87..4dff74f4 100644 --- a/ranger-core/src/main/java/io/appform/ranger/core/finder/serviceregistry/ServiceRegistryUpdater.java +++ b/ranger-core/src/main/java/io/appform/ranger/core/finder/serviceregistry/ServiceRegistryUpdater.java @@ -23,6 +23,7 @@ import io.appform.ranger.core.healthcheck.HealthcheckStatus; import io.appform.ranger.core.model.Deserializer; import io.appform.ranger.core.model.NodeDataSource; +import io.appform.ranger.core.model.ServiceNode; import io.appform.ranger.core.model.ServiceRegistry; import io.appform.ranger.core.signals.Signal; import io.appform.ranger.core.util.Exceptions; @@ -145,12 +146,17 @@ private void updateRegistry() throws InterruptedException { try { val nodeList = nodeDataSource.refresh(deserializer).orElse(null); if (null != nodeList) { + MetricRecorder.recordNodesFetchedCount(serviceRegistry.getService().getServiceName(), + nodeDataSource.getDataStoreType(), nodeDataSource.getUpstreamId(), nodeList.size()); log.debug("Updating nodeList of size: {} for [{}]", nodeList.size(), serviceRegistry.getService().getServiceName()); val livenessCheckMaxAge = nodeDataSource.healthcheckZombieCheckThresholdTime(serviceRegistry.getService()); //Remove all stale nodes before updating. This is done centrally to ensure some data sources //don't skip this check. Some control is still provided so that they can overload. - serviceRegistry.updateNodes(FinderUtils.filterValidNodes(serviceRegistry.getService(), nodeList, livenessCheckMaxAge)); + List> validNodes = FinderUtils.filterValidNodes(serviceRegistry.getService(), nodeList, livenessCheckMaxAge); + MetricRecorder.recordServiceRegistryUpdateNodeCount(serviceRegistry.getService().getServiceName(), + nodeDataSource.getDataStoreType(), nodeDataSource.getUpstreamId(), validNodes.size()); + serviceRegistry.updateNodes(validNodes); MetricRecorder.recordNodeDataRefreshSuccess(nodeDataSource.getDataStoreType(), nodeDataSource.getUpstreamId(), stopwatch.elapsed(TimeUnit.MILLISECONDS)); } @@ -176,13 +182,14 @@ private void updateRegistry() throws InterruptedException { log.warn("Node data source seems to be down. Keeping old list for {}." + " Will update timestamp to keep stale date relevant.", serviceRegistry.getService().getServiceName()); - serviceRegistry.updateNodes(serviceRegistry.nodeList() - .stream() - .filter(node -> HealthcheckStatus.healthy == node.getHealthcheckStatus()) - .map(node -> node.setLastUpdatedTimeStamp(currTime)) - .toList()); + val retainedNodes = serviceRegistry.nodeList() + .stream() + .filter(node -> HealthcheckStatus.healthy == node.getHealthcheckStatus()) + .map(node -> node.setLastUpdatedTimeStamp(currTime)) + .toList(); + serviceRegistry.updateNodes(retainedNodes); MetricRecorder.recordStaleDataRetained(serviceRegistry.getService().getServiceName(), - nodeDataSource.getDataStoreType(), nodeDataSource.getUpstreamId()); + nodeDataSource.getDataStoreType(), nodeDataSource.getUpstreamId(), retainedNodes.size()); } } diff --git a/ranger-core/src/main/java/io/appform/ranger/core/util/MetricRecorder.java b/ranger-core/src/main/java/io/appform/ranger/core/util/MetricRecorder.java index 2e827bb8..e434b461 100644 --- a/ranger-core/src/main/java/io/appform/ranger/core/util/MetricRecorder.java +++ b/ranger-core/src/main/java/io/appform/ranger/core/util/MetricRecorder.java @@ -36,6 +36,7 @@ public class MetricRecorder { public static final String STALE_DATA_RETAINED = "staleDataRetained"; public static final String ZK_READ = "zkRead"; public static final String HEALTH_CHECKER = "healthChecker"; + public static final String NODE_COUNT = "nodeCount"; private static MetricRegistry metricRegistry; @@ -75,12 +76,14 @@ public static void recordNodeDataRefreshFailure(String serviceName, DataStoreTyp } } - public static void recordStaleDataRetained(String serviceName, DataStoreType dataStoreType, String upstreamId) { + public static void recordStaleDataRetained(String serviceName, DataStoreType dataStoreType, String upstreamId, int size) { if (metricRegistry != null) { metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), DATA_SOURCE, upstreamId, STALE_DATA_RETAINED)).mark(); metricRegistry.meter(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), DATA_SOURCE, upstreamId, SERVICE_NAME, serviceName, STALE_DATA_RETAINED)).mark(); + metricRegistry.histogram(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), + DATA_SOURCE, upstreamId, SERVICE_NAME, serviceName, STALE_DATA_RETAINED, NODE_COUNT)).update(size); } } @@ -252,4 +255,19 @@ public static void recordServicesReturned(int services) { } } + public static void recordServiceRegistryUpdateNodeCount(String serviceName, DataStoreType dataStoreType, String upstreamId, int size) { + if (metricRegistry != null) { + metricRegistry.histogram(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), + DATA_SOURCE, upstreamId, "serviceRegistryUpdate", SERVICE_NAME, serviceName, NODE_COUNT)) + .update(size); + } + } + + public static void recordNodesFetchedCount(String serviceName, DataStoreType dataStoreType, String upstreamId, int size) { + if (metricRegistry != null) { + metricRegistry.histogram(MetricRegistry.name(PACKAGE_PREFIX, DATA_STORE_TYPE, dataStoreType.name(), + DATA_SOURCE, upstreamId, LIST_NODES, SERVICE_NAME, serviceName, NODE_COUNT)) + .update(size); + } + } } diff --git a/ranger-core/src/test/java/io/appform/ranger/core/finder/serviceregistry/ServiceRegistryUpdaterMetricsIntegrationTest.java b/ranger-core/src/test/java/io/appform/ranger/core/finder/serviceregistry/ServiceRegistryUpdaterMetricsIntegrationTest.java index 0298859b..2ad52c62 100644 --- a/ranger-core/src/test/java/io/appform/ranger/core/finder/serviceregistry/ServiceRegistryUpdaterMetricsIntegrationTest.java +++ b/ranger-core/src/test/java/io/appform/ranger/core/finder/serviceregistry/ServiceRegistryUpdaterMetricsIntegrationTest.java @@ -199,6 +199,216 @@ void testSuccessfulRefresh_zombieNodesFiltered_recordsZombieMetric() { assertTrue(timer.getCount() >= 1); } + @Test + void testSuccessfulRefresh_recordsNodesFetchedCount() { + val registry = new MapBasedServiceRegistry(TEST_SERVICE); + val nodes = List.of( + ServiceNode.builder() + .host("host1").port(8080) + .nodeData(TestNodeData.builder().shardId(1).build()) + .healthcheckStatus(HealthcheckStatus.healthy) + .lastUpdatedTimeStamp(System.currentTimeMillis()) + .build(), + ServiceNode.builder() + .host("host2").port(8081) + .nodeData(TestNodeData.builder().shardId(2).build()) + .healthcheckStatus(HealthcheckStatus.healthy) + .lastUpdatedTimeStamp(System.currentTimeMillis()) + .build() + ); + val dataSource = new TestNodeDataSource(METRIC_ID, DataStoreType.ZK, true, nodes, false); + val signal = new TestSignal(); + + updater = new ServiceRegistryUpdater<>(registry, dataSource, List.of(signal), new TestDeserializer()); + updater.start(); + awaitRefresh(registry); + + val histName = "io.appform.ranger.dataStoreType.ZK.dataSource." + METRIC_ID + + ".listNodes.serviceName." + TEST_SERVICE.getServiceName() + ".nodeCount"; + await() + .atMost(Duration.ofSeconds(5)) + .pollInterval(Duration.ofMillis(50)) + .untilAsserted(() -> { + val histogram = metricRegistry.getHistograms().get(histName); + assertNotNull(histogram, "listNodes nodeCount histogram should be recorded"); + assertTrue(histogram.getCount() >= 1, "Histogram should have at least one update"); + assertEquals(2, histogram.getSnapshot().getMax(), + "Fetched count should equal total nodes returned by data source (2)"); + }); + } + + @Test + void testSuccessfulRefresh_recordsServiceRegistryUpdateNodeCount() { + val registry = new MapBasedServiceRegistry(TEST_SERVICE); + val nodes = List.of( + ServiceNode.builder() + .host("host1").port(8080) + .nodeData(TestNodeData.builder().shardId(1).build()) + .healthcheckStatus(HealthcheckStatus.healthy) + .lastUpdatedTimeStamp(System.currentTimeMillis()) + .build(), + ServiceNode.builder() + .host("host2").port(8081) + .nodeData(TestNodeData.builder().shardId(2).build()) + .healthcheckStatus(HealthcheckStatus.healthy) + .lastUpdatedTimeStamp(System.currentTimeMillis()) + .build() + ); + val dataSource = new TestNodeDataSource(METRIC_ID, DataStoreType.ZK, true, nodes, false); + val signal = new TestSignal(); + + updater = new ServiceRegistryUpdater<>(registry, dataSource, List.of(signal), new TestDeserializer()); + updater.start(); + awaitRefresh(registry); + + val histName = "io.appform.ranger.dataStoreType.ZK.dataSource." + METRIC_ID + + ".serviceRegistryUpdate.serviceName." + TEST_SERVICE.getServiceName() + ".nodeCount"; + await() + .atMost(Duration.ofSeconds(5)) + .pollInterval(Duration.ofMillis(50)) + .untilAsserted(() -> { + val histogram = metricRegistry.getHistograms().get(histName); + assertNotNull(histogram, "serviceRegistryUpdate nodeCount histogram should be recorded"); + assertTrue(histogram.getCount() >= 1, "Histogram should have at least one update"); + assertEquals(2, histogram.getSnapshot().getMax(), + "Valid node count should equal 2 healthy, non-zombie nodes"); + }); + } + + @Test + void testFetchedCountExceedsValidCount_whenZombiesPresent() { + val registry = new MapBasedServiceRegistry(TEST_SERVICE); + // 1 healthy + 1 zombie (very old timestamp). Fetched = 2, valid = 1. + val nodes = List.of( + ServiceNode.builder() + .host("healthy-host").port(8080) + .nodeData(TestNodeData.builder().shardId(1).build()) + .healthcheckStatus(HealthcheckStatus.healthy) + .lastUpdatedTimeStamp(System.currentTimeMillis()) + .build(), + ServiceNode.builder() + .host("zombie-host").port(8081) + .nodeData(TestNodeData.builder().shardId(2).build()) + .healthcheckStatus(HealthcheckStatus.healthy) + .lastUpdatedTimeStamp(0L) // zombie + .build() + ); + val dataSource = new TestNodeDataSource(METRIC_ID, DataStoreType.HTTP, true, nodes, false); + val signal = new TestSignal(); + + updater = new ServiceRegistryUpdater<>(registry, dataSource, List.of(signal), new TestDeserializer()); + updater.start(); + awaitRefresh(registry); + + val fetchedHistName = "io.appform.ranger.dataStoreType.HTTP.dataSource." + METRIC_ID + + ".listNodes.serviceName." + TEST_SERVICE.getServiceName() + ".nodeCount"; + val validHistName = "io.appform.ranger.dataStoreType.HTTP.dataSource." + METRIC_ID + + ".serviceRegistryUpdate.serviceName." + TEST_SERVICE.getServiceName() + ".nodeCount"; + + await() + .atMost(Duration.ofSeconds(5)) + .pollInterval(Duration.ofMillis(50)) + .untilAsserted(() -> { + val fetchedHist = metricRegistry.getHistograms().get(fetchedHistName); + val validHist = metricRegistry.getHistograms().get(validHistName); + assertNotNull(fetchedHist, "Fetched count histogram should exist"); + assertNotNull(validHist, "Valid count histogram should exist"); + assertEquals(2, fetchedHist.getSnapshot().getMax(), + "Fetched count should be 2 (all nodes including zombie)"); + assertEquals(1, validHist.getSnapshot().getMax(), + "Valid count should be 1 (zombie filtered out)"); + }); + } + + @Test + void testInactiveDataSource_recordsStaleDataRetainedWithNodeCountHistogram() { + val registry = new MapBasedServiceRegistry(TEST_SERVICE); + val nodes = List.of( + ServiceNode.builder() + .host("host1").port(8080) + .nodeData(TestNodeData.builder().shardId(1).build()) + .healthcheckStatus(HealthcheckStatus.healthy) + .lastUpdatedTimeStamp(System.currentTimeMillis()) + .build(), + ServiceNode.builder() + .host("host2").port(8081) + .nodeData(TestNodeData.builder().shardId(2).build()) + .healthcheckStatus(HealthcheckStatus.healthy) + .lastUpdatedTimeStamp(System.currentTimeMillis()) + .build() + ); + val dataSource = new TestNodeDataSource(METRIC_ID, DataStoreType.ZK, true, nodes, false); + val signal = new TestSignal(); + + updater = new ServiceRegistryUpdater<>(registry, dataSource, List.of(signal), new TestDeserializer()); + updater.start(); + awaitRefresh(registry); + + // Deactivate and trigger stale path + dataSource.setActive(false); + signal.fire(); + + val staleNodeCountHistName = "io.appform.ranger.dataStoreType.ZK.dataSource." + METRIC_ID + + ".serviceName." + TEST_SERVICE.getServiceName() + ".staleDataRetained.nodeCount"; + await() + .atMost(Duration.ofSeconds(5)) + .pollInterval(Duration.ofMillis(50)) + .untilAsserted(() -> { + val histogram = metricRegistry.getHistograms().get(staleNodeCountHistName); + assertNotNull(histogram, "staleDataRetained nodeCount histogram should be recorded"); + assertTrue(histogram.getCount() >= 1, "Histogram should have at least one update"); + // The stale path retains healthy nodes only; 2 healthy nodes were present + assertTrue(histogram.getSnapshot().getMax() >= 0, + "Histogram should record a non-negative node count"); + }); + } + + @Test + void testCallFailure_recordsStaleDataRetainedWithNodeCountHistogram() { + val registry = new MapBasedServiceRegistry(TEST_SERVICE); + val signal = new TestSignal(); + + // First start with a healthy node so the registry has something to retain + val initialNodes = List.of( + ServiceNode.builder() + .host("host1").port(8080) + .nodeData(TestNodeData.builder().shardId(1).build()) + .healthcheckStatus(HealthcheckStatus.healthy) + .lastUpdatedTimeStamp(System.currentTimeMillis()) + .build() + ); + val dataSource = new TestNodeDataSource(METRIC_ID, DataStoreType.HTTP, true, initialNodes, false); + + updater = new ServiceRegistryUpdater<>(registry, dataSource, List.of(signal), new TestDeserializer()); + updater.start(); + awaitRefresh(registry); + + // Now trigger a call failure (while still active, but throws on refresh) + val failingDataSource = new TestNodeDataSource(METRIC_ID, DataStoreType.HTTP, true, null, true); + // Create a new updater with the failing data source and trigger + updater.stop(); + val signal2 = new TestSignal(); + val updater2 = new ServiceRegistryUpdater<>(registry, failingDataSource, List.of(signal2), new TestDeserializer()); + try { + updater2.start(); + } catch (Exception ignored) { + // May throw on initial update failure + } + + val staleNodeCountHistName = "io.appform.ranger.dataStoreType.HTTP.dataSource." + METRIC_ID + + ".serviceName." + TEST_SERVICE.getServiceName() + ".staleDataRetained.nodeCount"; + await() + .atMost(Duration.ofSeconds(5)) + .pollInterval(Duration.ofMillis(50)) + .untilAsserted(() -> { + val histogram = metricRegistry.getHistograms().get(staleNodeCountHistName); + assertNotNull(histogram, "staleDataRetained nodeCount histogram should be recorded on call failure"); + assertTrue(histogram.getCount() >= 1, "Histogram should have at least one update"); + }); + + updater2.stop(); + } + @Test void testRefreshReturnsNull_noSuccessOrFailureTimer_butStaleRetained() { val registry = new MapBasedServiceRegistry(TEST_SERVICE); From 44e92ba745221367852dfa85ce0be4642920838a Mon Sep 17 00:00:00 2001 From: Jitendra Dhawan Date: Tue, 2 Jun 2026 21:08:51 +0530 Subject: [PATCH 24/24] Add unit tests for MetricRecorder methods including node count metrics --- .../ranger/core/util/MetricRecorderTest.java | 276 ++++++++++++++++++ 1 file changed, 276 insertions(+) create mode 100644 ranger-core/src/test/java/io/appform/ranger/core/util/MetricRecorderTest.java diff --git a/ranger-core/src/test/java/io/appform/ranger/core/util/MetricRecorderTest.java b/ranger-core/src/test/java/io/appform/ranger/core/util/MetricRecorderTest.java new file mode 100644 index 00000000..ebc0f1e1 --- /dev/null +++ b/ranger-core/src/test/java/io/appform/ranger/core/util/MetricRecorderTest.java @@ -0,0 +1,276 @@ +/* + * Copyright 2024 Authors, Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.appform.ranger.core.util; + +import com.codahale.metrics.MetricRegistry; +import io.appform.ranger.core.model.DataStoreType; +import lombok.val; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Unit tests for the new and modified methods introduced in MetricRecorder: + *
    + *
  • {@link MetricRecorder#recordStaleDataRetained} – now also records a nodeCount histogram
    • + *
  • {@link MetricRecorder#recordServiceRegistryUpdateNodeCount} – new method
    • + *
  • {@link MetricRecorder#recordNodesFetchedCount} – new method
    • + *
+ */ +class MetricRecorderTest { + + private static final String SERVICE_NAME = "test-service"; + private static final String UPSTREAM_ID = "test-upstream"; + private static final String PACKAGE_PREFIX = "io.appform.ranger"; + + private MetricRegistry metricRegistry; + + @BeforeEach + void setUp() { + metricRegistry = new MetricRegistry(); + MetricRecorder.initialize(metricRegistry); + } + + @AfterEach + void tearDown() { + // Reset the static registry so other test classes are not affected + MetricRecorder.initialize(null); + } + + // ───────────────────────────────────────────────────────────────────────── + // recordStaleDataRetained (modified: size parameter + histogram) + // ───────────────────────────────────────────────────────────────────────── + + @Test + void recordStaleDataRetained_marksGlobalMeter() { + MetricRecorder.recordStaleDataRetained(SERVICE_NAME, DataStoreType.ZK, UPSTREAM_ID, 3); + + val meterName = PACKAGE_PREFIX + ".dataStoreType.ZK.dataSource." + UPSTREAM_ID + ".staleDataRetained"; + val meter = metricRegistry.getMeters().get(meterName); + assertNotNull(meter, "Global staleDataRetained meter should be created"); + assertEquals(1, meter.getCount(), "Global meter should be marked once"); + } + + @Test + void recordStaleDataRetained_marksServiceNameMeter() { + MetricRecorder.recordStaleDataRetained(SERVICE_NAME, DataStoreType.HTTP, UPSTREAM_ID, 5); + + val meterName = PACKAGE_PREFIX + ".dataStoreType.HTTP.dataSource." + UPSTREAM_ID + + ".serviceName." + SERVICE_NAME + ".staleDataRetained"; + val meter = metricRegistry.getMeters().get(meterName); + assertNotNull(meter, "Per-service staleDataRetained meter should be created"); + assertEquals(1, meter.getCount(), "Per-service meter should be marked once"); + } + + @Test + void recordStaleDataRetained_updatesNodeCountHistogram() { + MetricRecorder.recordStaleDataRetained(SERVICE_NAME, DataStoreType.DROVE, UPSTREAM_ID, 7); + + val histName = PACKAGE_PREFIX + ".dataStoreType.DROVE.dataSource." + UPSTREAM_ID + + ".serviceName." + SERVICE_NAME + ".staleDataRetained.nodeCount"; + val histogram = metricRegistry.getHistograms().get(histName); + assertNotNull(histogram, "staleDataRetained nodeCount histogram should be created"); + assertEquals(1, histogram.getCount(), "Histogram should have one update"); + assertEquals(7, histogram.getSnapshot().getMax(), "Histogram max should equal size passed (7)"); + } + + @Test + void recordStaleDataRetained_zeroSize_histogramUpdated() { + MetricRecorder.recordStaleDataRetained(SERVICE_NAME, DataStoreType.ZK, UPSTREAM_ID, 0); + + val histName = PACKAGE_PREFIX + ".dataStoreType.ZK.dataSource." + UPSTREAM_ID + + ".serviceName." + SERVICE_NAME + ".staleDataRetained.nodeCount"; + val histogram = metricRegistry.getHistograms().get(histName); + assertNotNull(histogram); + assertEquals(0, histogram.getSnapshot().getMax(), "Histogram max should be 0 when size is 0"); + } + + @Test + void recordStaleDataRetained_noRegistry_doesNotThrow() { + MetricRecorder.initialize(null); + // Should be a safe no-op + assertDoesNotThrow( + () -> MetricRecorder.recordStaleDataRetained(SERVICE_NAME, DataStoreType.ZK, UPSTREAM_ID, 3), + "recordStaleDataRetained should not throw when registry is not initialized" + ); + } + + @Test + void recordStaleDataRetained_multipleCalls_metersAccumulate() { + MetricRecorder.recordStaleDataRetained(SERVICE_NAME, DataStoreType.ZK, UPSTREAM_ID, 2); + MetricRecorder.recordStaleDataRetained(SERVICE_NAME, DataStoreType.ZK, UPSTREAM_ID, 4); + + val globalMeterName = PACKAGE_PREFIX + ".dataStoreType.ZK.dataSource." + UPSTREAM_ID + ".staleDataRetained"; + val globalMeter = metricRegistry.getMeters().get(globalMeterName); + assertEquals(2, globalMeter.getCount(), "Global meter should accumulate 2 marks"); + + val histName = PACKAGE_PREFIX + ".dataStoreType.ZK.dataSource." + UPSTREAM_ID + + ".serviceName." + SERVICE_NAME + ".staleDataRetained.nodeCount"; + val histogram = metricRegistry.getHistograms().get(histName); + assertEquals(2, histogram.getCount(), "Histogram should have 2 updates"); + assertEquals(4, histogram.getSnapshot().getMax(), "Histogram max should be 4 (latest max)"); + } + + // ───────────────────────────────────────────────────────────────────────── + // recordServiceRegistryUpdateNodeCount (new method) + // ───────────────────────────────────────────────────────────────────────── + + @Test + void recordServiceRegistryUpdateNodeCount_createsHistogramWithCorrectName() { + MetricRecorder.recordServiceRegistryUpdateNodeCount(SERVICE_NAME, DataStoreType.ZK, UPSTREAM_ID, 10); + + val histName = PACKAGE_PREFIX + ".dataStoreType.ZK.dataSource." + UPSTREAM_ID + + ".serviceRegistryUpdate.serviceName." + SERVICE_NAME + ".nodeCount"; + val histogram = metricRegistry.getHistograms().get(histName); + assertNotNull(histogram, "serviceRegistryUpdate nodeCount histogram should be created"); + } + + @Test + void recordServiceRegistryUpdateNodeCount_updatesHistogramWithSize() { + MetricRecorder.recordServiceRegistryUpdateNodeCount(SERVICE_NAME, DataStoreType.HTTP, UPSTREAM_ID, 15); + + val histName = PACKAGE_PREFIX + ".dataStoreType.HTTP.dataSource." + UPSTREAM_ID + + ".serviceRegistryUpdate.serviceName." + SERVICE_NAME + ".nodeCount"; + val histogram = metricRegistry.getHistograms().get(histName); + assertNotNull(histogram); + assertEquals(1, histogram.getCount(), "Histogram should have one update"); + assertEquals(15, histogram.getSnapshot().getMax(), "Histogram max should equal size passed (15)"); + } + + @Test + void recordServiceRegistryUpdateNodeCount_zeroNodes_histogramUpdated() { + MetricRecorder.recordServiceRegistryUpdateNodeCount(SERVICE_NAME, DataStoreType.ZK, UPSTREAM_ID, 0); + + val histName = PACKAGE_PREFIX + ".dataStoreType.ZK.dataSource." + UPSTREAM_ID + + ".serviceRegistryUpdate.serviceName." + SERVICE_NAME + ".nodeCount"; + val histogram = metricRegistry.getHistograms().get(histName); + assertNotNull(histogram); + assertEquals(0, histogram.getSnapshot().getMax(), "Histogram max should be 0 when no valid nodes"); + } + + @Test + void recordServiceRegistryUpdateNodeCount_noRegistry_doesNotThrow() { + MetricRecorder.initialize(null); + assertDoesNotThrow( + () -> MetricRecorder.recordServiceRegistryUpdateNodeCount(SERVICE_NAME, DataStoreType.ZK, UPSTREAM_ID, 5), + "recordServiceRegistryUpdateNodeCount should not throw when registry is not initialized" + ); + } + + @Test + void recordServiceRegistryUpdateNodeCount_multipleCalls_histogramAccumulates() { + MetricRecorder.recordServiceRegistryUpdateNodeCount(SERVICE_NAME, DataStoreType.DROVE, UPSTREAM_ID, 3); + MetricRecorder.recordServiceRegistryUpdateNodeCount(SERVICE_NAME, DataStoreType.DROVE, UPSTREAM_ID, 8); + + val histName = PACKAGE_PREFIX + ".dataStoreType.DROVE.dataSource." + UPSTREAM_ID + + ".serviceRegistryUpdate.serviceName." + SERVICE_NAME + ".nodeCount"; + val histogram = metricRegistry.getHistograms().get(histName); + assertEquals(2, histogram.getCount(), "Histogram should have 2 updates"); + assertEquals(8, histogram.getSnapshot().getMax(), "Histogram max should be 8"); + } + + @Test + void recordServiceRegistryUpdateNodeCount_differentDataStoreTypes_separateHistograms() { + MetricRecorder.recordServiceRegistryUpdateNodeCount(SERVICE_NAME, DataStoreType.ZK, UPSTREAM_ID, 5); + MetricRecorder.recordServiceRegistryUpdateNodeCount(SERVICE_NAME, DataStoreType.HTTP, UPSTREAM_ID, 10); + + val zkHistName = PACKAGE_PREFIX + ".dataStoreType.ZK.dataSource." + UPSTREAM_ID + + ".serviceRegistryUpdate.serviceName." + SERVICE_NAME + ".nodeCount"; + val httpHistName = PACKAGE_PREFIX + ".dataStoreType.HTTP.dataSource." + UPSTREAM_ID + + ".serviceRegistryUpdate.serviceName." + SERVICE_NAME + ".nodeCount"; + + assertNotNull(metricRegistry.getHistograms().get(zkHistName), "ZK histogram should exist"); + assertNotNull(metricRegistry.getHistograms().get(httpHistName), "HTTP histogram should exist"); + assertEquals(5, metricRegistry.getHistograms().get(zkHistName).getSnapshot().getMax()); + assertEquals(10, metricRegistry.getHistograms().get(httpHistName).getSnapshot().getMax()); + } + + // ───────────────────────────────────────────────────────────────────────── + // recordNodesFetchedCount (new method) + // ───────────────────────────────────────────────────────────────────────── + + @Test + void recordNodesFetchedCount_createsHistogramWithCorrectName() { + MetricRecorder.recordNodesFetchedCount(SERVICE_NAME, DataStoreType.ZK, UPSTREAM_ID, 6); + + val histName = PACKAGE_PREFIX + ".dataStoreType.ZK.dataSource." + UPSTREAM_ID + + ".listNodes.serviceName." + SERVICE_NAME + ".nodeCount"; + val histogram = metricRegistry.getHistograms().get(histName); + assertNotNull(histogram, "listNodes nodeCount histogram should be created"); + } + + @Test + void recordNodesFetchedCount_updatesHistogramWithSize() { + MetricRecorder.recordNodesFetchedCount(SERVICE_NAME, DataStoreType.HTTP, UPSTREAM_ID, 20); + + val histName = PACKAGE_PREFIX + ".dataStoreType.HTTP.dataSource." + UPSTREAM_ID + + ".listNodes.serviceName." + SERVICE_NAME + ".nodeCount"; + val histogram = metricRegistry.getHistograms().get(histName); + assertNotNull(histogram); + assertEquals(1, histogram.getCount(), "Histogram should have one update"); + assertEquals(20, histogram.getSnapshot().getMax(), "Histogram max should equal size passed (20)"); + } + + @Test + void recordNodesFetchedCount_zeroNodes_histogramUpdated() { + MetricRecorder.recordNodesFetchedCount(SERVICE_NAME, DataStoreType.ZK, UPSTREAM_ID, 0); + + val histName = PACKAGE_PREFIX + ".dataStoreType.ZK.dataSource." + UPSTREAM_ID + + ".listNodes.serviceName." + SERVICE_NAME + ".nodeCount"; + val histogram = metricRegistry.getHistograms().get(histName); + assertNotNull(histogram); + assertEquals(0, histogram.getSnapshot().getMax(), "Histogram max should be 0 when no nodes fetched"); + } + + @Test + void recordNodesFetchedCount_noRegistry_doesNotThrow() { + MetricRecorder.initialize(null); + assertDoesNotThrow( + () -> MetricRecorder.recordNodesFetchedCount(SERVICE_NAME, DataStoreType.ZK, UPSTREAM_ID, 5), + "recordNodesFetchedCount should not throw when registry is not initialized" + ); + } + + @Test + void recordNodesFetchedCount_multipleCalls_histogramAccumulates() { + MetricRecorder.recordNodesFetchedCount(SERVICE_NAME, DataStoreType.ZK, UPSTREAM_ID, 4); + MetricRecorder.recordNodesFetchedCount(SERVICE_NAME, DataStoreType.ZK, UPSTREAM_ID, 9); + + val histName = PACKAGE_PREFIX + ".dataStoreType.ZK.dataSource." + UPSTREAM_ID + + ".listNodes.serviceName." + SERVICE_NAME + ".nodeCount"; + val histogram = metricRegistry.getHistograms().get(histName); + assertEquals(2, histogram.getCount(), "Histogram should have 2 updates"); + assertEquals(9, histogram.getSnapshot().getMax(), "Histogram max should be 9"); + } + + @Test + void recordNodesFetchedCount_differentUpstreamIds_separateHistograms() { + MetricRecorder.recordNodesFetchedCount(SERVICE_NAME, DataStoreType.ZK, "upstream-a", 3); + MetricRecorder.recordNodesFetchedCount(SERVICE_NAME, DataStoreType.ZK, "upstream-b", 7); + + val histNameA = PACKAGE_PREFIX + ".dataStoreType.ZK.dataSource.upstream-a" + + ".listNodes.serviceName." + SERVICE_NAME + ".nodeCount"; + val histNameB = PACKAGE_PREFIX + ".dataStoreType.ZK.dataSource.upstream-b" + + ".listNodes.serviceName." + SERVICE_NAME + ".nodeCount"; + + assertNotNull(metricRegistry.getHistograms().get(histNameA), "Histogram for upstream-a should exist"); + assertNotNull(metricRegistry.getHistograms().get(histNameB), "Histogram for upstream-b should exist"); + assertEquals(3, metricRegistry.getHistograms().get(histNameA).getSnapshot().getMax()); + assertEquals(7, metricRegistry.getHistograms().get(histNameB).getSnapshot().getMax()); + } +}