From 4fcfe3627390f250607db868bbe5dd43f64ab5a3 Mon Sep 17 00:00:00 2001 From: Koushik R Date: Mon, 18 Oct 2021 15:01:19 +0530 Subject: [PATCH 01/69] 0) Renamed UnshardedClusterInfo into UnshardedFinder and introduced a ListBasedServiceRegistry along with the MapBased one 1) Added a criteria interface in the ShardSelector to keep it seperate from the T in ServiceNode and trickled the changes upstream to whereever ShardSelector is getting built. 2) Added test cases to the new criteria interface 3) Added Http serivceNodeProvider and serviceFinders, along with the HttpServiceFinderHub 4) Added tests for Http interfaces as well. 5) Added license to the missing files 6) Fixed sonar issues --- pom.xml | 11 +- .../core/finder/BaseServiceFinderBuilder.java | 22 +-- .../ranger/core/finder/ServiceFinder.java | 17 +-- .../SimpleShardedServiceFinder.java | 13 +- .../SimpleShardedServiceFinderBuilder.java | 24 ++-- ...java => SimpleUnshardedServiceFinder.java} | 21 +-- .../SimpleUnshardedServiceFinderBuilder.java | 37 +++++ .../RandomServiceNodeSelector.java | 5 +- .../RoundRobinServiceNodeSelector.java | 5 +- .../ListBasedServiceRegistry.java} | 19 +-- .../MapBasedServiceRegistry.java | 24 ++-- .../ServiceRegistryUpdater.java | 29 ++-- .../signal/ScheduledRegistryUpdateSignal.java | 36 +++++ .../shardselector/ListShardSelector.java | 37 +++++ .../shardselector/MatchingShardSelector.java | 40 ++++++ .../NoopShardSelector.java} | 16 +-- .../ScheduledRegistryUpdateSignal.java | 21 --- .../unsharded/UnshardedFinderBuilder.java | 37 ----- .../core/finderhub/ServiceDataSource.java | 15 ++ .../core/finderhub/ServiceFinderFactory.java | 21 ++- .../core/finderhub/ServiceFinderHub.java | 35 +++-- .../finderhub}/ServiceFinderHubBuilder.java | 53 ++++--- .../core/finderhub/StaticDataSource.java | 49 +++++++ .../core/healthcheck/HealthChecker.java | 3 +- .../ranger/core/healthcheck/Healthcheck.java | 10 +- .../core/healthcheck/HealthcheckResult.java | 15 ++ .../core/healthcheck/HealthcheckStatus.java | 9 +- .../ranger/core/healthcheck/Healthchecks.java | 15 ++ .../core/healthservice/HealthService.java | 10 +- .../ServiceHealthAggregator.java | 8 +- .../ranger/core/healthservice/TimeEntity.java | 11 +- .../monitor/IsolatedHealthMonitor.java | 12 +- .../core/healthservice/monitor/Monitor.java | 10 +- .../core/healthservice/monitor/Monitors.java | 11 +- .../monitor/RollingWindowHealthQueue.java | 10 +- .../monitor/sample/CountMonitor.java | 13 +- .../monitor/sample/DiskSpaceMonitor.java | 11 +- .../monitor/sample/PingCheckMonitor.java | 11 +- .../monitor/sample/RotationStatusMonitor.java | 23 ++- .../Criteria.java} | 22 ++- .../ranger/core/model/Deserializer.java | 9 +- .../ranger/core/model/NodeDataSink.java | 15 ++ .../ranger/core/model/NodeDataSource.java | 15 ++ .../core/model/NodeDataStoreConnector.java | 15 ++ .../ranger/core/model/Serializer.java | 8 +- .../flipkart/ranger/core/model/Service.java | 15 +- .../ranger/core/model/ServiceNode.java | 4 +- .../core/model/ServiceNodeSelector.java | 3 +- .../ranger/core/model/ServiceRegistry.java | 11 +- .../ranger/core/model/ShardSelector.java | 15 +- .../BaseServiceProviderBuilder.java | 35 +++-- .../core/serviceprovider/ServiceProvider.java | 16 ++- .../core/signals/ExternalTriggeredSignal.java | 15 ++ .../ranger/core/signals/ScheduledSignal.java | 15 ++ .../flipkart/ranger/core/signals/Signal.java | 15 ++ .../flipkart/ranger/core/util/Exceptions.java | 15 ++ .../ranger/core/util/FinderUtils.java | 15 ++ .../com/flipkart/ranger/core/TestUtils.java | 29 ++++ .../core/finder/SimpleShardFinderTest.java | 54 +++++++ .../finder/UnshardedClusterFinderTest.java | 53 +++++++ .../RoundRobinServiceNodeSelectorTest.java | 41 ++++++ .../MapBasedServiceRegistryTest.java | 40 ++++++ .../shardselector/ListShardSelectorTest.java | 39 +++++ .../MatchingShardSelectorTest.java | 46 ++++++ .../serviceprovider/ServiceProviderTest.java | 136 ++++++++++++++++++ .../ranger/core/units/TestNodeData.java | 47 ++++++ .../ranger/core/utils/CriteriaUtils.java | 26 ++++ .../ranger/core/utils/RegistryTestUtils.java | 50 +++++++ .../flipkart/ranger/core/utils/TestUtils.java | 14 -- .../http/HttpServiceFinderBuilders.java | 35 +++++ .../http/HttpServiceProviderBuilders.java | 29 ++++ .../http/HttpShardedServiceFinderBuilder.java | 32 ----- .../HttpNodeDataStoreConnector.java | 27 ++-- .../ranger/http/config/HttpClientConfig.java | 24 +++- .../http/model/ServiceDataSourceResponse.java | 31 ++++ .../http/model/ServiceNodesRequest.java | 18 --- .../http/model/ServiceNodesResponse.java | 24 +++- .../model/ServiceRegistrationResponse.java | 32 +++++ .../serde/HTTPResponseDataDeserializer.java | 21 ++- .../http/serde/HttpRequestDataSerializer.java | 24 ++-- .../JacksonHTTPResponseDataDeserializer.java | 33 ----- .../HttpNodeDataSource.java | 61 +++++--- .../HttpShardedServiceFinderBuilder.java | 55 +++++++ .../HttpUnshardedServiceFinderBuilider.java | 54 +++++++ .../HttpServiceDataSource.java | 89 ++++++++++++ .../HttpServiceFinderHubBuilder.java | 36 +++++ .../HttpShardedServiceFinderFactory.java | 75 ++++++++++ .../HttpUnshardedServiceFinderFactory.java | 75 ++++++++++ .../serviceprovider/HttpNodeDataSink.java | 98 +++++++++++++ .../HttpShardedServiceProviderBuilder.java | 52 +++++++ .../flipkart/ranger/http/ResourceHelper.java | 44 ++++++ .../HttpNodeDataStoreConnectorTest.java | 36 +++++ .../http/config/HttpClientConfigTest.java | 32 +++++ .../http/model/ServiceNodeResponseTest.java | 45 ++++++ .../ServiceRegistrationResponseTest.java | 30 ++++ .../HttpShardedServiceFinderBuilderTest.java | 49 ++++--- .../HttpServiceDataSourceTest.java | 85 +++++++++++ ...HttpShardedServiceProviderBuilderTest.java | 87 +++++++++++ .../resources/fixtures/httpClientConfig.json | 8 ++ .../fixtures/serviceNodesResponse.json | 23 +++ .../resources/fixtures/serviceResponse.json | 3 + .../zookeeper/ServiceFinderBuilders.java | 20 +-- .../zookeeper/ServiceProviderBuilders.java | 13 +- .../ZkNodeDataStoreConnector.java | 17 ++- .../serde/ZkNodeDataDeserializer.java | 15 ++ .../zookeeper/serde/ZkNodeDataSerializer.java | 15 ++ .../ZkNodeDataSource.java | 18 ++- .../ZkSimpleShardedServiceFinderBuilder.java | 32 +++-- ...kSimpleUnshardedServiceFinderBuilder.java} | 39 +++-- .../ZkWatcherRegistryUpdateSignal.java | 19 ++- .../ZKUnshardedServiceFinderFactory.java | 78 ++++++++++ .../servicefinderhub/ZkServiceDataSource.java | 86 +++++++++++ .../ZkServiceFinderHubBuilder.java | 29 +++- .../ZkShardedServiceFinderFactory.java | 33 +++-- .../ZkNodeDataSink.java | 18 ++- .../ZkServiceProviderBuilder.java | 17 ++- .../ranger/zookeeper/util/PathBuilder.java | 8 +- .../zookeeper/zk/ZkServiceDataSource.java | 50 ------- .../ServiceHealthAggregatorTest.java | 13 +- .../ServiceProviderIntegrationTest.java | 31 ++-- .../monitor/RollingWindowHealthQueueTest.java | 11 +- .../monitor/sample/DiskSpaceMonitorTest.java | 10 +- .../monitor/sample/PingCheckMonitorTest.java | 11 +- .../sample/RotationStatusMonitorTest.java | 11 +- .../model/CustomShardSelectorTest.java | 33 +++-- .../model/ServiceNoProviderTest.java | 23 +-- .../model/ServiceProviderExtCuratorTest.java | 29 ++-- .../model/ServiceProviderHealthcheckTest.java | 25 ++-- .../zookeeper/model/ServiceProviderTest.java | 42 +++--- .../model/SimpleServiceProviderTest.java | 37 +++-- .../zookeeper/servicehub/ServiceHubTest.java | 38 +++-- .../BaseServiceProviderBuilderTest.java | 11 +- 132 files changed, 3065 insertions(+), 736 deletions(-) rename ranger-core/src/main/java/com/flipkart/ranger/core/finder/{sharded => }/SimpleShardedServiceFinder.java (70%) rename ranger-core/src/main/java/com/flipkart/ranger/core/finder/{sharded => }/SimpleShardedServiceFinderBuilder.java (63%) rename ranger-core/src/main/java/com/flipkart/ranger/core/finder/{unsharded/UnshardedClusterFinder.java => SimpleUnshardedServiceFinder.java} (57%) create mode 100644 ranger-core/src/main/java/com/flipkart/ranger/core/finder/SimpleUnshardedServiceFinderBuilder.java rename ranger-core/src/main/java/com/flipkart/ranger/core/finder/{ => nodeselector}/RandomServiceNodeSelector.java (94%) rename ranger-core/src/main/java/com/flipkart/ranger/core/finder/{ => nodeselector}/RoundRobinServiceNodeSelector.java (95%) rename ranger-core/src/main/java/com/flipkart/ranger/core/finder/{unsharded/UnshardedClusterServiceRegistry.java => serviceregistry/ListBasedServiceRegistry.java} (67%) rename ranger-core/src/main/java/com/flipkart/ranger/core/finder/{sharded => serviceregistry}/MapBasedServiceRegistry.java (76%) rename ranger-core/src/main/java/com/flipkart/ranger/core/finder/{ => serviceregistry}/ServiceRegistryUpdater.java (89%) create mode 100644 ranger-core/src/main/java/com/flipkart/ranger/core/finder/serviceregistry/signal/ScheduledRegistryUpdateSignal.java create mode 100644 ranger-core/src/main/java/com/flipkart/ranger/core/finder/shardselector/ListShardSelector.java create mode 100644 ranger-core/src/main/java/com/flipkart/ranger/core/finder/shardselector/MatchingShardSelector.java rename ranger-core/src/main/java/com/flipkart/ranger/core/finder/{sharded/MatchingShardSelector.java => shardselector/NoopShardSelector.java} (62%) delete mode 100644 ranger-core/src/main/java/com/flipkart/ranger/core/finder/signals/ScheduledRegistryUpdateSignal.java delete mode 100644 ranger-core/src/main/java/com/flipkart/ranger/core/finder/unsharded/UnshardedFinderBuilder.java rename {ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/zk => ranger-core/src/main/java/com/flipkart/ranger/core/finderhub}/ServiceFinderHubBuilder.java (53%) create mode 100644 ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/StaticDataSource.java rename ranger-core/src/main/java/com/flipkart/ranger/core/{finder/unsharded/UnshardedClusterInfo.java => model/Criteria.java} (69%) create mode 100644 ranger-core/src/test/java/com/flipkart/ranger/core/TestUtils.java create mode 100644 ranger-core/src/test/java/com/flipkart/ranger/core/finder/SimpleShardFinderTest.java create mode 100644 ranger-core/src/test/java/com/flipkart/ranger/core/finder/UnshardedClusterFinderTest.java create mode 100644 ranger-core/src/test/java/com/flipkart/ranger/core/finder/nodeselector/RoundRobinServiceNodeSelectorTest.java create mode 100644 ranger-core/src/test/java/com/flipkart/ranger/core/finder/serviceregistry/MapBasedServiceRegistryTest.java create mode 100644 ranger-core/src/test/java/com/flipkart/ranger/core/finder/shardselector/ListShardSelectorTest.java create mode 100644 ranger-core/src/test/java/com/flipkart/ranger/core/finder/shardselector/MatchingShardSelectorTest.java create mode 100644 ranger-core/src/test/java/com/flipkart/ranger/core/serviceprovider/ServiceProviderTest.java create mode 100644 ranger-core/src/test/java/com/flipkart/ranger/core/units/TestNodeData.java create mode 100644 ranger-core/src/test/java/com/flipkart/ranger/core/utils/CriteriaUtils.java create mode 100644 ranger-core/src/test/java/com/flipkart/ranger/core/utils/RegistryTestUtils.java delete mode 100644 ranger-core/src/test/java/com/flipkart/ranger/core/utils/TestUtils.java create mode 100644 ranger-http/src/main/java/com/flipkart/ranger/http/HttpServiceFinderBuilders.java create mode 100644 ranger-http/src/main/java/com/flipkart/ranger/http/HttpServiceProviderBuilders.java delete mode 100644 ranger-http/src/main/java/com/flipkart/ranger/http/HttpShardedServiceFinderBuilder.java rename ranger-http/src/main/java/com/flipkart/ranger/http/{ => common}/HttpNodeDataStoreConnector.java (68%) create mode 100644 ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceDataSourceResponse.java delete mode 100644 ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceNodesRequest.java create mode 100644 ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceRegistrationResponse.java rename ranger-core/src/main/java/com/flipkart/ranger/core/finder/unsharded/NoOpShardSelector.java => ranger-http/src/main/java/com/flipkart/ranger/http/serde/HttpRequestDataSerializer.java (56%) delete mode 100644 ranger-http/src/main/java/com/flipkart/ranger/http/serde/JacksonHTTPResponseDataDeserializer.java rename ranger-http/src/main/java/com/flipkart/ranger/http/{ => servicefinder}/HttpNodeDataSource.java (54%) create mode 100644 ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilder.java create mode 100644 ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpUnshardedServiceFinderBuilider.java create mode 100644 ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSource.java create mode 100644 ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceFinderHubBuilder.java create mode 100644 ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpShardedServiceFinderFactory.java create mode 100644 ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpUnshardedServiceFinderFactory.java create mode 100644 ranger-http/src/main/java/com/flipkart/ranger/http/serviceprovider/HttpNodeDataSink.java create mode 100644 ranger-http/src/main/java/com/flipkart/ranger/http/serviceprovider/HttpShardedServiceProviderBuilder.java create mode 100644 ranger-http/src/test/java/com/flipkart/ranger/http/ResourceHelper.java create mode 100644 ranger-http/src/test/java/com/flipkart/ranger/http/common/HttpNodeDataStoreConnectorTest.java create mode 100644 ranger-http/src/test/java/com/flipkart/ranger/http/config/HttpClientConfigTest.java create mode 100644 ranger-http/src/test/java/com/flipkart/ranger/http/model/ServiceNodeResponseTest.java create mode 100644 ranger-http/src/test/java/com/flipkart/ranger/http/model/ServiceRegistrationResponseTest.java rename ranger-http/src/test/java/com/flipkart/ranger/http/{ => servicefinder}/HttpShardedServiceFinderBuilderTest.java (65%) create mode 100644 ranger-http/src/test/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSourceTest.java create mode 100644 ranger-http/src/test/java/com/flipkart/ranger/http/serviceprovider/HttpShardedServiceProviderBuilderTest.java create mode 100644 ranger-http/src/test/resources/fixtures/httpClientConfig.json create mode 100644 ranger-http/src/test/resources/fixtures/serviceNodesResponse.json create mode 100644 ranger-http/src/test/resources/fixtures/serviceResponse.json rename ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/{zk => common}/ZkNodeDataStoreConnector.java (85%) rename ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/{zk => servicefinder}/ZkNodeDataSource.java (83%) rename ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/{zk => servicefinder}/ZkSimpleShardedServiceFinderBuilder.java (59%) rename ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/{zk/ZkUnshardedFinderBuilder.java => servicefinder/ZkSimpleUnshardedServiceFinderBuilder.java} (52%) rename ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/{zk => servicefinder/signals}/ZkWatcherRegistryUpdateSignal.java (75%) create mode 100644 ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZKUnshardedServiceFinderFactory.java create mode 100644 ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZkServiceDataSource.java rename ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/{zk => servicefinderhub}/ZkServiceFinderHubBuilder.java (52%) rename ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/{zk => servicefinderhub}/ZkShardedServiceFinderFactory.java (58%) rename ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/{zk => serviceprovider}/ZkNodeDataSink.java (79%) rename ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/{zk => serviceprovider}/ZkServiceProviderBuilder.java (74%) delete mode 100644 ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/zk/ZkServiceDataSource.java diff --git a/pom.xml b/pom.xml index 5e6d95f0..7573ca59 100644 --- a/pom.xml +++ b/pom.xml @@ -50,18 +50,23 @@ Tushar Naik tushar.naik@flipkart.com + + koushikr + Koushik Ramachandra + rkoushik.14@gmail.com + UTF-8 30.1.1-jre - 4.0.1 + 4.2.0 2.12.5 1.7.32 3.8.0 1.8 - 1.18.20 + 1.18.16 4.13.2 4.1.0 @@ -78,7 +83,7 @@ com.google.code.findbugs annotations - 3.0.1u2 + 3.0.1 com.google.guava diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/BaseServiceFinderBuilder.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/BaseServiceFinderBuilder.java index b9af724a..a83df33c 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/BaseServiceFinderBuilder.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/BaseServiceFinderBuilder.java @@ -1,4 +1,4 @@ -/** +/* * Copyright 2015 Flipkart Internet Pvt. Ltd. *

* Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,7 +16,9 @@ package com.flipkart.ranger.core.finder; -import com.flipkart.ranger.core.finder.signals.ScheduledRegistryUpdateSignal; +import com.flipkart.ranger.core.finder.nodeselector.RandomServiceNodeSelector; +import com.flipkart.ranger.core.finder.serviceregistry.ServiceRegistryUpdater; +import com.flipkart.ranger.core.finder.serviceregistry.signal.ScheduledRegistryUpdateSignal; import com.flipkart.ranger.core.model.*; import com.flipkart.ranger.core.signals.Signal; import com.google.common.base.Preconditions; @@ -31,20 +33,22 @@ import java.util.function.Consumer; @Slf4j +@SuppressWarnings("unchecked") public abstract class BaseServiceFinderBuilder < T, R extends ServiceRegistry, - F extends ServiceFinder, - B extends BaseServiceFinderBuilder, - D extends Deserializer> { + F extends ServiceFinder, + B extends BaseServiceFinderBuilder, + D extends Deserializer, + C extends Criteria> { protected String namespace; protected String serviceName; protected int nodeRefreshIntervalMs; protected boolean disablePushUpdaters; protected D deserializer; - protected ShardSelector shardSelector; + protected ShardSelector shardSelector; protected ServiceNodeSelector nodeSelector = new RandomServiceNodeSelector<>(); protected final List> additionalRefreshSignals = new ArrayList<>(); protected final List> startSignalHandlers = Lists.newArrayList(); @@ -65,7 +69,7 @@ public B withDeserializer(D deserializer) { return (B)this; } - public B withShardSelector(ShardSelector shardSelector) { + public B withShardSelector(ShardSelector shardSelector) { this.shardSelector = shardSelector; return (B)this; } @@ -150,7 +154,7 @@ protected F buildFinder() { log.debug("Added additional signal handlers"); } - val updater = new ServiceRegistryUpdater(registry, nodeDataSource, signalGenerators, deserializer); + val updater = new ServiceRegistryUpdater<>(registry, nodeDataSource, signalGenerators, deserializer); finder.getStartSignal() .registerConsumers(startSignalHandlers) .registerConsumer(x -> nodeDataSource.start()) @@ -173,7 +177,7 @@ protected List> implementationSpecificRefreshSignals(Service service, protected abstract F buildFinder( Service service, - ShardSelector shardSelector, + ShardSelector shardSelector, ServiceNodeSelector nodeSelector); } diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/ServiceFinder.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/ServiceFinder.java index d6004f9f..0ab7ab8a 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/ServiceFinder.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/ServiceFinder.java @@ -1,4 +1,4 @@ -/** +/* * Copyright 2015 Flipkart Internet Pvt. Ltd. *

* Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,11 +16,8 @@ package com.flipkart.ranger.core.finder; -import com.flipkart.ranger.core.model.ServiceNode; -import com.flipkart.ranger.core.model.ServiceRegistry; -import com.flipkart.ranger.core.model.ShardSelector; +import com.flipkart.ranger.core.model.*; import com.flipkart.ranger.core.signals.ExternalTriggeredSignal; -import com.flipkart.ranger.core.model.ServiceNodeSelector; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -28,10 +25,10 @@ import java.util.List; @Slf4j -public abstract class ServiceFinder> { +public abstract class ServiceFinder, ServiceRegistryType extends ServiceRegistry> { @Getter private final ServiceRegistryType serviceRegistry; - private final ShardSelector shardSelector; + private final ShardSelector shardSelector; private final ServiceNodeSelector nodeSelector; @Getter private final ExternalTriggeredSignal startSignal = new ExternalTriggeredSignal<>(() -> null, Collections.emptyList()); @@ -40,14 +37,14 @@ public abstract class ServiceFinder shardSelector, + ShardSelector shardSelector, ServiceNodeSelector nodeSelector) { this.serviceRegistry = serviceRegistry; this.shardSelector = shardSelector; this.nodeSelector = nodeSelector; } - public ServiceNode get(T criteria) { + public ServiceNode get(C criteria) { List> nodes = shardSelector.nodes(criteria, serviceRegistry); if (null == nodes || nodes.isEmpty()) { return null; @@ -55,7 +52,7 @@ public ServiceNode get(T criteria) { return nodeSelector.select(nodes); } - public List> getAll(T criteria) { + public List> getAll(C criteria) { return shardSelector.nodes(criteria, serviceRegistry); } diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/sharded/SimpleShardedServiceFinder.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/SimpleShardedServiceFinder.java similarity index 70% rename from ranger-core/src/main/java/com/flipkart/ranger/core/finder/sharded/SimpleShardedServiceFinder.java rename to ranger-core/src/main/java/com/flipkart/ranger/core/finder/SimpleShardedServiceFinder.java index 5c0405c3..44e66eb5 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/sharded/SimpleShardedServiceFinder.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/SimpleShardedServiceFinder.java @@ -1,4 +1,4 @@ -/** +/* * Copyright 2015 Flipkart Internet Pvt. Ltd. * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,15 +14,16 @@ * limitations under the License. */ -package com.flipkart.ranger.core.finder.sharded; +package com.flipkart.ranger.core.finder; -import com.flipkart.ranger.core.model.ShardSelector; -import com.flipkart.ranger.core.finder.ServiceFinder; +import com.flipkart.ranger.core.finder.serviceregistry.MapBasedServiceRegistry; +import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.ServiceNodeSelector; +import com.flipkart.ranger.core.model.ShardSelector; -public class SimpleShardedServiceFinder extends ServiceFinder> { +public class SimpleShardedServiceFinder> extends ServiceFinder> { public SimpleShardedServiceFinder(MapBasedServiceRegistry serviceRegistry, - ShardSelector> shardSelector, + ShardSelector> shardSelector, ServiceNodeSelector nodeSelector) { super(serviceRegistry, shardSelector, nodeSelector); } diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/sharded/SimpleShardedServiceFinderBuilder.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/SimpleShardedServiceFinderBuilder.java similarity index 63% rename from ranger-core/src/main/java/com/flipkart/ranger/core/finder/sharded/SimpleShardedServiceFinderBuilder.java rename to ranger-core/src/main/java/com/flipkart/ranger/core/finder/SimpleShardedServiceFinderBuilder.java index 25a2a596..a6aafb65 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/sharded/SimpleShardedServiceFinderBuilder.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/SimpleShardedServiceFinderBuilder.java @@ -1,4 +1,4 @@ -/** +/* * Copyright 2015 Flipkart Internet Pvt. Ltd. *

* Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,27 +14,25 @@ * limitations under the License. */ -package com.flipkart.ranger.core.finder.sharded; +package com.flipkart.ranger.core.finder; -import com.flipkart.ranger.core.finder.BaseServiceFinderBuilder; -import com.flipkart.ranger.core.model.Deserializer; -import com.flipkart.ranger.core.model.Service; -import com.flipkart.ranger.core.model.ServiceNodeSelector; -import com.flipkart.ranger.core.model.ShardSelector; +import com.flipkart.ranger.core.finder.serviceregistry.MapBasedServiceRegistry; +import com.flipkart.ranger.core.finder.shardselector.MatchingShardSelector; +import com.flipkart.ranger.core.model.*; -public abstract class SimpleShardedServiceFinderBuilder, D extends Deserializer> - extends BaseServiceFinderBuilder, SimpleShardedServiceFinder, B, D> { +public abstract class SimpleShardedServiceFinderBuilder, D extends Deserializer, C extends Criteria> + extends BaseServiceFinderBuilder, SimpleShardedServiceFinder, B, D, C> { @Override - protected SimpleShardedServiceFinder buildFinder( + protected SimpleShardedServiceFinder buildFinder( Service service, - ShardSelector> shardSelector, - ServiceNodeSelector nodeSelector) { + ShardSelector> shardSelector, + ServiceNodeSelector nodeSelector + ) { if (null == shardSelector) { shardSelector = new MatchingShardSelector<>(); } final MapBasedServiceRegistry serviceRegistry = new MapBasedServiceRegistry<>(service); return new SimpleShardedServiceFinder<>(serviceRegistry, shardSelector, nodeSelector); } - } diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/unsharded/UnshardedClusterFinder.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/SimpleUnshardedServiceFinder.java similarity index 57% rename from ranger-core/src/main/java/com/flipkart/ranger/core/finder/unsharded/UnshardedClusterFinder.java rename to ranger-core/src/main/java/com/flipkart/ranger/core/finder/SimpleUnshardedServiceFinder.java index be55c096..783a7a1c 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/unsharded/UnshardedClusterFinder.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/SimpleUnshardedServiceFinder.java @@ -1,12 +1,12 @@ -/** +/* * Copyright 2015 Flipkart Internet Pvt. Ltd. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -14,16 +14,17 @@ * limitations under the License. */ -package com.flipkart.ranger.core.finder.unsharded; +package com.flipkart.ranger.core.finder; -import com.flipkart.ranger.core.finder.ServiceFinder; +import com.flipkart.ranger.core.finder.serviceregistry.ListBasedServiceRegistry; +import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.ServiceNodeSelector; import com.flipkart.ranger.core.model.ShardSelector; -public class UnshardedClusterFinder extends ServiceFinder { - public UnshardedClusterFinder(UnshardedClusterServiceRegistry serviceRegistry, - ShardSelector shardSelector, - ServiceNodeSelector nodeSelector) { +public class SimpleUnshardedServiceFinder> extends ServiceFinder> { + public SimpleUnshardedServiceFinder(ListBasedServiceRegistry serviceRegistry, + ShardSelector> shardSelector, + ServiceNodeSelector nodeSelector) { super(serviceRegistry, shardSelector, nodeSelector); } } diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/SimpleUnshardedServiceFinderBuilder.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/SimpleUnshardedServiceFinderBuilder.java new file mode 100644 index 00000000..7917b705 --- /dev/null +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/SimpleUnshardedServiceFinderBuilder.java @@ -0,0 +1,37 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.core.finder; + +import com.flipkart.ranger.core.finder.serviceregistry.ListBasedServiceRegistry; +import com.flipkart.ranger.core.finder.shardselector.ListShardSelector; +import com.flipkart.ranger.core.model.*; + +public abstract class SimpleUnshardedServiceFinderBuilder, D extends Deserializer, C extends Criteria> + extends BaseServiceFinderBuilder, SimpleUnshardedServiceFinder, B, D, C> { + + @Override + protected SimpleUnshardedServiceFinder buildFinder( + Service service, + ShardSelector> shardSelector, + ServiceNodeSelector nodeSelector + ) { + if (null == shardSelector) { + shardSelector = new ListShardSelector<>(); + } + final ListBasedServiceRegistry serviceRegistry = new ListBasedServiceRegistry<>(service); + return new SimpleUnshardedServiceFinder<>(serviceRegistry, shardSelector, nodeSelector); + } +} \ No newline at end of file diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/RandomServiceNodeSelector.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/nodeselector/RandomServiceNodeSelector.java similarity index 94% rename from ranger-core/src/main/java/com/flipkart/ranger/core/finder/RandomServiceNodeSelector.java rename to ranger-core/src/main/java/com/flipkart/ranger/core/finder/nodeselector/RandomServiceNodeSelector.java index 3dc91885..6db1c034 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/RandomServiceNodeSelector.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/nodeselector/RandomServiceNodeSelector.java @@ -1,4 +1,4 @@ -/** +/* * Copyright 2015 Flipkart Internet Pvt. Ltd. * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,8 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -package com.flipkart.ranger.core.finder; +package com.flipkart.ranger.core.finder.nodeselector; import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.model.ServiceNodeSelector; diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/RoundRobinServiceNodeSelector.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/nodeselector/RoundRobinServiceNodeSelector.java similarity index 95% rename from ranger-core/src/main/java/com/flipkart/ranger/core/finder/RoundRobinServiceNodeSelector.java rename to ranger-core/src/main/java/com/flipkart/ranger/core/finder/nodeselector/RoundRobinServiceNodeSelector.java index 62ae215d..99b81d14 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/RoundRobinServiceNodeSelector.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/nodeselector/RoundRobinServiceNodeSelector.java @@ -1,4 +1,4 @@ -/** +/* * Copyright 2015 Flipkart Internet Pvt. Ltd. * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,8 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -package com.flipkart.ranger.core.finder; +package com.flipkart.ranger.core.finder.nodeselector; import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.model.ServiceNodeSelector; diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/unsharded/UnshardedClusterServiceRegistry.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/serviceregistry/ListBasedServiceRegistry.java similarity index 67% rename from ranger-core/src/main/java/com/flipkart/ranger/core/finder/unsharded/UnshardedClusterServiceRegistry.java rename to ranger-core/src/main/java/com/flipkart/ranger/core/finder/serviceregistry/ListBasedServiceRegistry.java index d5fe01af..c0e7f0ab 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/unsharded/UnshardedClusterServiceRegistry.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/serviceregistry/ListBasedServiceRegistry.java @@ -1,4 +1,4 @@ -/** +/* * Copyright 2015 Flipkart Internet Pvt. Ltd. * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,31 +13,32 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -package com.flipkart.ranger.core.finder.unsharded; +package com.flipkart.ranger.core.finder.serviceregistry; import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.model.ServiceRegistry; import com.google.common.collect.ImmutableList; +import java.util.Collections; import java.util.List; import java.util.concurrent.atomic.AtomicReference; -public class UnshardedClusterServiceRegistry extends ServiceRegistry { - private AtomicReference>> nodes +public class ListBasedServiceRegistry extends ServiceRegistry { + private final AtomicReference>> nodes = new AtomicReference<>(); - public UnshardedClusterServiceRegistry(Service service) { + public ListBasedServiceRegistry(Service service) { super(service); } - public List> nodes() { - return nodes.get(); + public List> nodeList() { + List> nodes = this.nodes.get(); + return null == nodes ? Collections.emptyList() : nodes; } @Override - public void updateNodes(List> serviceNodes) { + public void updateNodes(List> serviceNodes) { nodes.set(ImmutableList.copyOf(serviceNodes)); } } diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/sharded/MapBasedServiceRegistry.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/serviceregistry/MapBasedServiceRegistry.java similarity index 76% rename from ranger-core/src/main/java/com/flipkart/ranger/core/finder/sharded/MapBasedServiceRegistry.java rename to ranger-core/src/main/java/com/flipkart/ranger/core/finder/serviceregistry/MapBasedServiceRegistry.java index 116b411e..afaf6fb3 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/sharded/MapBasedServiceRegistry.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/serviceregistry/MapBasedServiceRegistry.java @@ -1,20 +1,19 @@ -/** +/* * Copyright 2015 Flipkart Internet Pvt. Ltd. - *

+ * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

+ * * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ - -package com.flipkart.ranger.core.finder.sharded; +package com.flipkart.ranger.core.finder.serviceregistry; import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.core.model.ServiceNode; @@ -23,14 +22,15 @@ import com.google.common.collect.ImmutableListMultimap; import com.google.common.collect.ListMultimap; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.concurrent.atomic.AtomicReference; public class MapBasedServiceRegistry extends ServiceRegistry { - private AtomicReference>> nodes = new AtomicReference<>(); + private final AtomicReference>> nodes = new AtomicReference<>(); - public MapBasedServiceRegistry( - Service service) { + public MapBasedServiceRegistry(Service service) { super(service); } @@ -39,6 +39,12 @@ public ListMultimap> nodes() { return null == nodeList ? ImmutableListMultimap.of() : nodeList; } + @Override + public List> nodeList() { + final ListMultimap> nodeList = nodes.get(); + return null == nodeList ? Collections.emptyList() : new ArrayList<>(nodeList.values()); + } + @Override public void updateNodes(List> nodes) { ListMultimap> serviceNodes = ArrayListMultimap.create(); diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/ServiceRegistryUpdater.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/serviceregistry/ServiceRegistryUpdater.java similarity index 89% rename from ranger-core/src/main/java/com/flipkart/ranger/core/finder/ServiceRegistryUpdater.java rename to ranger-core/src/main/java/com/flipkart/ranger/core/finder/serviceregistry/ServiceRegistryUpdater.java index c348036f..5a5a2634 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/ServiceRegistryUpdater.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/serviceregistry/ServiceRegistryUpdater.java @@ -1,20 +1,19 @@ -/** +/* * Copyright 2015 Flipkart Internet Pvt. Ltd. - *

+ * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

+ * * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ - -package com.flipkart.ranger.core.finder; +package com.flipkart.ranger.core.finder.serviceregistry; import com.flipkart.ranger.core.model.Deserializer; import com.flipkart.ranger.core.model.NodeDataSource; @@ -44,13 +43,13 @@ public class ServiceRegistryUpdater> { private final NodeDataSource nodeDataSource; private final D deserializer; - private Lock checkLock = new ReentrantLock(); - private Condition checkCondition = checkLock.newCondition(); + private final Lock checkLock = new ReentrantLock(); + private final Condition checkCondition = checkLock.newCondition(); private boolean checkForUpdate = false; private Future queryThreadFuture; - private ExecutorService executorService = Executors.newFixedThreadPool(1); - private AtomicBoolean initialRefreshCompleted = new AtomicBoolean(false); + private final ExecutorService executorService = Executors.newFixedThreadPool(1); + private final AtomicBoolean initialRefreshCompleted = new AtomicBoolean(false); public ServiceRegistryUpdater( ServiceRegistry serviceRegistry, @@ -126,7 +125,7 @@ private Void queryExecutor() { } } - private void updateRegistry() { + private void updateRegistry() throws InterruptedException { log.debug("Checking for updates on data source for service: {}", serviceRegistry.getService().getServiceName()); if(!nodeDataSource.isActive()) { @@ -134,11 +133,11 @@ private void updateRegistry() { serviceRegistry.getService().getServiceName()); return; } - val nodes = nodeDataSource.refresh(deserializer).orElse(null); - if (null != nodes) { - log.debug("Updating nodelist of size: {} for [{}]", nodes.size(), + val nodeList = nodeDataSource.refresh(deserializer); + if (nodeList.isPresent()) { + log.debug("Updating nodelist of size: {} for [{}]", nodeList.get().size(), serviceRegistry.getService().getServiceName()); - serviceRegistry.updateNodes(nodes); + serviceRegistry.updateNodes(nodeList.get()); initialRefreshCompleted.compareAndSet(false, true); } else { diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/serviceregistry/signal/ScheduledRegistryUpdateSignal.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/serviceregistry/signal/ScheduledRegistryUpdateSignal.java new file mode 100644 index 00000000..c728f923 --- /dev/null +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/serviceregistry/signal/ScheduledRegistryUpdateSignal.java @@ -0,0 +1,36 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.core.finder.serviceregistry.signal; + +import com.flipkart.ranger.core.model.Service; +import com.flipkart.ranger.core.signals.ScheduledSignal; +import lombok.extern.slf4j.Slf4j; + +import java.util.Collections; + +/** + * + */ +@Slf4j +public class ScheduledRegistryUpdateSignal extends ScheduledSignal { + + public ScheduledRegistryUpdateSignal( + Service service, + long refreshIntervalMillis) { + super(service, () -> null, Collections.emptyList(), refreshIntervalMillis); + } + +} diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/shardselector/ListShardSelector.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/shardselector/ListShardSelector.java new file mode 100644 index 00000000..d72fa6ae --- /dev/null +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/shardselector/ListShardSelector.java @@ -0,0 +1,37 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.core.finder.shardselector; + +import com.flipkart.ranger.core.finder.serviceregistry.ListBasedServiceRegistry; +import com.flipkart.ranger.core.model.Criteria; +import com.flipkart.ranger.core.model.ServiceNode; +import com.flipkart.ranger.core.model.ShardSelector; + +import java.util.List; +import java.util.stream.Collectors; + +public class ListShardSelector> implements ShardSelector> { + + @Override + public List> nodes(C criteria, ListBasedServiceRegistry serviceRegistry) { + if(null == criteria){ + return serviceRegistry.nodeList(); + } + return serviceRegistry.nodeList().stream().filter(node -> criteria.apply(node.getNodeData())).collect(Collectors.toList()); + } + + +} diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/shardselector/MatchingShardSelector.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/shardselector/MatchingShardSelector.java new file mode 100644 index 00000000..f94b2363 --- /dev/null +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/shardselector/MatchingShardSelector.java @@ -0,0 +1,40 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.flipkart.ranger.core.finder.shardselector; + +import com.flipkart.ranger.core.finder.serviceregistry.MapBasedServiceRegistry; +import com.flipkart.ranger.core.model.Criteria; +import com.flipkart.ranger.core.model.ServiceNode; +import com.flipkart.ranger.core.model.ShardSelector; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public class MatchingShardSelector> implements ShardSelector> { + + @Override + public List> nodes(C criteria, MapBasedServiceRegistry serviceRegistry) { + if(null == criteria) return serviceRegistry.nodeList(); + return serviceRegistry.nodes() + .entries() + .stream() + .filter(e -> criteria.apply(e.getKey())) + .map(Map.Entry::getValue) + .collect(Collectors.toList()); + } +} diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/sharded/MatchingShardSelector.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/shardselector/NoopShardSelector.java similarity index 62% rename from ranger-core/src/main/java/com/flipkart/ranger/core/finder/sharded/MatchingShardSelector.java rename to ranger-core/src/main/java/com/flipkart/ranger/core/finder/shardselector/NoopShardSelector.java index 5dc212ae..41d3c8fa 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/sharded/MatchingShardSelector.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/shardselector/NoopShardSelector.java @@ -1,4 +1,4 @@ -/** +/* * Copyright 2015 Flipkart Internet Pvt. Ltd. * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,18 +13,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +package com.flipkart.ranger.core.finder.shardselector; -package com.flipkart.ranger.core.finder.sharded; - -import com.flipkart.ranger.core.model.ShardSelector; +import com.flipkart.ranger.core.finder.serviceregistry.ListBasedServiceRegistry; +import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.ServiceNode; +import com.flipkart.ranger.core.model.ShardSelector; import java.util.List; -public class MatchingShardSelector implements ShardSelector> { - +public class NoopShardSelector> implements ShardSelector> { @Override - public List> nodes(T criteria, MapBasedServiceRegistry serviceRegistry) { - return serviceRegistry.nodes().get(criteria); + public List> nodes(C criteria, ListBasedServiceRegistry serviceRegistry) { + return serviceRegistry.nodeList(); } } diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/signals/ScheduledRegistryUpdateSignal.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/signals/ScheduledRegistryUpdateSignal.java deleted file mode 100644 index 63ae6289..00000000 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/signals/ScheduledRegistryUpdateSignal.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.flipkart.ranger.core.finder.signals; - -import com.flipkart.ranger.core.model.Service; -import com.flipkart.ranger.core.signals.ScheduledSignal; -import lombok.extern.slf4j.Slf4j; - -import java.util.Collections; - -/** - * - */ -@Slf4j -public class ScheduledRegistryUpdateSignal extends ScheduledSignal { - - public ScheduledRegistryUpdateSignal( - Service service, - long refreshIntervalMillis) { - super(service, () -> null, Collections.emptyList(), refreshIntervalMillis); - } - -} diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/unsharded/UnshardedFinderBuilder.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/unsharded/UnshardedFinderBuilder.java deleted file mode 100644 index b9af94aa..00000000 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/unsharded/UnshardedFinderBuilder.java +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright 2015 Flipkart Internet Pvt. Ltd. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.flipkart.ranger.core.finder.unsharded; - -import com.flipkart.ranger.core.model.Deserializer; -import com.flipkart.ranger.core.model.Service; -import com.flipkart.ranger.core.model.ShardSelector; -import com.flipkart.ranger.core.finder.BaseServiceFinderBuilder; -import com.flipkart.ranger.core.model.ServiceNodeSelector; - -public abstract class UnshardedFinderBuilder, D extends Deserializer> - extends BaseServiceFinderBuilder { - - @Override - protected UnshardedClusterFinder buildFinder( - Service service, - ShardSelector shardSelector, - ServiceNodeSelector nodeSelector) { - final UnshardedClusterServiceRegistry unshardedClusterServiceRegistry - = new UnshardedClusterServiceRegistry(service); - return new UnshardedClusterFinder(unshardedClusterServiceRegistry, new NoOpShardSelector(), nodeSelector); - } -} diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/ServiceDataSource.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/ServiceDataSource.java index 7a014377..dc579606 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/ServiceDataSource.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/ServiceDataSource.java @@ -1,3 +1,18 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.flipkart.ranger.core.finderhub; import com.flipkart.ranger.core.model.Service; diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/ServiceFinderFactory.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/ServiceFinderFactory.java index 5c89b2ca..d8b9c52e 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/ServiceFinderFactory.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/ServiceFinderFactory.java @@ -1,13 +1,30 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.flipkart.ranger.core.finderhub; import com.flipkart.ranger.core.finder.ServiceFinder; +import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.core.model.ServiceRegistry; /** * */ -public interface ServiceFinderFactory> { +public interface ServiceFinderFactory, R extends ServiceRegistry> { + + ServiceFinder buildFinder(final Service service); - ServiceFinder buildFinder(final Service service); } diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/ServiceFinderHub.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/ServiceFinderHub.java index aceeb761..d47298dc 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/ServiceFinderHub.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/ServiceFinderHub.java @@ -1,6 +1,22 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.flipkart.ranger.core.finderhub; import com.flipkart.ranger.core.finder.ServiceFinder; +import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.core.model.ServiceRegistry; import com.flipkart.ranger.core.signals.ExternalTriggeredSignal; @@ -26,12 +42,12 @@ * */ @Slf4j -public class ServiceFinderHub> { - private final AtomicReference>> finders = new AtomicReference<>(new HashMap<>()); +public class ServiceFinderHub, R extends ServiceRegistry> { + private final AtomicReference>> finders = new AtomicReference<>(new HashMap<>()); private final Lock updateLock = new ReentrantLock(); private final Condition updateCond = updateLock.newCondition(); private boolean updateAvailable = false; - private ExecutorService executorService = Executors.newFixedThreadPool(1); + private final ExecutorService executorService = Executors.newFixedThreadPool(1); @Getter private final ExternalTriggeredSignal startSignal @@ -42,15 +58,16 @@ public class ServiceFinderHub> { private final List> refreshSignals = new ArrayList<>(); + @Getter private final ServiceDataSource serviceDataSource; - private final ServiceFinderFactory finderFactory; + private final ServiceFinderFactory finderFactory; - private AtomicBoolean alreadyUpdating = new AtomicBoolean(false); + private final AtomicBoolean alreadyUpdating = new AtomicBoolean(false); private Future monitorFuture = null; public ServiceFinderHub( ServiceDataSource serviceDataSource, - ServiceFinderFactory finderFactory) { + ServiceFinderFactory finderFactory) { this.serviceDataSource = serviceDataSource; this.finderFactory = finderFactory; this.refreshSignals.add(new ScheduledSignal<>("service-hub-updater", @@ -59,7 +76,7 @@ public ServiceFinderHub( 10_000)); } - public Optional> finder(final Service service) { + public Optional> finder(final Service service) { return Optional.ofNullable(finders.get().get(service)); } @@ -126,10 +143,10 @@ private void updateRegistry() { return; } alreadyUpdating.set(true); - final Map> updatedFinders = new HashMap<>(); + final Map> updatedFinders = new HashMap<>(); try { final Collection services = serviceDataSource.services(); - final Map> knownServiceFinders = finders.get(); + final Map> knownServiceFinders = finders.get(); val newFinders = services.stream() .filter(service -> !knownServiceFinders.containsKey(service)) .collect(Collectors.toMap(Function.identity(), finderFactory::buildFinder)); diff --git a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/zk/ServiceFinderHubBuilder.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/ServiceFinderHubBuilder.java similarity index 53% rename from ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/zk/ServiceFinderHubBuilder.java rename to ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/ServiceFinderHubBuilder.java index 91d28c3b..1b883e3b 100644 --- a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/zk/ServiceFinderHubBuilder.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/ServiceFinderHubBuilder.java @@ -1,8 +1,21 @@ -package com.flipkart.ranger.zookeeper.zk; +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.core.finderhub; -import com.flipkart.ranger.core.finderhub.ServiceDataSource; -import com.flipkart.ranger.core.finderhub.ServiceFinderFactory; -import com.flipkart.ranger.core.finderhub.ServiceFinderHub; +import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.ServiceRegistry; import com.flipkart.ranger.core.signals.ScheduledSignal; import com.flipkart.ranger.core.signals.Signal; @@ -17,47 +30,49 @@ /** * */ -public abstract class ServiceFinderHubBuilder> { +public abstract class ServiceFinderHubBuilder, R extends ServiceRegistry> { private ServiceDataSource serviceDataSource; - private ServiceFinderFactory serviceFinderFactory; + private ServiceFinderFactory serviceFinderFactory; private long refreshFrequencyMs = 10_000; - private List> extraStartSignalConsumers = new ArrayList<>(); - private List> extraStopSignalConsumers = new ArrayList<>(); - private List> extraRefreshSignals = new ArrayList<>(); + private final List> extraStartSignalConsumers = new ArrayList<>(); + private final List> extraStopSignalConsumers = new ArrayList<>(); + private final List> extraRefreshSignals = new ArrayList<>(); - public ServiceFinderHubBuilder withServiceDataSource(ServiceDataSource serviceDataSource) { + public ServiceFinderHubBuilder withServiceDataSource(ServiceDataSource serviceDataSource) { this.serviceDataSource = serviceDataSource; return this; } - public ServiceFinderHubBuilder withServiceFinderFactory(ServiceFinderFactory serviceFinderFactory) { + public ServiceFinderHubBuilder withServiceFinderFactory(ServiceFinderFactory serviceFinderFactory) { this.serviceFinderFactory = serviceFinderFactory; return this; } - - public ServiceFinderHubBuilder withRefreshFrequencyMs(long refreshFrequencyMs) { + + public ServiceFinderHubBuilder withRefreshFrequencyMs(long refreshFrequencyMs) { this.refreshFrequencyMs = refreshFrequencyMs; return this; } - public ServiceFinderHubBuilder withExtraStartSignalConsumer(Consumer consumer) { + public ServiceFinderHubBuilder withExtraStartSignalConsumer(Consumer consumer) { this.extraStartSignalConsumers.add(consumer); return this; } - public ServiceFinderHubBuilder withExtraStopSignalConsumer(Consumer consumer) { + public ServiceFinderHubBuilder withExtraStopSignalConsumer(Consumer consumer) { this.extraStopSignalConsumers.add(consumer); return this; } - public ServiceFinderHubBuilder withExtraRefreshSignal(Signal extraRefreshSignal) { + public ServiceFinderHubBuilder withExtraRefreshSignal(Signal extraRefreshSignal) { this.extraRefreshSignals.add(extraRefreshSignal); return this; } - public ServiceFinderHub build() { + public ServiceFinderHub build() { + preBuild(); Preconditions.checkNotNull(serviceDataSource, "Provide a non-null service data source"); Preconditions.checkNotNull(serviceFinderFactory, "Provide a non-null service finder factory"); + val hub = new ServiceFinderHub<>(serviceDataSource, serviceFinderFactory); final ScheduledSignal refreshSignal = new ScheduledSignal<>("service-hub-refresh-timer", () -> null, @@ -74,9 +89,11 @@ public ServiceFinderHub build() { .registerConsumers(extraStopSignalConsumers) .registerConsumer(x -> refreshSignal.stop()) .registerConsumer(x -> serviceDataSource.stop()); + postBuild(hub); return hub; } protected abstract void preBuild(); - protected abstract void postBuild(ServiceFinderHub serviceFinderHub); + + protected abstract void postBuild(ServiceFinderHub serviceFinderHub); } diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/StaticDataSource.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/StaticDataSource.java new file mode 100644 index 00000000..11ebe0e6 --- /dev/null +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/StaticDataSource.java @@ -0,0 +1,49 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.core.finderhub; + +import com.flipkart.ranger.core.model.Service; +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import java.util.Collection; +import java.util.List; + +/* +A static data source to be used when we know the services beforehand and don't have to fetch from a source. + */ + +@Slf4j +@AllArgsConstructor +public class StaticDataSource implements ServiceDataSource{ + + private final List services; + + @Override + public Collection services() throws Exception { + return services; + } + + @Override + public void start() { + + } + + @Override + public void stop() { + + } +} diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/healthcheck/HealthChecker.java b/ranger-core/src/main/java/com/flipkart/ranger/core/healthcheck/HealthChecker.java index 70d6cbb1..3d7dbd3b 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/healthcheck/HealthChecker.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/healthcheck/HealthChecker.java @@ -1,4 +1,4 @@ -/** +/* * Copyright 2015 Flipkart Internet Pvt. Ltd. *

* Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,7 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package com.flipkart.ranger.core.healthcheck; import lombok.extern.slf4j.Slf4j; diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/healthcheck/Healthcheck.java b/ranger-core/src/main/java/com/flipkart/ranger/core/healthcheck/Healthcheck.java index ab266709..100372f0 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/healthcheck/Healthcheck.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/healthcheck/Healthcheck.java @@ -1,12 +1,12 @@ -/** +/* * Copyright 2015 Flipkart Internet Pvt. Ltd. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -26,5 +26,5 @@ public interface Healthcheck { * * @return health status */ - public HealthcheckStatus check(); + HealthcheckStatus check(); } diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/healthcheck/HealthcheckResult.java b/ranger-core/src/main/java/com/flipkart/ranger/core/healthcheck/HealthcheckResult.java index f77f1efd..e7b38167 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/healthcheck/HealthcheckResult.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/healthcheck/HealthcheckResult.java @@ -1,3 +1,18 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.flipkart.ranger.core.healthcheck; import lombok.Builder; diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/healthcheck/HealthcheckStatus.java b/ranger-core/src/main/java/com/flipkart/ranger/core/healthcheck/HealthcheckStatus.java index 22ba575a..4b84fa68 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/healthcheck/HealthcheckStatus.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/healthcheck/HealthcheckStatus.java @@ -1,19 +1,18 @@ -/** +/* * Copyright 2015 Flipkart Internet Pvt. Ltd. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ - package com.flipkart.ranger.core.healthcheck; public enum HealthcheckStatus { diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/healthcheck/Healthchecks.java b/ranger-core/src/main/java/com/flipkart/ranger/core/healthcheck/Healthchecks.java index 42c24505..a07cc271 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/healthcheck/Healthchecks.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/healthcheck/Healthchecks.java @@ -1,3 +1,18 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.flipkart.ranger.core.healthcheck; import java.io.File; diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/HealthService.java b/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/HealthService.java index 02fa9802..3c79b04b 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/HealthService.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/HealthService.java @@ -1,12 +1,12 @@ -/** - * Copyright 2016 Flipkart Internet Pvt. Ltd. - * +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/ServiceHealthAggregator.java b/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/ServiceHealthAggregator.java index 094e43fc..2468a633 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/ServiceHealthAggregator.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/ServiceHealthAggregator.java @@ -1,5 +1,5 @@ -/** - * Copyright 2016 Flipkart Internet Pvt. Ltd. +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,10 +16,10 @@ package com.flipkart.ranger.core.healthservice; -import com.flipkart.ranger.core.healthservice.monitor.IsolatedHealthMonitor; -import com.flipkart.ranger.core.healthservice.monitor.Monitor; import com.flipkart.ranger.core.healthcheck.Healthcheck; import com.flipkart.ranger.core.healthcheck.HealthcheckStatus; +import com.flipkart.ranger.core.healthservice.monitor.IsolatedHealthMonitor; +import com.flipkart.ranger.core.healthservice.monitor.Monitor; import com.google.common.collect.Lists; import lombok.extern.slf4j.Slf4j; diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/TimeEntity.java b/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/TimeEntity.java index ffdadefb..cde306cd 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/TimeEntity.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/TimeEntity.java @@ -1,19 +1,18 @@ -/** - * Copyright 2016 Flipkart Internet Pvt. Ltd. - * +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ - package com.flipkart.ranger.core.healthservice; import com.google.common.base.MoreObjects; diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/IsolatedHealthMonitor.java b/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/IsolatedHealthMonitor.java index 19511975..d3bf3bf6 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/IsolatedHealthMonitor.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/IsolatedHealthMonitor.java @@ -1,12 +1,12 @@ -/** - * Copyright 2016 Flipkart Internet Pvt. Ltd. - * +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -16,10 +16,10 @@ package com.flipkart.ranger.core.healthservice.monitor; +import com.flipkart.ranger.core.healthservice.TimeEntity; import com.flipkart.ranger.core.healthservice.monitor.sample.CountMonitor; import com.flipkart.ranger.core.healthservice.monitor.sample.PingCheckMonitor; import com.flipkart.ranger.core.healthservice.monitor.sample.RotationStatusMonitor; -import com.flipkart.ranger.core.healthservice.TimeEntity; import java.util.Date; import java.util.concurrent.atomic.AtomicBoolean; diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/Monitor.java b/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/Monitor.java index c74b31e0..2f2ab700 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/Monitor.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/Monitor.java @@ -1,12 +1,12 @@ -/** - * Copyright 2016 Flipkart Internet Pvt. Ltd. - * +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/Monitors.java b/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/Monitors.java index b4578a16..769ae8cd 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/Monitors.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/Monitors.java @@ -1,19 +1,18 @@ -/** - * Copyright 2016 Flipkart Internet Pvt. Ltd. - * +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ - package com.flipkart.ranger.core.healthservice.monitor; import com.flipkart.ranger.core.healthcheck.HealthcheckStatus; diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/RollingWindowHealthQueue.java b/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/RollingWindowHealthQueue.java index 3fdb0b27..6a0aab1d 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/RollingWindowHealthQueue.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/RollingWindowHealthQueue.java @@ -1,12 +1,12 @@ -/** - * Copyright 2016 Flipkart Internet Pvt. Ltd. - * +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/sample/CountMonitor.java b/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/sample/CountMonitor.java index 2529bfa5..de3e1551 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/sample/CountMonitor.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/sample/CountMonitor.java @@ -1,24 +1,23 @@ -/** - * Copyright 2016 Flipkart Internet Pvt. Ltd. - * +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ - package com.flipkart.ranger.core.healthservice.monitor.sample; -import com.flipkart.ranger.core.healthservice.monitor.IsolatedHealthMonitor; import com.flipkart.ranger.core.healthcheck.HealthcheckStatus; import com.flipkart.ranger.core.healthservice.TimeEntity; +import com.flipkart.ranger.core.healthservice.monitor.IsolatedHealthMonitor; /** * A monitor that can be used as a counting monitor to check if any countable entity breaches a threashhold diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/sample/DiskSpaceMonitor.java b/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/sample/DiskSpaceMonitor.java index 59088729..10cca535 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/sample/DiskSpaceMonitor.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/sample/DiskSpaceMonitor.java @@ -1,19 +1,18 @@ -/** - * Copyright 2016 Flipkart Internet Pvt. Ltd. - * +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ - package com.flipkart.ranger.core.healthservice.monitor.sample; import com.flipkart.ranger.core.healthservice.TimeEntity; diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/sample/PingCheckMonitor.java b/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/sample/PingCheckMonitor.java index 92fdb7c1..b9385320 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/sample/PingCheckMonitor.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/sample/PingCheckMonitor.java @@ -1,19 +1,18 @@ -/** - * Copyright 2016 Flipkart Internet Pvt. Ltd. - * +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ - package com.flipkart.ranger.core.healthservice.monitor.sample; import com.flipkart.ranger.core.healthcheck.HealthcheckStatus; diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/sample/RotationStatusMonitor.java b/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/sample/RotationStatusMonitor.java index 5a2f9039..655eadc0 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/sample/RotationStatusMonitor.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/sample/RotationStatusMonitor.java @@ -1,5 +1,23 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.flipkart.ranger.core.healthservice.monitor.sample; + /** - * Copyright 2016 Flipkart Internet Pvt. Ltd. + * Copyright 2015 Flipkart Internet Pvt. Ltd. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,9 +31,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -package com.flipkart.ranger.core.healthservice.monitor.sample; - import com.flipkart.ranger.core.healthcheck.HealthcheckStatus; import com.flipkart.ranger.core.healthservice.TimeEntity; import com.flipkart.ranger.core.healthservice.monitor.IsolatedHealthMonitor; diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/unsharded/UnshardedClusterInfo.java b/ranger-core/src/main/java/com/flipkart/ranger/core/model/Criteria.java similarity index 69% rename from ranger-core/src/main/java/com/flipkart/ranger/core/finder/unsharded/UnshardedClusterInfo.java rename to ranger-core/src/main/java/com/flipkart/ranger/core/model/Criteria.java index 93d45fce..a9baf215 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/unsharded/UnshardedClusterInfo.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/model/Criteria.java @@ -1,29 +1,23 @@ -/** +/* * Copyright 2015 Flipkart Internet Pvt. Ltd. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ +package com.flipkart.ranger.core.model; -package com.flipkart.ranger.core.finder.unsharded; +@FunctionalInterface +public interface Criteria { -public class UnshardedClusterInfo { - @Override - public int hashCode() { - return 0; - } + boolean apply(T nodeData); - @Override - public boolean equals(Object obj) { - return super.equals(obj); - } } diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/model/Deserializer.java b/ranger-core/src/main/java/com/flipkart/ranger/core/model/Deserializer.java index c4acf561..74aebe5e 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/model/Deserializer.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/model/Deserializer.java @@ -1,19 +1,18 @@ -/** +/* * Copyright 2015 Flipkart Internet Pvt. Ltd. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ - package com.flipkart.ranger.core.model; public interface Deserializer { diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/model/NodeDataSink.java b/ranger-core/src/main/java/com/flipkart/ranger/core/model/NodeDataSink.java index 92684451..2b6b8ac8 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/model/NodeDataSink.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/model/NodeDataSink.java @@ -1,3 +1,18 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.flipkart.ranger.core.model; /** diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/model/NodeDataSource.java b/ranger-core/src/main/java/com/flipkart/ranger/core/model/NodeDataSource.java index e11ccac9..79f75a8d 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/model/NodeDataSource.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/model/NodeDataSource.java @@ -1,3 +1,18 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.flipkart.ranger.core.model; import java.util.List; diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/model/NodeDataStoreConnector.java b/ranger-core/src/main/java/com/flipkart/ranger/core/model/NodeDataStoreConnector.java index e2cab33a..1b60442b 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/model/NodeDataStoreConnector.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/model/NodeDataStoreConnector.java @@ -1,3 +1,18 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.flipkart.ranger.core.model; /** diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/model/Serializer.java b/ranger-core/src/main/java/com/flipkart/ranger/core/model/Serializer.java index c4e8d401..ab37c6ad 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/model/Serializer.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/model/Serializer.java @@ -1,12 +1,12 @@ -/** +/* * Copyright 2015 Flipkart Internet Pvt. Ltd. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/model/Service.java b/ranger-core/src/main/java/com/flipkart/ranger/core/model/Service.java index ec3e7fe0..a9f77786 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/model/Service.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/model/Service.java @@ -1,19 +1,18 @@ -/** +/* * Copyright 2015 Flipkart Internet Pvt. Ltd. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ - package com.flipkart.ranger.core.model; import lombok.AccessLevel; @@ -23,8 +22,10 @@ @Data @AllArgsConstructor(access = AccessLevel.PUBLIC) public class Service { - private final String namespace; - private final String serviceName; + private String namespace; + private String serviceName; + + public Service(){} public String name() { return String.format("%s/%s", namespace, serviceName); diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/model/ServiceNode.java b/ranger-core/src/main/java/com/flipkart/ranger/core/model/ServiceNode.java index a0563f31..383c6a36 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/model/ServiceNode.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/model/ServiceNode.java @@ -1,4 +1,4 @@ -/** +/* * Copyright 2015 Flipkart Internet Pvt. Ltd. * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -12,8 +12,8 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - */ + */ package com.flipkart.ranger.core.model; import com.flipkart.ranger.core.healthcheck.HealthcheckStatus; diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/model/ServiceNodeSelector.java b/ranger-core/src/main/java/com/flipkart/ranger/core/model/ServiceNodeSelector.java index d655b7ff..786c03c4 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/model/ServiceNodeSelector.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/model/ServiceNodeSelector.java @@ -1,4 +1,4 @@ -/** +/* * Copyright 2015 Flipkart Internet Pvt. Ltd. *

* Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,7 +13,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package com.flipkart.ranger.core.model; import java.util.List; diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/model/ServiceRegistry.java b/ranger-core/src/main/java/com/flipkart/ranger/core/model/ServiceRegistry.java index 09209aaf..60bd8041 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/model/ServiceRegistry.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/model/ServiceRegistry.java @@ -1,19 +1,18 @@ -/** +/* * Copyright 2015 Flipkart Internet Pvt. Ltd. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ - package com.flipkart.ranger.core.model; import lombok.AccessLevel; @@ -27,5 +26,7 @@ public abstract class ServiceRegistry { private final Service service; + public abstract List> nodeList(); + public abstract void updateNodes(List> nodes); } diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/model/ShardSelector.java b/ranger-core/src/main/java/com/flipkart/ranger/core/model/ShardSelector.java index 7cea8c84..4053fe07 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/model/ShardSelector.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/model/ShardSelector.java @@ -1,23 +1,24 @@ -/** +/* * Copyright 2015 Flipkart Internet Pvt. Ltd. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ - package com.flipkart.ranger.core.model; import java.util.List; -public interface ShardSelector> { - List> nodes(T criteria, ServiceRegistryType serviceRegistry); +public interface ShardSelector, ServiceRegistryType extends ServiceRegistry> { + + List> nodes(C criteria, ServiceRegistryType serviceRegistry); + } diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/serviceprovider/BaseServiceProviderBuilder.java b/ranger-core/src/main/java/com/flipkart/ranger/core/serviceprovider/BaseServiceProviderBuilder.java index 38db70af..ff6c0597 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/serviceprovider/BaseServiceProviderBuilder.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/serviceprovider/BaseServiceProviderBuilder.java @@ -1,12 +1,12 @@ -/** +/* * Copyright 2015 Flipkart Internet Pvt. Ltd. - *

+ *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

+ *

* http://www.apache.org/licenses/LICENSE-2.0 - *

+ *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -22,7 +22,10 @@ import com.flipkart.ranger.core.healthservice.HealthService; import com.flipkart.ranger.core.healthservice.ServiceHealthAggregator; import com.flipkart.ranger.core.healthservice.monitor.IsolatedHealthMonitor; -import com.flipkart.ranger.core.model.*; +import com.flipkart.ranger.core.model.NodeDataSink; +import com.flipkart.ranger.core.model.Serializer; +import com.flipkart.ranger.core.model.Service; +import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.signals.ScheduledSignal; import com.flipkart.ranger.core.signals.Signal; import com.google.common.base.Preconditions; @@ -48,11 +51,11 @@ public abstract class BaseServiceProviderBuilder healthchecks = Lists.newArrayList(); protected NodeDataSink nodeDataSource = null; + protected List healthchecks = Lists.newArrayList(); protected final List> startSignalHandlers = Lists.newArrayList(); protected final List> stopSignalHandlers = Lists.newArrayList(); - protected final List> extraRefreshSignals = Lists.newArrayList(); + protected final List> additionalRefreshSignals = Lists.newArrayList(); /* list of isolated monitors */ private List isolatedMonitors = Lists.newArrayList(); @@ -141,13 +144,13 @@ public B withStopSignalHandlers(List> stopSignalHandlers) { return (B)this; } - public B withExtraRefreshSignal(Signal extraRefreshSignal) { - this.extraRefreshSignals.add(extraRefreshSignal); + public B withadditionalRefreshSignal(Signal additionalRefreshSignal) { + this.additionalRefreshSignals.add(additionalRefreshSignal); return (B)this; } - public B withExtraRefreshSignals(List> extraRefreshSignals) { - this.extraRefreshSignals.addAll(extraRefreshSignals); + public B withAdditionalRefreshSignals(List> additionalRefreshSignals) { + this.additionalRefreshSignals.addAll(additionalRefreshSignals); return (B)this; } @@ -175,21 +178,25 @@ protected final ServiceProvider buildProvider() { for (IsolatedHealthMonitor isolatedMonitor : isolatedMonitors) { serviceHealthAggregator.addIsolatedMonitor(isolatedMonitor); } + healthchecks.add(serviceHealthAggregator); final Service service = new Service(namespace, serviceName); + final NodeDataSink usableNodeDataSource = dataSink(service); + final ScheduledSignal healthcheckUpdateSignalGenerator = new ScheduledSignal<>( service, new HealthChecker(healthchecks, staleUpdateThresholdMs), Collections.emptyList(), - healthUpdateIntervalMs); - final NodeDataSink usableNodeDataSource = dataSink(service); + healthUpdateIntervalMs + ); + final List healthServices = Collections.singletonList(serviceHealthAggregator); final List> signalGenerators = ImmutableList.>builder() .add(healthcheckUpdateSignalGenerator) - .addAll(extraRefreshSignals) + .addAll(additionalRefreshSignals) .build(); final ServiceProvider serviceProvider = new ServiceProvider<>(service, new ServiceNode<>(hostname, port, nodeData), diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/serviceprovider/ServiceProvider.java b/ranger-core/src/main/java/com/flipkart/ranger/core/serviceprovider/ServiceProvider.java index 2c19997e..b2f381f0 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/serviceprovider/ServiceProvider.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/serviceprovider/ServiceProvider.java @@ -1,12 +1,12 @@ -/** +/* * Copyright 2015 Flipkart Internet Pvt. Ltd. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -16,10 +16,13 @@ package com.flipkart.ranger.core.serviceprovider; -import com.flipkart.ranger.core.model.*; +import com.flipkart.ranger.core.healthcheck.HealthcheckResult; +import com.flipkart.ranger.core.model.NodeDataSink; +import com.flipkart.ranger.core.model.Serializer; +import com.flipkart.ranger.core.model.Service; +import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.signals.ExternalTriggeredSignal; import com.flipkart.ranger.core.signals.Signal; -import com.flipkart.ranger.core.healthcheck.HealthcheckResult; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -32,6 +35,7 @@ public class ServiceProvider> { private final Service service; private final ServiceNode serviceNode; private final S serializer; + @Getter private final NodeDataSink dataSink; @Getter private final ExternalTriggeredSignal startSignal = new ExternalTriggeredSignal<>(() -> null, Collections.emptyList()); diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/signals/ExternalTriggeredSignal.java b/ranger-core/src/main/java/com/flipkart/ranger/core/signals/ExternalTriggeredSignal.java index 2dc41acd..350a2005 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/signals/ExternalTriggeredSignal.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/signals/ExternalTriggeredSignal.java @@ -1,3 +1,18 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.flipkart.ranger.core.signals; import java.util.List; diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/signals/ScheduledSignal.java b/ranger-core/src/main/java/com/flipkart/ranger/core/signals/ScheduledSignal.java index 71b39886..ab6b93dd 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/signals/ScheduledSignal.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/signals/ScheduledSignal.java @@ -1,3 +1,18 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.flipkart.ranger.core.signals; import com.flipkart.ranger.core.model.Service; diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/signals/Signal.java b/ranger-core/src/main/java/com/flipkart/ranger/core/signals/Signal.java index 354b3d4a..cb03ff6f 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/signals/Signal.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/signals/Signal.java @@ -1,3 +1,18 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.flipkart.ranger.core.signals; import lombok.val; diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/util/Exceptions.java b/ranger-core/src/main/java/com/flipkart/ranger/core/util/Exceptions.java index 69f75f2f..d93d167e 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/util/Exceptions.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/util/Exceptions.java @@ -1,3 +1,18 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.flipkart.ranger.core.util; /** diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/util/FinderUtils.java b/ranger-core/src/main/java/com/flipkart/ranger/core/util/FinderUtils.java index 204dd4b5..54fa9545 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/util/FinderUtils.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/util/FinderUtils.java @@ -1,3 +1,18 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.flipkart.ranger.core.util; import com.flipkart.ranger.core.healthcheck.HealthcheckStatus; diff --git a/ranger-core/src/test/java/com/flipkart/ranger/core/TestUtils.java b/ranger-core/src/test/java/com/flipkart/ranger/core/TestUtils.java new file mode 100644 index 00000000..cde529a6 --- /dev/null +++ b/ranger-core/src/test/java/com/flipkart/ranger/core/TestUtils.java @@ -0,0 +1,29 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.core; + +import java.util.concurrent.TimeUnit; + +import static org.awaitility.Awaitility.await; + +/** + * + */ +public class TestUtils { + public static void sleepForSeconds(int numSeconds) { + await().pollDelay(numSeconds, TimeUnit.SECONDS).until(() -> true); + } +} diff --git a/ranger-core/src/test/java/com/flipkart/ranger/core/finder/SimpleShardFinderTest.java b/ranger-core/src/test/java/com/flipkart/ranger/core/finder/SimpleShardFinderTest.java new file mode 100644 index 00000000..404d92fa --- /dev/null +++ b/ranger-core/src/test/java/com/flipkart/ranger/core/finder/SimpleShardFinderTest.java @@ -0,0 +1,54 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.core.finder; + +import com.flipkart.ranger.core.finder.nodeselector.RoundRobinServiceNodeSelector; +import com.flipkart.ranger.core.finder.serviceregistry.MapBasedServiceRegistry; +import com.flipkart.ranger.core.model.Criteria; +import com.flipkart.ranger.core.model.ServiceNode; +import com.flipkart.ranger.core.model.ShardSelector; +import com.flipkart.ranger.core.units.TestNodeData; +import com.flipkart.ranger.core.utils.CriteriaUtils; +import com.flipkart.ranger.core.utils.RegistryTestUtils; +import com.google.common.collect.Lists; +import lombok.val; +import org.junit.Assert; +import org.junit.Test; + +import java.util.List; + +public class SimpleShardFinderTest { + + static class TestSimpleShardSelector> implements ShardSelector>{ + + @Override + public List> nodes(C criteria, MapBasedServiceRegistry serviceRegistry) { + return Lists.newArrayList(); + } + } + + @Test + public void testSimpleShardedFinder(){ + final MapBasedServiceRegistry serviceRegistry = RegistryTestUtils.getServiceRegistry(); + final TestSimpleShardSelector> shardSelector = new TestSimpleShardSelector<>(); + final RoundRobinServiceNodeSelector roundRobinServiceNodeSelector = new RoundRobinServiceNodeSelector<>(); + val simpleShardedFinder = new SimpleShardedServiceFinder<>( + serviceRegistry, shardSelector, roundRobinServiceNodeSelector); + val testNodeDataServiceNode = simpleShardedFinder.get( + CriteriaUtils.getCriteria(2)); + Assert.assertNull(testNodeDataServiceNode); + } +} diff --git a/ranger-core/src/test/java/com/flipkart/ranger/core/finder/UnshardedClusterFinderTest.java b/ranger-core/src/test/java/com/flipkart/ranger/core/finder/UnshardedClusterFinderTest.java new file mode 100644 index 00000000..ad35be35 --- /dev/null +++ b/ranger-core/src/test/java/com/flipkart/ranger/core/finder/UnshardedClusterFinderTest.java @@ -0,0 +1,53 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.core.finder; + +import com.flipkart.ranger.core.finder.serviceregistry.ListBasedServiceRegistry; +import com.flipkart.ranger.core.finder.shardselector.ListShardSelector; +import com.flipkart.ranger.core.model.Criteria; +import com.flipkart.ranger.core.model.ServiceNode; +import com.flipkart.ranger.core.model.ServiceNodeSelector; +import com.flipkart.ranger.core.units.TestNodeData; +import com.flipkart.ranger.core.utils.CriteriaUtils; +import com.flipkart.ranger.core.utils.RegistryTestUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.util.List; + +public class UnshardedClusterFinderTest { + + static class TestUnshardedNodeSelector implements ServiceNodeSelector{ + + @Override + public ServiceNode select(List> serviceNodes) { + return serviceNodes.get(0); + } + } + @Test + public void unshardedClusterFinder(){ + final ListBasedServiceRegistry unshardedRegistry = RegistryTestUtils.getUnshardedRegistry(); + final ListShardSelector> shardSelector = new ListShardSelector<>(); + SimpleUnshardedServiceFinder> simpleUnshardedServiceFinder = new SimpleUnshardedServiceFinder<>( + unshardedRegistry, + shardSelector, + new TestUnshardedNodeSelector() + ); + final ServiceNode serviceNode = simpleUnshardedServiceFinder.get(CriteriaUtils.getCriteria(1)); + Assert.assertNotNull(serviceNode); + Assert.assertEquals("localhost-1", serviceNode.getHost()); + } +} diff --git a/ranger-core/src/test/java/com/flipkart/ranger/core/finder/nodeselector/RoundRobinServiceNodeSelectorTest.java b/ranger-core/src/test/java/com/flipkart/ranger/core/finder/nodeselector/RoundRobinServiceNodeSelectorTest.java new file mode 100644 index 00000000..70563099 --- /dev/null +++ b/ranger-core/src/test/java/com/flipkart/ranger/core/finder/nodeselector/RoundRobinServiceNodeSelectorTest.java @@ -0,0 +1,41 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.core.finder.nodeselector; + +import com.flipkart.ranger.core.model.ServiceNode; +import com.flipkart.ranger.core.units.TestNodeData; +import com.google.common.collect.Lists; +import org.junit.Assert; +import org.junit.Test; + +import java.util.List; + +public class RoundRobinServiceNodeSelectorTest { + @Test + public void testRandomNodeSelector(){ + final RoundRobinServiceNodeSelector roundRobinSelector = new RoundRobinServiceNodeSelector(); + List> serviceNodes = Lists.newArrayList(); + serviceNodes.add(new ServiceNode<>("localhost-1", 9000, TestNodeData.builder().nodeId(1).build())); + serviceNodes.add(new ServiceNode<>("localhost-2", 9001, TestNodeData.builder().nodeId(2).build())); + serviceNodes.add(new ServiceNode<>("localhost-3", 9002, TestNodeData.builder().nodeId(3).build())); + ServiceNode select = roundRobinSelector.select(serviceNodes); + Assert.assertTrue(select.getHost().equalsIgnoreCase("localhost-2")); + select = roundRobinSelector.select(serviceNodes); + Assert.assertTrue(select.getHost().equalsIgnoreCase("localhost-3")); + select = roundRobinSelector.select(serviceNodes); + Assert.assertTrue(select.getHost().equalsIgnoreCase("localhost-1")); + } +} diff --git a/ranger-core/src/test/java/com/flipkart/ranger/core/finder/serviceregistry/MapBasedServiceRegistryTest.java b/ranger-core/src/test/java/com/flipkart/ranger/core/finder/serviceregistry/MapBasedServiceRegistryTest.java new file mode 100644 index 00000000..effc6f41 --- /dev/null +++ b/ranger-core/src/test/java/com/flipkart/ranger/core/finder/serviceregistry/MapBasedServiceRegistryTest.java @@ -0,0 +1,40 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.core.finder.serviceregistry; + +import com.flipkart.ranger.core.finder.shardselector.MatchingShardSelector; +import com.flipkart.ranger.core.model.Criteria; +import com.flipkart.ranger.core.units.TestNodeData; +import com.flipkart.ranger.core.utils.CriteriaUtils; +import com.flipkart.ranger.core.utils.RegistryTestUtils; +import lombok.val; +import org.junit.Assert; +import org.junit.Test; + +public class MapBasedServiceRegistryTest { + + @Test + public void testMapBasedServiceRegistryWithMatchingShardSelector(){ + final MapBasedServiceRegistry serviceRegistry = RegistryTestUtils.getServiceRegistry(); + Assert.assertTrue(null != serviceRegistry.nodes() && !serviceRegistry.nodes().isEmpty()); + final MatchingShardSelector> matchingShardSelector = new MatchingShardSelector<>(); + val nodes = matchingShardSelector.nodes( + CriteriaUtils.getCriteria(1), serviceRegistry); + Assert.assertFalse(nodes.isEmpty()); + Assert.assertEquals("localhost-1", nodes.get(0).getHost()); + } + +} diff --git a/ranger-core/src/test/java/com/flipkart/ranger/core/finder/shardselector/ListShardSelectorTest.java b/ranger-core/src/test/java/com/flipkart/ranger/core/finder/shardselector/ListShardSelectorTest.java new file mode 100644 index 00000000..1f747c24 --- /dev/null +++ b/ranger-core/src/test/java/com/flipkart/ranger/core/finder/shardselector/ListShardSelectorTest.java @@ -0,0 +1,39 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.core.finder.shardselector; + +import com.flipkart.ranger.core.finder.serviceregistry.ListBasedServiceRegistry; +import com.flipkart.ranger.core.model.Criteria; +import com.flipkart.ranger.core.model.ServiceNode; +import com.flipkart.ranger.core.units.TestNodeData; +import com.flipkart.ranger.core.utils.CriteriaUtils; +import com.flipkart.ranger.core.utils.RegistryTestUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.util.List; + +public class ListShardSelectorTest { + + @Test + public void testNoOpShardSelector(){ + final ListBasedServiceRegistry serviceRegistry = RegistryTestUtils.getUnshardedRegistry(); + final ListShardSelector> shardSelector = new ListShardSelector<>(); + final List> nodes = shardSelector.nodes(CriteriaUtils.getCriteria(1), serviceRegistry); + Assert.assertFalse(nodes.isEmpty()); + Assert.assertEquals("localhost-1", nodes.get(0).getHost()); + } +} diff --git a/ranger-core/src/test/java/com/flipkart/ranger/core/finder/shardselector/MatchingShardSelectorTest.java b/ranger-core/src/test/java/com/flipkart/ranger/core/finder/shardselector/MatchingShardSelectorTest.java new file mode 100644 index 00000000..ce53b3f9 --- /dev/null +++ b/ranger-core/src/test/java/com/flipkart/ranger/core/finder/shardselector/MatchingShardSelectorTest.java @@ -0,0 +1,46 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.core.finder.shardselector; + +import com.flipkart.ranger.core.finder.serviceregistry.MapBasedServiceRegistry; +import com.flipkart.ranger.core.model.Criteria; +import com.flipkart.ranger.core.model.ServiceNode; +import com.flipkart.ranger.core.units.TestNodeData; +import com.flipkart.ranger.core.utils.RegistryTestUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.util.List; + +public class MatchingShardSelectorTest { + + private static final class TestCriteria implements Criteria{ + @Override + public boolean apply(TestNodeData nodeData) { + return nodeData.getNodeId() == 1; + } + } + + @Test + public void testMatchingShardSelector(){ + final MapBasedServiceRegistry serviceRegistry = RegistryTestUtils.getServiceRegistry(); + final MatchingShardSelector> shardSelector = new MatchingShardSelector<>(); + final List> nodes = shardSelector.nodes( + new TestCriteria(), serviceRegistry); + Assert.assertFalse(nodes.isEmpty()); + Assert.assertEquals("localhost-1", nodes.get(0).getHost()); + } +} diff --git a/ranger-core/src/test/java/com/flipkart/ranger/core/serviceprovider/ServiceProviderTest.java b/ranger-core/src/test/java/com/flipkart/ranger/core/serviceprovider/ServiceProviderTest.java new file mode 100644 index 00000000..50e470a7 --- /dev/null +++ b/ranger-core/src/test/java/com/flipkart/ranger/core/serviceprovider/ServiceProviderTest.java @@ -0,0 +1,136 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.core.serviceprovider; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.flipkart.ranger.core.healthcheck.Healthchecks; +import com.flipkart.ranger.core.model.NodeDataSink; +import com.flipkart.ranger.core.model.Serializer; +import com.flipkart.ranger.core.model.Service; +import com.flipkart.ranger.core.model.ServiceNode; +import com.flipkart.ranger.core.units.TestNodeData; +import org.junit.Assert; +import org.junit.Test; + +public class ServiceProviderTest { + + static TestNodeData testNodeData = null; + + interface TestSerializer extends Serializer { + byte[] serialize(final ServiceNode node); + } + + static class TestSerializerImpl implements TestSerializer{ + private ObjectMapper objectMapper; + + public TestSerializerImpl(){ + objectMapper = new ObjectMapper(); + } + + @Override + public byte[] serialize(ServiceNode node) { + try{ + return objectMapper.writeValueAsBytes(node); + }catch (JsonProcessingException jpe){ + return null; + } + } + } + + static class TestNodeDataSink> implements NodeDataSink{ + + public TestNodeDataSink(){ + } + + @Override + public void updateState(S serializer, ServiceNode serviceNode) { + testNodeData = (TestNodeData) serviceNode.getNodeData(); + } + + @Override + public void start() { + + } + + @Override + public void ensureConnected() { + + } + + @Override + public void stop() { + + } + + @Override + public boolean isActive() { + return true; + } + } + + public class TestServiceProviderBuilder extends BaseServiceProviderBuilder, TestSerializer> { + + @Override + public ServiceProvider> build() { + return super.buildProvider(); + } + + @Override + protected NodeDataSink> dataSink(Service service) { + return new TestNodeDataSink<>(); + } + } + + @Test(expected = NullPointerException.class) + public void testInvalidServiceProvider(){ + new TestServiceProviderBuilder() + .withServiceName("test-service") + .withNamespace("test") + .withHostname("localhost-1") + .withPort(9000) + .build(); + } + + @Test(expected = IllegalArgumentException.class) + public void testInvalidServiceProviderNoHealthCheck(){ + new TestServiceProviderBuilder() + .withServiceName("test-service") + .withNamespace("test") + .withHostname("localhost-1") + .withPort(9000) + .withSerializer(new TestSerializerImpl()) + .build(); + } + + @Test + public void testBuildServiceProvider(){ + ServiceProvider> testProvider = new TestServiceProviderBuilder() + .withServiceName("test-service") + .withNamespace("test") + .withHostname("localhost-1") + .withPort(9000) + .withSerializer(new TestSerializerImpl()) + .withNodeData(TestNodeData.builder().nodeId(1).build()) + .withHealthcheck(Healthchecks.defaultHealthyCheck()) + .withHealthUpdateIntervalMs(1000) + .build(); + testProvider.start(); + Assert.assertNotNull(testNodeData); + Assert.assertEquals(1, testNodeData.getNodeId()); + } + +} diff --git a/ranger-core/src/test/java/com/flipkart/ranger/core/units/TestNodeData.java b/ranger-core/src/test/java/com/flipkart/ranger/core/units/TestNodeData.java new file mode 100644 index 00000000..c38eaa19 --- /dev/null +++ b/ranger-core/src/test/java/com/flipkart/ranger/core/units/TestNodeData.java @@ -0,0 +1,47 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.core.units; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class TestNodeData { + @JsonProperty + private int nodeId; + + @Override + public boolean equals(Object obj){ + if(this == obj) + return true; + if(obj == null || obj.getClass()!= this.getClass()) + return false; + TestNodeData that = (TestNodeData) obj; + return (that.nodeId == this.nodeId); + } + + @Override + public int hashCode(){ + return this.nodeId; + } + +} diff --git a/ranger-core/src/test/java/com/flipkart/ranger/core/utils/CriteriaUtils.java b/ranger-core/src/test/java/com/flipkart/ranger/core/utils/CriteriaUtils.java new file mode 100644 index 00000000..6231dce6 --- /dev/null +++ b/ranger-core/src/test/java/com/flipkart/ranger/core/utils/CriteriaUtils.java @@ -0,0 +1,26 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.core.utils; + +import com.flipkart.ranger.core.model.Criteria; +import com.flipkart.ranger.core.units.TestNodeData; + +public class CriteriaUtils { + + public static Criteria getCriteria(int shardId){ + return nodeData -> nodeData.getNodeId() == shardId; + } +} diff --git a/ranger-core/src/test/java/com/flipkart/ranger/core/utils/RegistryTestUtils.java b/ranger-core/src/test/java/com/flipkart/ranger/core/utils/RegistryTestUtils.java new file mode 100644 index 00000000..5ea19537 --- /dev/null +++ b/ranger-core/src/test/java/com/flipkart/ranger/core/utils/RegistryTestUtils.java @@ -0,0 +1,50 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.core.utils; + +import com.flipkart.ranger.core.finder.serviceregistry.ListBasedServiceRegistry; +import com.flipkart.ranger.core.finder.serviceregistry.MapBasedServiceRegistry; +import com.flipkart.ranger.core.model.Service; +import com.flipkart.ranger.core.model.ServiceNode; +import com.flipkart.ranger.core.units.TestNodeData; +import com.google.common.collect.Lists; + +import java.util.List; + +public class RegistryTestUtils { + + public static MapBasedServiceRegistry getServiceRegistry(){ + final Service service = new Service("test", "test-service"); + final MapBasedServiceRegistry serviceRegistry = new MapBasedServiceRegistry<>(service); + List> serviceNodes = Lists.newArrayList(); + serviceNodes.add(new ServiceNode<>("localhost-1", 9000, TestNodeData.builder().nodeId(1).build())); + serviceNodes.add(new ServiceNode<>("localhost-2", 9001, TestNodeData.builder().nodeId(2).build())); + serviceNodes.add(new ServiceNode<>("localhost-3", 9002, TestNodeData.builder().nodeId(3).build())); + serviceRegistry.updateNodes(serviceNodes); + return serviceRegistry; + } + + public static ListBasedServiceRegistry getUnshardedRegistry(){ + final Service service = new Service("test", "test-service"); + final ListBasedServiceRegistry serviceRegistry = new ListBasedServiceRegistry<>(service); + List> serviceNodes = Lists.newArrayList(); + serviceNodes.add(new ServiceNode<>("localhost-1", 9000, TestNodeData.builder().nodeId(1).build())); + serviceNodes.add(new ServiceNode<>("localhost-2", 9001, TestNodeData.builder().nodeId(2).build())); + serviceNodes.add(new ServiceNode<>("localhost-3", 9002, TestNodeData.builder().nodeId(3).build())); + serviceRegistry.updateNodes(serviceNodes); + return serviceRegistry; + } +} diff --git a/ranger-core/src/test/java/com/flipkart/ranger/core/utils/TestUtils.java b/ranger-core/src/test/java/com/flipkart/ranger/core/utils/TestUtils.java deleted file mode 100644 index cb98cc62..00000000 --- a/ranger-core/src/test/java/com/flipkart/ranger/core/utils/TestUtils.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.flipkart.ranger.core.utils; - -import java.util.concurrent.TimeUnit; - -import static org.awaitility.Awaitility.await; - -/** - * - */ -public class TestUtils { - public static void sleepForSeconds(int numSeconds) { - await().pollDelay(numSeconds, TimeUnit.SECONDS).until(() -> true); - } -} diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/HttpServiceFinderBuilders.java b/ranger-http/src/main/java/com/flipkart/ranger/http/HttpServiceFinderBuilders.java new file mode 100644 index 00000000..f578937d --- /dev/null +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/HttpServiceFinderBuilders.java @@ -0,0 +1,35 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.http; + +import com.flipkart.ranger.core.model.Criteria; +import com.flipkart.ranger.http.servicefinder.HttpShardedServiceFinderBuilder; +import com.flipkart.ranger.http.servicefinder.HttpUnshardedServiceFinderBuilider; + +public class HttpServiceFinderBuilders { + + private void HttpServiceProviderBuilders(){ + throw new InstantiationError("Must not instantiate this class"); + } + + public static > HttpShardedServiceFinderBuilder httpShardedServiceFinderBuilder(){ + return new HttpShardedServiceFinderBuilder<>(); + } + + public static > HttpUnshardedServiceFinderBuilider httpUnshardedServiceFinderBuilider(){ + return new HttpUnshardedServiceFinderBuilider<>(); + } +} diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/HttpServiceProviderBuilders.java b/ranger-http/src/main/java/com/flipkart/ranger/http/HttpServiceProviderBuilders.java new file mode 100644 index 00000000..605b3b36 --- /dev/null +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/HttpServiceProviderBuilders.java @@ -0,0 +1,29 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.http; + +import com.flipkart.ranger.http.serviceprovider.HttpShardedServiceProviderBuilder; + +public class HttpServiceProviderBuilders { + + private HttpServiceProviderBuilders() { + throw new InstantiationError("Must not instantiate this class"); + } + + public static HttpShardedServiceProviderBuilder httpServiceProviderBuilder() { + return new HttpShardedServiceProviderBuilder<>(); + } +} diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/HttpShardedServiceFinderBuilder.java b/ranger-http/src/main/java/com/flipkart/ranger/http/HttpShardedServiceFinderBuilder.java deleted file mode 100644 index 042f56e2..00000000 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/HttpShardedServiceFinderBuilder.java +++ /dev/null @@ -1,32 +0,0 @@ -package com.flipkart.ranger.http; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.flipkart.ranger.core.finder.sharded.SimpleShardedServiceFinder; -import com.flipkart.ranger.core.finder.sharded.SimpleShardedServiceFinderBuilder; -import com.flipkart.ranger.core.model.NodeDataSource; -import com.flipkart.ranger.core.model.Service; -import com.flipkart.ranger.http.config.HttpClientConfig; -import com.flipkart.ranger.http.serde.HTTPResponseDataDeserializer; - -/** - * - */ -public class HttpShardedServiceFinderBuilder> extends SimpleShardedServiceFinderBuilder, D> { - - private HttpClientConfig clientConfig; - - public HttpShardedServiceFinderBuilder withClientConfig(final HttpClientConfig clientConfig) { - this.clientConfig = clientConfig; - return this; - } - - @Override - public SimpleShardedServiceFinder build() { - return buildFinder(); - } - - @Override - protected NodeDataSource dataSource(Service service) { - return new HttpNodeDataSource<>(service, clientConfig, new ObjectMapper()); - } -} diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/HttpNodeDataStoreConnector.java b/ranger-http/src/main/java/com/flipkart/ranger/http/common/HttpNodeDataStoreConnector.java similarity index 68% rename from ranger-http/src/main/java/com/flipkart/ranger/http/HttpNodeDataStoreConnector.java rename to ranger-http/src/main/java/com/flipkart/ranger/http/common/HttpNodeDataStoreConnector.java index ee0e61be..4cb3bd7e 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/HttpNodeDataStoreConnector.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/common/HttpNodeDataStoreConnector.java @@ -1,15 +1,28 @@ -package com.flipkart.ranger.http; +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.http.common; import com.fasterxml.jackson.databind.ObjectMapper; import com.flipkart.ranger.core.model.NodeDataStoreConnector; -import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.http.config.HttpClientConfig; import lombok.extern.slf4j.Slf4j; import okhttp3.ConnectionPool; import okhttp3.OkHttpClient; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; /** * @@ -17,17 +30,13 @@ @Slf4j public class HttpNodeDataStoreConnector implements NodeDataStoreConnector { - protected final Service service; protected final HttpClientConfig config; - protected final OkHttpClient httpClient; protected final ObjectMapper mapper; - protected final AtomicBoolean firstCall = new AtomicBoolean(false); + protected final OkHttpClient httpClient; public HttpNodeDataStoreConnector( - Service service, final HttpClientConfig config, ObjectMapper mapper) { - this.service = service; this.httpClient = new OkHttpClient.Builder() .callTimeout(config.getOperationTimeoutMs() == 0 ? 3000 @@ -39,7 +48,7 @@ public HttpNodeDataStoreConnector( .connectionPool(new ConnectionPool(1, 30, TimeUnit.SECONDS)) .build(); this.config = config; - this.mapper = mapper; + this.mapper = null != mapper ? mapper : new ObjectMapper(); } diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/config/HttpClientConfig.java b/ranger-http/src/main/java/com/flipkart/ranger/http/config/HttpClientConfig.java index ebac969d..bf1f2f4d 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/config/HttpClientConfig.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/config/HttpClientConfig.java @@ -1,19 +1,35 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.flipkart.ranger.http.config; -import lombok.Builder; -import lombok.Data; +import lombok.*; /** * */ @Data +@AllArgsConstructor @Builder +@ToString +@NoArgsConstructor public class HttpClientConfig { private String host; private int port; private boolean secure; private long connectionTimeoutMs; private long operationTimeoutMs; - private String nodesRequestPath; - + private long refreshIntervalMillis; } diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceDataSourceResponse.java b/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceDataSourceResponse.java new file mode 100644 index 00000000..59964417 --- /dev/null +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceDataSourceResponse.java @@ -0,0 +1,31 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.http.model; + +import com.flipkart.ranger.core.model.Service; +import lombok.*; + +import java.util.List; + +@AllArgsConstructor +@NoArgsConstructor +@Getter +@Setter +@Builder +public class ServiceDataSourceResponse { + private boolean success; + private List data; +} diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceNodesRequest.java b/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceNodesRequest.java deleted file mode 100644 index 548560a9..00000000 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceNodesRequest.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.flipkart.ranger.http.model; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Builder; -import lombok.Data; - -/** - * - */ -@Data -public class ServiceNodesRequest { - private final String serviceName; - - @Builder - public ServiceNodesRequest(@JsonProperty("serviceName") String serviceName) { - this.serviceName = serviceName; - } -} diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceNodesResponse.java b/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceNodesResponse.java index 2de6cc0d..0d713cff 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceNodesResponse.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceNodesResponse.java @@ -1,10 +1,24 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.flipkart.ranger.http.model; import com.fasterxml.jackson.annotation.JsonProperty; import com.flipkart.ranger.core.model.ServiceNode; import lombok.Builder; import lombok.Data; -import lombok.Singular; import java.util.List; @@ -12,15 +26,15 @@ * */ @Data -public class ServiceNodesResponse { +public class ServiceNodesResponse { private final boolean success; - final List> nodes; + final List> data; @Builder public ServiceNodesResponse( @JsonProperty("success") boolean success, - @JsonProperty("nodes") @Singular List> nodes) { + @JsonProperty("data") List> data) { this.success = success; - this.nodes = nodes; + this.data = data; } } diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceRegistrationResponse.java b/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceRegistrationResponse.java new file mode 100644 index 00000000..897d760e --- /dev/null +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceRegistrationResponse.java @@ -0,0 +1,32 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.http.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Builder; +import lombok.Data; + +@Data +public class ServiceRegistrationResponse { + private final boolean success; + + @Builder + public ServiceRegistrationResponse( + @JsonProperty("success") boolean success + ) { + this.success = success; + } +} diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/serde/HTTPResponseDataDeserializer.java b/ranger-http/src/main/java/com/flipkart/ranger/http/serde/HTTPResponseDataDeserializer.java index 3ece5323..df018a0c 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/serde/HTTPResponseDataDeserializer.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/serde/HTTPResponseDataDeserializer.java @@ -1,14 +1,27 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.flipkart.ranger.http.serde; import com.flipkart.ranger.core.model.Deserializer; -import com.flipkart.ranger.core.model.ServiceNode; - -import java.util.Collection; +import com.flipkart.ranger.http.model.ServiceNodesResponse; /** * */ @FunctionalInterface public interface HTTPResponseDataDeserializer extends Deserializer { - Collection> deserialize(byte []data); + ServiceNodesResponse deserialize(byte []data); } diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/unsharded/NoOpShardSelector.java b/ranger-http/src/main/java/com/flipkart/ranger/http/serde/HttpRequestDataSerializer.java similarity index 56% rename from ranger-core/src/main/java/com/flipkart/ranger/core/finder/unsharded/NoOpShardSelector.java rename to ranger-http/src/main/java/com/flipkart/ranger/http/serde/HttpRequestDataSerializer.java index 3718eb50..9a1d5184 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/unsharded/NoOpShardSelector.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/serde/HttpRequestDataSerializer.java @@ -1,30 +1,26 @@ -/** +/* * Copyright 2015 Flipkart Internet Pvt. Ltd. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ +package com.flipkart.ranger.http.serde; -package com.flipkart.ranger.core.finder.unsharded; - -import com.flipkart.ranger.core.model.ShardSelector; +import com.flipkart.ranger.core.model.Serializer; import com.flipkart.ranger.core.model.ServiceNode; -import java.util.List; +@FunctionalInterface +public interface HttpRequestDataSerializer extends Serializer { + + byte[] serialize(final ServiceNode node); -public class NoOpShardSelector implements ShardSelector { - @Override - public List> nodes(UnshardedClusterInfo criteria, - UnshardedClusterServiceRegistry serviceRegistry) { - return serviceRegistry.nodes(); - } } diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/serde/JacksonHTTPResponseDataDeserializer.java b/ranger-http/src/main/java/com/flipkart/ranger/http/serde/JacksonHTTPResponseDataDeserializer.java deleted file mode 100644 index 64ff3600..00000000 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/serde/JacksonHTTPResponseDataDeserializer.java +++ /dev/null @@ -1,33 +0,0 @@ -package com.flipkart.ranger.http.serde; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.flipkart.ranger.core.model.ServiceNode; -import com.flipkart.ranger.http.model.ServiceNodesResponse; - -import java.io.IOException; -import java.util.Collection; -import java.util.Collections; - -/** - * - */ -public class JacksonHTTPResponseDataDeserializer implements HTTPResponseDataDeserializer { - private final ObjectMapper mapper; - - public JacksonHTTPResponseDataDeserializer(ObjectMapper mapper) { - this.mapper = mapper; - } - - @Override - public Collection> deserialize(byte[] data) { - try { - final ServiceNodesResponse response - = mapper.readValue(data, new TypeReference>() {}); - return response.isSuccess() ? response.getNodes() : Collections.emptyList(); - } - catch (IOException e) { - throw new IllegalArgumentException(e); - } - } -} diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/HttpNodeDataSource.java b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpNodeDataSource.java similarity index 54% rename from ranger-http/src/main/java/com/flipkart/ranger/http/HttpNodeDataSource.java rename to ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpNodeDataSource.java index 9ec4d2d7..7ca0bfc0 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/HttpNodeDataSource.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpNodeDataSource.java @@ -1,4 +1,19 @@ -package com.flipkart.ranger.http; +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.http.servicefinder; import com.fasterxml.jackson.databind.ObjectMapper; import com.flipkart.ranger.core.model.NodeDataSource; @@ -6,8 +21,11 @@ import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.util.Exceptions; import com.flipkart.ranger.core.util.FinderUtils; +import com.flipkart.ranger.http.common.HttpNodeDataStoreConnector; import com.flipkart.ranger.http.config.HttpClientConfig; +import com.flipkart.ranger.http.model.ServiceNodesResponse; import com.flipkart.ranger.http.serde.HTTPResponseDataDeserializer; +import com.google.common.base.Preconditions; import lombok.extern.slf4j.Slf4j; import lombok.val; import okhttp3.HttpUrl; @@ -23,53 +41,60 @@ */ @Slf4j public class HttpNodeDataSource> extends HttpNodeDataStoreConnector implements NodeDataSource { + + private final Service service; + public HttpNodeDataSource( Service service, final HttpClientConfig config, ObjectMapper mapper) { - super(service, config, mapper); + super(config, mapper); + this.service = service; } - @Override public Optional>> refresh(D deserializer) { + Preconditions.checkNotNull(config, "client config has not been set for node data"); + Preconditions.checkNotNull(mapper, "mapper has not been set for node data"); val httpUrl = new HttpUrl.Builder() .scheme(config.isSecure() ? "https" : "http") .host(config.getHost()) .port(config.getPort() == 0 - ? defaultPort() - : config.getPort()) - .encodedPath(String.format("/ranger/nodes/v1/%s/%s", service.getNamespace(), service.getServiceName())) + ? defaultPort() + : config.getPort()) + .encodedPath(String.format("/v1/ranger/nodes/%s/%s", service.getNamespace(), service.getServiceName())) .build(); val request = new Request.Builder() .url(httpUrl) .get() .build(); + ServiceNodesResponse serviceNodesResponse = null; try (val response = httpClient.newCall(request).execute()) { if (response.isSuccessful()) { try (final ResponseBody body = response.body()) { if (null == body) { log.warn("HTTP call to {} returned empty body", httpUrl); - } - else { + } else { final byte[] bytes = body.bytes(); - return Optional.of(FinderUtils.filterValidNodes( - service, - deserializer.deserialize(bytes), - healthcheckZombieCheckThresholdTime(service))); + serviceNodesResponse = deserializer.deserialize(bytes); } } + } else { + log.warn("HTTP call to {} returned: {}", httpUrl, response.code()); } - else { - log.warn("HTTP call to {} returned: {}", httpUrl.toString(), response.code()); - } - } - catch (IOException e) { + } catch (IOException e) { Exceptions.illegalState("Error fetching data from server: " + httpUrl, e); } - log.error("No data returned from server: " + httpUrl); + + if (null != serviceNodesResponse && null != serviceNodesResponse.getData() && + !serviceNodesResponse.getData().isEmpty()) { + return Optional.of(FinderUtils.filterValidNodes( + service, + serviceNodesResponse.getData(), + healthcheckZombieCheckThresholdTime(service))); + } return Optional.empty(); } diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilder.java b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilder.java new file mode 100644 index 00000000..bdb3b487 --- /dev/null +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilder.java @@ -0,0 +1,55 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.http.servicefinder; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.flipkart.ranger.core.finder.SimpleShardedServiceFinder; +import com.flipkart.ranger.core.finder.SimpleShardedServiceFinderBuilder; +import com.flipkart.ranger.core.model.Criteria; +import com.flipkart.ranger.core.model.NodeDataSource; +import com.flipkart.ranger.core.model.Service; +import com.flipkart.ranger.http.config.HttpClientConfig; +import com.flipkart.ranger.http.serde.HTTPResponseDataDeserializer; + +/** + * + */ +public class HttpShardedServiceFinderBuilder> extends SimpleShardedServiceFinderBuilder, HTTPResponseDataDeserializer, C> { + + private HttpClientConfig clientConfig; + private ObjectMapper mapper; + + public HttpShardedServiceFinderBuilder withClientConfig(final HttpClientConfig clientConfig) { + this.clientConfig = clientConfig; + return this; + } + + public HttpShardedServiceFinderBuilder withObjectMapper(final ObjectMapper mapper){ + this.mapper = mapper; + return this; + } + + @Override + public SimpleShardedServiceFinder build() { + return buildFinder(); + } + + @Override + protected NodeDataSource> dataSource(Service service) { + return new HttpNodeDataSource<>(service, clientConfig, mapper); + } + +} diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpUnshardedServiceFinderBuilider.java b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpUnshardedServiceFinderBuilider.java new file mode 100644 index 00000000..fbc1ece8 --- /dev/null +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpUnshardedServiceFinderBuilider.java @@ -0,0 +1,54 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.http.servicefinder; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.flipkart.ranger.core.finder.SimpleUnshardedServiceFinder; +import com.flipkart.ranger.core.finder.SimpleUnshardedServiceFinderBuilder; +import com.flipkart.ranger.core.model.Criteria; +import com.flipkart.ranger.core.model.NodeDataSource; +import com.flipkart.ranger.core.model.Service; +import com.flipkart.ranger.http.config.HttpClientConfig; +import com.flipkart.ranger.http.serde.HTTPResponseDataDeserializer; + +public class HttpUnshardedServiceFinderBuilider> + extends SimpleUnshardedServiceFinderBuilder, HTTPResponseDataDeserializer, C> { + + private HttpClientConfig clientConfig; + private ObjectMapper mapper; + + public HttpUnshardedServiceFinderBuilider withClientConfig(final HttpClientConfig clientConfig) { + this.clientConfig = clientConfig; + return this; + } + + public HttpUnshardedServiceFinderBuilider withObjectMapper(final ObjectMapper mapper) { + this.mapper = mapper; + return this; + } + + @Override + public SimpleUnshardedServiceFinder build() { + return buildFinder(); + } + + @Override + protected NodeDataSource> dataSource(Service service) { + return new HttpNodeDataSource<>(service, clientConfig, mapper); + } + +} + diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSource.java b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSource.java new file mode 100644 index 00000000..cb0af3f7 --- /dev/null +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSource.java @@ -0,0 +1,89 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.http.servicefinderhub; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.flipkart.ranger.core.finderhub.ServiceDataSource; +import com.flipkart.ranger.core.model.Service; +import com.flipkart.ranger.core.util.Exceptions; +import com.flipkart.ranger.http.common.HttpNodeDataStoreConnector; +import com.flipkart.ranger.http.config.HttpClientConfig; +import com.flipkart.ranger.http.model.ServiceDataSourceResponse; +import com.google.common.base.Preconditions; +import lombok.extern.slf4j.Slf4j; +import lombok.val; +import okhttp3.HttpUrl; +import okhttp3.Request; +import okhttp3.ResponseBody; + +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; + +@Slf4j +public class HttpServiceDataSource extends HttpNodeDataStoreConnector implements ServiceDataSource { + + public HttpServiceDataSource(HttpClientConfig config, ObjectMapper mapper) { + super(config, mapper); + } + + @Override + public Collection services() { + Preconditions.checkNotNull(config, "client config has not been set for node data"); + Preconditions.checkNotNull(mapper, "mapper has not been set for node data"); + + val httpUrl = new HttpUrl.Builder() + .scheme(config.isSecure() + ? "https" + : "http") + .host(config.getHost()) + .port(config.getPort() == 0 + ? defaultPort() + : config.getPort()) + .encodedPath("/v1/ranger/services") + .build(); + val request = new Request.Builder() + .url(httpUrl) + .get() + .build(); + + try (val response = httpClient.newCall(request).execute()) { + if (response.isSuccessful()) { + try (final ResponseBody body = response.body()) { + if (null == body) { + log.warn("HTTP call to {} returned empty body", httpUrl); + } + else { + final byte[] bytes = body.bytes(); + val serviceDataSourceResponse = mapper.readValue(bytes, ServiceDataSourceResponse.class); + if(serviceDataSourceResponse.isSuccess()){ + return serviceDataSourceResponse.getData(); + } + } + } + } + else { + log.warn("HTTP call to {} returned: {}", httpUrl, response.code()); + } + } + catch (IOException e) { + Exceptions.illegalState("Error fetching data from server: " + httpUrl, e); + } + + log.error("No data returned from server: " + httpUrl); + return Collections.emptyList(); + } +} diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceFinderHubBuilder.java b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceFinderHubBuilder.java new file mode 100644 index 00000000..2c100288 --- /dev/null +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceFinderHubBuilder.java @@ -0,0 +1,36 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.http.servicefinderhub; + +import com.flipkart.ranger.core.finderhub.ServiceFinderHub; +import com.flipkart.ranger.core.finderhub.ServiceFinderHubBuilder; +import com.flipkart.ranger.core.model.Criteria; +import com.flipkart.ranger.core.model.ServiceRegistry; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class HttpServiceFinderHubBuilder, R extends ServiceRegistry> extends ServiceFinderHubBuilder { + + @Override + protected void preBuild() { + log.info("No pre-build actions necessary"); + } + + @Override + protected void postBuild(ServiceFinderHub serviceFinderHub) { + log.info("No post build actions necessary"); + } +} diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpShardedServiceFinderFactory.java b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpShardedServiceFinderFactory.java new file mode 100644 index 00000000..c5f7c340 --- /dev/null +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpShardedServiceFinderFactory.java @@ -0,0 +1,75 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.http.servicefinderhub; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.flipkart.ranger.core.finder.ServiceFinder; +import com.flipkart.ranger.core.finder.serviceregistry.MapBasedServiceRegistry; +import com.flipkart.ranger.core.finderhub.ServiceFinderFactory; +import com.flipkart.ranger.core.model.Criteria; +import com.flipkart.ranger.core.model.Service; +import com.flipkart.ranger.core.model.ServiceNodeSelector; +import com.flipkart.ranger.core.model.ShardSelector; +import com.flipkart.ranger.http.config.HttpClientConfig; +import com.flipkart.ranger.http.serde.HTTPResponseDataDeserializer; +import com.flipkart.ranger.http.servicefinder.HttpShardedServiceFinderBuilder; +import lombok.Builder; +import lombok.Getter; +import lombok.val; + +@Getter +public class HttpShardedServiceFinderFactory > implements ServiceFinderFactory> { + + private final HttpClientConfig clientConfig; + private final ObjectMapper mapper; + private final HTTPResponseDataDeserializer deserializer; + private final ShardSelector> shardSelector; + private final ServiceNodeSelector nodeSelector; + private final int nodeRefreshIntervalMs; + + @Builder + public HttpShardedServiceFinderFactory( + HttpClientConfig httpClientConfig, + ObjectMapper mapper, + HTTPResponseDataDeserializer deserializer, + ShardSelector> shardSelector, + ServiceNodeSelector nodeSelector, + int nodeRefreshIntervalMs) + { + this.clientConfig = httpClientConfig; + this.mapper = mapper; + this.deserializer = deserializer; + this.shardSelector = shardSelector; + this.nodeSelector = nodeSelector; + this.nodeRefreshIntervalMs = nodeRefreshIntervalMs; + } + + @Override + public ServiceFinder> buildFinder(Service service) { + val serviceFinder = new HttpShardedServiceFinderBuilder() + .withClientConfig(clientConfig) + .withObjectMapper(mapper) + .withDeserializer(deserializer) + .withNamespace(service.getNamespace()) + .withServiceName(service.getServiceName()) + .withNodeRefreshIntervalMs(nodeRefreshIntervalMs) + .withShardSelector(shardSelector) + .withNodeSelector(nodeSelector) + .build(); + serviceFinder.start(); + return serviceFinder; + } +} diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpUnshardedServiceFinderFactory.java b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpUnshardedServiceFinderFactory.java new file mode 100644 index 00000000..eeb63c76 --- /dev/null +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpUnshardedServiceFinderFactory.java @@ -0,0 +1,75 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.http.servicefinderhub; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.flipkart.ranger.core.finder.ServiceFinder; +import com.flipkart.ranger.core.finder.serviceregistry.ListBasedServiceRegistry; +import com.flipkart.ranger.core.finderhub.ServiceFinderFactory; +import com.flipkart.ranger.core.model.Criteria; +import com.flipkart.ranger.core.model.Service; +import com.flipkart.ranger.core.model.ServiceNodeSelector; +import com.flipkart.ranger.core.model.ShardSelector; +import com.flipkart.ranger.http.config.HttpClientConfig; +import com.flipkart.ranger.http.serde.HTTPResponseDataDeserializer; +import com.flipkart.ranger.http.servicefinder.HttpUnshardedServiceFinderBuilider; +import lombok.Builder; +import lombok.Getter; +import lombok.val; + +@Getter +public class HttpUnshardedServiceFinderFactory> implements ServiceFinderFactory> { + + private final HttpClientConfig clientConfig; + private final ObjectMapper mapper; + private final HTTPResponseDataDeserializer deserializer; + private final ShardSelector> shardSelector; + private final ServiceNodeSelector nodeSelector; + private final int nodeRefreshIntervalMs; + + @Builder + public HttpUnshardedServiceFinderFactory( + HttpClientConfig httpClientConfig, + ObjectMapper mapper, + HTTPResponseDataDeserializer deserializer, + ShardSelector> shardSelector, + ServiceNodeSelector nodeSelector, + int nodeRefreshIntervalMs) + { + this.clientConfig = httpClientConfig; + this.mapper = mapper; + this.deserializer = deserializer; + this.shardSelector = shardSelector; + this.nodeSelector = nodeSelector; + this.nodeRefreshIntervalMs = nodeRefreshIntervalMs; + } + + @Override + public ServiceFinder> buildFinder(Service service) { + val serviceFinder = new HttpUnshardedServiceFinderBuilider() + .withClientConfig(clientConfig) + .withObjectMapper(mapper) + .withDeserializer(deserializer) + .withNamespace(service.getNamespace()) + .withServiceName(service.getServiceName()) + .withNodeRefreshIntervalMs(nodeRefreshIntervalMs) + .withShardSelector(shardSelector) + .withNodeSelector(nodeSelector) + .build(); + serviceFinder.start(); + return serviceFinder; + } +} diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/serviceprovider/HttpNodeDataSink.java b/ranger-http/src/main/java/com/flipkart/ranger/http/serviceprovider/HttpNodeDataSink.java new file mode 100644 index 00000000..92263bf8 --- /dev/null +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/serviceprovider/HttpNodeDataSink.java @@ -0,0 +1,98 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.http.serviceprovider; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.flipkart.ranger.core.model.NodeDataSink; +import com.flipkart.ranger.core.model.Service; +import com.flipkart.ranger.core.model.ServiceNode; +import com.flipkart.ranger.core.util.Exceptions; +import com.flipkart.ranger.http.common.HttpNodeDataStoreConnector; +import com.flipkart.ranger.http.config.HttpClientConfig; +import com.flipkart.ranger.http.model.ServiceRegistrationResponse; +import com.flipkart.ranger.http.serde.HttpRequestDataSerializer; +import com.google.common.base.Preconditions; +import lombok.extern.slf4j.Slf4j; +import lombok.val; +import okhttp3.HttpUrl; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.ResponseBody; + +import java.io.IOException; +import java.util.Optional; + +@Slf4j +public class HttpNodeDataSink> extends HttpNodeDataStoreConnector implements NodeDataSink { + + private final Service service; + + public HttpNodeDataSink(Service service, HttpClientConfig config, ObjectMapper mapper) { + super(config, mapper); + this.service = service; + } + + @Override + public void updateState(S serializer, ServiceNode serviceNode) { + Preconditions.checkNotNull(config, "client config has not been set for node data"); + Preconditions.checkNotNull(mapper, "mapper has not been set for node data"); + + val httpUrl = new HttpUrl.Builder() + .scheme(config.isSecure() + ? "https" + : "http") + .host(config.getHost()) + .port(config.getPort() == 0 + ? defaultPort() + : config.getPort()) + .encodedPath(String.format("/v1/ranger/nodes/add/%s/%s", service.getNamespace(), service.getServiceName())) + .build(); + val requestBody = RequestBody.create(serializer.serialize(serviceNode)); + val serviceRegistrationResponse = registerService(httpUrl, requestBody); + if(!serviceRegistrationResponse.isPresent() || !serviceRegistrationResponse.get().isSuccess()){ + Exceptions.illegalState("Error updating state on the server for nodedata: " + httpUrl); + } + } + + private Optional registerService(HttpUrl httpUrl, RequestBody requestBody){ + val request = new Request.Builder() + .url(httpUrl) + .post(requestBody) + .build(); + try (val response = httpClient.newCall(request).execute()) { + if (response.isSuccessful()) { + try (final ResponseBody body = response.body()) { + if (null == body) { + log.warn("HTTP call to {} returned empty body", httpUrl); + } + else { + final byte[] bytes = body.bytes(); + return Optional.of( + mapper.readValue(bytes, ServiceRegistrationResponse.class) + ); + } + } + } + else { + log.warn("HTTP call to {} has returned: {}", httpUrl, response.code()); + } + } + catch (IOException e) { + Exceptions.illegalState("Error updating state on the server: " + httpUrl, e); + } + return Optional.empty(); + } +} diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/serviceprovider/HttpShardedServiceProviderBuilder.java b/ranger-http/src/main/java/com/flipkart/ranger/http/serviceprovider/HttpShardedServiceProviderBuilder.java new file mode 100644 index 00000000..64160015 --- /dev/null +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/serviceprovider/HttpShardedServiceProviderBuilder.java @@ -0,0 +1,52 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.http.serviceprovider; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.flipkart.ranger.core.model.NodeDataSink; +import com.flipkart.ranger.core.model.Service; +import com.flipkart.ranger.core.serviceprovider.BaseServiceProviderBuilder; +import com.flipkart.ranger.core.serviceprovider.ServiceProvider; +import com.flipkart.ranger.http.config.HttpClientConfig; +import com.flipkart.ranger.http.serde.HttpRequestDataSerializer; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class HttpShardedServiceProviderBuilder extends BaseServiceProviderBuilder, HttpRequestDataSerializer> { + + private HttpClientConfig clientConfig; + private ObjectMapper mapper; + + public HttpShardedServiceProviderBuilder withClientConfiguration(final HttpClientConfig clientConfig) { + this.clientConfig = clientConfig; + return this; + } + + public HttpShardedServiceProviderBuilder withObjectMapper(final ObjectMapper mapper){ + this.mapper = mapper; + return this; + } + + @Override + public ServiceProvider> build() { + return super.buildProvider(); + } + + @Override + protected NodeDataSink> dataSink(Service service) { + return new HttpNodeDataSink<>(service, clientConfig, mapper); + } +} diff --git a/ranger-http/src/test/java/com/flipkart/ranger/http/ResourceHelper.java b/ranger-http/src/test/java/com/flipkart/ranger/http/ResourceHelper.java new file mode 100644 index 00000000..94dcacf0 --- /dev/null +++ b/ranger-http/src/test/java/com/flipkart/ranger/http/ResourceHelper.java @@ -0,0 +1,44 @@ +/** + * Copyright 2015 Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.http; + +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.SneakyThrows; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.stream.Collectors; + +public class ResourceHelper { + + private static final ObjectMapper objectMapper = new ObjectMapper(); + + public static String getResource(String path) { + final InputStream data = ResourceHelper.class.getClassLoader().getResourceAsStream(path); + return new BufferedReader( + new InputStreamReader(data)) + .lines() + .collect(Collectors.joining("\n")); + } + + @SneakyThrows + public static T getResource(String path, Class klass) { + final String data = getResource(path); + return objectMapper.readValue(data, klass); + } + +} diff --git a/ranger-http/src/test/java/com/flipkart/ranger/http/common/HttpNodeDataStoreConnectorTest.java b/ranger-http/src/test/java/com/flipkart/ranger/http/common/HttpNodeDataStoreConnectorTest.java new file mode 100644 index 00000000..caf6d5fc --- /dev/null +++ b/ranger-http/src/test/java/com/flipkart/ranger/http/common/HttpNodeDataStoreConnectorTest.java @@ -0,0 +1,36 @@ +/** + * Copyright 2015 Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.http.common; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.flipkart.ranger.http.config.HttpClientConfig; +import org.junit.Assert; +import org.junit.Test; + +public class HttpNodeDataStoreConnectorTest { + + @Test + public void testHttpNodeDataStoreConnector(){ + final ObjectMapper objectMapper = new ObjectMapper(); + final HttpClientConfig httpClientConfig = HttpClientConfig.builder() + .host("localhost-1") + .port(80) + .build(); + HttpNodeDataStoreConnector httpNodeDataStoreConnector = new HttpNodeDataStoreConnector(httpClientConfig, objectMapper); + Assert.assertNotNull(httpNodeDataStoreConnector); + Assert.assertTrue(httpNodeDataStoreConnector.isActive()); + } +} diff --git a/ranger-http/src/test/java/com/flipkart/ranger/http/config/HttpClientConfigTest.java b/ranger-http/src/test/java/com/flipkart/ranger/http/config/HttpClientConfigTest.java new file mode 100644 index 00000000..336fb925 --- /dev/null +++ b/ranger-http/src/test/java/com/flipkart/ranger/http/config/HttpClientConfigTest.java @@ -0,0 +1,32 @@ +/** + * Copyright 2015 Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.http.config; + +import com.flipkart.ranger.http.ResourceHelper; +import org.junit.Assert; +import org.junit.Test; + +public class HttpClientConfigTest { + + @Test + public void testHttpClientConfig(){ + HttpClientConfig resource = ResourceHelper.getResource("fixtures/httpClientConfig.json", HttpClientConfig.class); + Assert.assertEquals("localhost-1", resource.getHost()); + Assert.assertEquals(80, resource.getPort()); + Assert.assertEquals(10, resource.getConnectionTimeoutMs()); + Assert.assertEquals(10, resource.getOperationTimeoutMs()); + } +} diff --git a/ranger-http/src/test/java/com/flipkart/ranger/http/model/ServiceNodeResponseTest.java b/ranger-http/src/test/java/com/flipkart/ranger/http/model/ServiceNodeResponseTest.java new file mode 100644 index 00000000..8c1f6f40 --- /dev/null +++ b/ranger-http/src/test/java/com/flipkart/ranger/http/model/ServiceNodeResponseTest.java @@ -0,0 +1,45 @@ +/** + * Copyright 2015 Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.http.model; + +import com.flipkart.ranger.core.model.ServiceNode; +import com.flipkart.ranger.http.ResourceHelper; +import lombok.*; +import org.junit.Assert; +import org.junit.Test; + +public class ServiceNodeResponseTest { + + @AllArgsConstructor + @NoArgsConstructor + @Getter + @Setter + static class TestNodeInfo{ + private int shardId; + private String region; + } + + @Test + public void testServiceNodesResponse(){ + val serviceNodesResponse = ResourceHelper.getResource("fixtures/serviceNodesResponse.json", ServiceNodesResponse.class); + Assert.assertNotNull(serviceNodesResponse); + Assert.assertFalse(serviceNodesResponse.getData().isEmpty()); + Assert.assertNotNull(((ServiceNode) serviceNodesResponse.getData().get(0)).getNodeData()); + Assert.assertNotNull(((ServiceNode) serviceNodesResponse.getData().get(1)).getNodeData()); + Assert.assertEquals(((ServiceNode) serviceNodesResponse.getData().get(0)).getHost(), "localhost-1"); + Assert.assertEquals(((ServiceNode) serviceNodesResponse.getData().get(1)).getHost(), "localhost-2"); + } +} diff --git a/ranger-http/src/test/java/com/flipkart/ranger/http/model/ServiceRegistrationResponseTest.java b/ranger-http/src/test/java/com/flipkart/ranger/http/model/ServiceRegistrationResponseTest.java new file mode 100644 index 00000000..42025c66 --- /dev/null +++ b/ranger-http/src/test/java/com/flipkart/ranger/http/model/ServiceRegistrationResponseTest.java @@ -0,0 +1,30 @@ +/** + * Copyright 2015 Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.http.model; + +import com.flipkart.ranger.http.ResourceHelper; +import org.junit.Assert; +import org.junit.Test; + +public class ServiceRegistrationResponseTest { + + @Test + public void testServiceRegistrationResponse(){ + ServiceRegistrationResponse resource = ResourceHelper.getResource("fixtures/serviceResponse.json", ServiceRegistrationResponse.class); + Assert.assertNotNull(resource); + Assert.assertTrue(resource.isSuccess()); + } +} diff --git a/ranger-http/src/test/java/com/flipkart/ranger/http/HttpShardedServiceFinderBuilderTest.java b/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java similarity index 65% rename from ranger-http/src/test/java/com/flipkart/ranger/http/HttpShardedServiceFinderBuilderTest.java rename to ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java index 7415aaa4..e01acac9 100644 --- a/ranger-http/src/test/java/com/flipkart/ranger/http/HttpShardedServiceFinderBuilderTest.java +++ b/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java @@ -1,16 +1,32 @@ -package com.flipkart.ranger.http; +/** + * Copyright 2015 Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.http.servicefinder; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import com.flipkart.ranger.core.finder.sharded.SimpleShardedServiceFinder; +import com.flipkart.ranger.core.TestUtils; +import com.flipkart.ranger.core.finder.SimpleShardedServiceFinder; import com.flipkart.ranger.core.healthcheck.HealthcheckStatus; +import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.ServiceNode; -import com.flipkart.ranger.core.utils.TestUtils; import com.flipkart.ranger.http.config.HttpClientConfig; import com.flipkart.ranger.http.model.ServiceNodesResponse; -import com.flipkart.ranger.http.serde.HTTPResponseDataDeserializer; import com.github.tomakehurst.wiremock.junit.WireMockRule; +import com.google.common.collect.Lists; import lombok.Data; import lombok.val; import org.junit.Assert; @@ -18,9 +34,6 @@ import org.junit.Test; import java.io.IOException; -import java.util.Collections; -import java.util.Map; -import java.util.stream.Collectors; import static com.github.tomakehurst.wiremock.client.WireMock.*; @@ -51,10 +64,10 @@ public void testFinder() throws Exception { node.setLastUpdatedTimeStamp(System.currentTimeMillis()); val payload = MAPPER.writeValueAsBytes( ServiceNodesResponse.builder() - .node(node) + .data(Lists.newArrayList(node)) .success(true) .build()); - server.stubFor(get(urlEqualTo("/ranger/nodes/v1/testns/test")) + server.stubFor(get(urlEqualTo("/v1/ranger/nodes/testns/test")) .willReturn(aResponse() .withBody(payload) .withStatus(200))); @@ -64,31 +77,25 @@ public void testFinder() throws Exception { .connectionTimeoutMs(30_000) .operationTimeoutMs(30_000) .build(); - final SimpleShardedServiceFinder finder = new HttpShardedServiceFinderBuilder>() + + final SimpleShardedServiceFinder> finder = new HttpShardedServiceFinderBuilder>() .withClientConfig(clientConfig) .withNamespace("testns") .withServiceName("test") + .withObjectMapper(MAPPER) .withDeserializer(data -> { - final ServiceNodesResponse response; try { - response = MAPPER.readValue(data, new TypeReference>() {}); + return MAPPER.readValue(data, new TypeReference>() {}); } catch (IOException e) { throw new IllegalArgumentException(e); } - return response.isSuccess() ? response.getNodes() : Collections.emptyList(); }) - .withShardSelector((criteria, registry) -> registry - .nodes() - .entries() - .stream() - .filter(e -> e.getKey().getName().equals(criteria.getName())) - .map(Map.Entry::getValue) - .collect(Collectors.toList())) + .withShardSelector((criteria, registry) -> registry.nodeList()) .build(); finder.start(); TestUtils.sleepForSeconds(3); - Assert.assertNotNull(finder.get(testNode)); + Assert.assertNotNull(finder.get(nodeData -> true)); } } \ No newline at end of file diff --git a/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSourceTest.java b/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSourceTest.java new file mode 100644 index 00000000..d5420f68 --- /dev/null +++ b/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSourceTest.java @@ -0,0 +1,85 @@ +/** + * Copyright 2015 Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.http.servicefinderhub; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.flipkart.ranger.core.model.Service; +import com.flipkart.ranger.http.config.HttpClientConfig; +import com.flipkart.ranger.http.model.ServiceDataSourceResponse; +import com.github.tomakehurst.wiremock.junit.WireMockRule; +import com.google.common.collect.Lists; +import lombok.Builder; +import lombok.Data; +import lombok.val; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; + +import java.io.IOException; +import java.util.Collection; + +import static com.github.tomakehurst.wiremock.client.WireMock.*; + +public class HttpServiceDataSourceTest { + + @Data + private static final class TestNodeData { + private final String farmId; + + @Builder + public TestNodeData(@JsonProperty("farmId") String farmId) { + this.farmId = farmId; + } + } + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + @Rule + public WireMockRule server = new WireMockRule(8888); + + @Test + public void testServiceDataSource() throws IOException { + val responseObj = ServiceDataSourceResponse.builder() + .success(true) + .data(Lists.newArrayList( + new Service("test-n", "test-s"), + new Service("test-n", "test-s1"), + new Service("test-n", "test-s2") + )) + .build(); + val response = MAPPER.writeValueAsBytes(responseObj); + server.stubFor(get(urlEqualTo("/v1/ranger/services")) + .willReturn(aResponse() + .withBody(response) + .withStatus(200))); + val clientConfig = HttpClientConfig.builder() + .host("127.0.0.1") + .port(server.port()) + .connectionTimeoutMs(30_000) + .operationTimeoutMs(30_000) + .build(); + HttpServiceDataSource httpServiceDataSource = new HttpServiceDataSource<>(clientConfig, MAPPER); + Collection services = httpServiceDataSource.services(); + Assert.assertNotNull(services); + Assert.assertFalse(services.isEmpty()); + Assert.assertEquals(3, services.size()); + Assert.assertFalse(services.stream().noneMatch(each -> each.getServiceName().equalsIgnoreCase("test-s"))); + Assert.assertFalse(services.stream().noneMatch(each -> each.getServiceName().equalsIgnoreCase("test-s1"))); + Assert.assertFalse(services.stream().noneMatch(each -> each.getServiceName().equalsIgnoreCase("test-s2"))); + } + +} diff --git a/ranger-http/src/test/java/com/flipkart/ranger/http/serviceprovider/HttpShardedServiceProviderBuilderTest.java b/ranger-http/src/test/java/com/flipkart/ranger/http/serviceprovider/HttpShardedServiceProviderBuilderTest.java new file mode 100644 index 00000000..3c325024 --- /dev/null +++ b/ranger-http/src/test/java/com/flipkart/ranger/http/serviceprovider/HttpShardedServiceProviderBuilderTest.java @@ -0,0 +1,87 @@ +/** + * Copyright 2015 Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.http.serviceprovider; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.flipkart.ranger.core.healthcheck.Healthchecks; +import com.flipkart.ranger.core.model.ServiceNode; +import com.flipkart.ranger.core.serviceprovider.ServiceProvider; +import com.flipkart.ranger.http.config.HttpClientConfig; +import com.flipkart.ranger.http.model.ServiceRegistrationResponse; +import com.flipkart.ranger.http.serde.HttpRequestDataSerializer; +import com.github.tomakehurst.wiremock.junit.WireMockRule; +import lombok.Builder; +import lombok.Data; +import lombok.val; +import org.junit.Rule; +import org.junit.Test; + +import static com.github.tomakehurst.wiremock.client.WireMock.*; + +public class HttpShardedServiceProviderBuilderTest { + + @Data + private static final class TestNodeData { + private final String farmId; + + @Builder + public TestNodeData(@JsonProperty("farmId") String farmId) { + this.farmId = farmId; + } + } + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + @Rule + public WireMockRule server = new WireMockRule(8888); + + @Test + public void testProvider() throws Exception { + TestNodeData nm5NodeData = TestNodeData.builder().farmId("nm5").build(); + ServiceNode testNode = new ServiceNode<>("localhost-1", 80, nm5NodeData); + val response = MAPPER.writeValueAsBytes( + ServiceRegistrationResponse.builder() + .success(true) + .build()); + byte[] requestBytes = MAPPER.writeValueAsBytes(testNode); + server.stubFor(post(urlEqualTo("/v1/ranger/nodes/add/testns/test")) + .withRequestBody(binaryEqualTo(requestBytes)) + .willReturn(aResponse() + .withBody(response) + .withStatus(200))); + val clientConfig = HttpClientConfig.builder() + .host("127.0.0.1") + .port(server.port()) + .connectionTimeoutMs(30_000) + .operationTimeoutMs(30_000) + .build(); + ServiceProvider> serviceProvider = new HttpShardedServiceProviderBuilder() + .withNamespace("testns") + .withServiceName("test") + .withHostname("localhost-1") + .withPort(80) + .withHealthcheck(Healthchecks.defaultHealthyCheck()) + .withHealthUpdateIntervalMs(1000) + .withObjectMapper(MAPPER) + .withClientConfiguration(clientConfig) + .withNodeData(nm5NodeData) + .withSerializer(node -> requestBytes) + .build(); + serviceProvider.start(); + } + +} diff --git a/ranger-http/src/test/resources/fixtures/httpClientConfig.json b/ranger-http/src/test/resources/fixtures/httpClientConfig.json new file mode 100644 index 00000000..3855f309 --- /dev/null +++ b/ranger-http/src/test/resources/fixtures/httpClientConfig.json @@ -0,0 +1,8 @@ +{ + "host" : "localhost-1", + "port" : 80, + "secure" : false, + "connectionTimeoutMs" : 10, + "operationTimeoutMs" : 10, + "refreshIntervalMillis" : 1000 +} \ No newline at end of file diff --git a/ranger-http/src/test/resources/fixtures/serviceNodesResponse.json b/ranger-http/src/test/resources/fixtures/serviceNodesResponse.json new file mode 100644 index 00000000..7ac55a8e --- /dev/null +++ b/ranger-http/src/test/resources/fixtures/serviceNodesResponse.json @@ -0,0 +1,23 @@ +{ + "success" : true, + "data" : [ + { + "host" : "localhost-1", + "port" : 80, + "nodeData" : { + "shardId" : 1, + "region" : "r1" + }, + "healthcheckStatus" : "healthy" + }, + { + "host" : "localhost-2", + "port" : 90, + "nodeData" : { + "shardId" : 2, + "region" : "r2" + }, + "healthcheckStatus" : "healthy" + } + ] +} \ No newline at end of file diff --git a/ranger-http/src/test/resources/fixtures/serviceResponse.json b/ranger-http/src/test/resources/fixtures/serviceResponse.json new file mode 100644 index 00000000..753c708a --- /dev/null +++ b/ranger-http/src/test/resources/fixtures/serviceResponse.json @@ -0,0 +1,3 @@ +{ + "success" : true +} \ No newline at end of file diff --git a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/ServiceFinderBuilders.java b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/ServiceFinderBuilders.java index 33c2dbc3..179068c5 100644 --- a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/ServiceFinderBuilders.java +++ b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/ServiceFinderBuilders.java @@ -1,23 +1,23 @@ -/** +/* * Copyright 2015 Flipkart Internet Pvt. Ltd. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ - package com.flipkart.ranger.zookeeper; -import com.flipkart.ranger.zookeeper.zk.ZkSimpleShardedServiceFinderBuilder; -import com.flipkart.ranger.zookeeper.zk.ZkUnshardedFinderBuilder; +import com.flipkart.ranger.core.model.Criteria; +import com.flipkart.ranger.zookeeper.servicefinder.ZkSimpleShardedServiceFinderBuilder; +import com.flipkart.ranger.zookeeper.servicefinder.ZkSimpleUnshardedServiceFinderBuilder; public class ServiceFinderBuilders { @@ -25,11 +25,11 @@ private ServiceFinderBuilders() { throw new InstantiationError("Must not instantiate this class"); } - public static ZkSimpleShardedServiceFinderBuilder shardedFinderBuilder() { + public static > ZkSimpleShardedServiceFinderBuilder shardedFinderBuilder() { return new ZkSimpleShardedServiceFinderBuilder<>(); } - public static ZkUnshardedFinderBuilder unshardedFinderBuilder() { - return new ZkUnshardedFinderBuilder(); + public static > ZkSimpleUnshardedServiceFinderBuilder unshardedFinderBuilder() { + return new ZkSimpleUnshardedServiceFinderBuilder<>(); } } diff --git a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/ServiceProviderBuilders.java b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/ServiceProviderBuilders.java index 9feac3c4..e83abfbd 100644 --- a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/ServiceProviderBuilders.java +++ b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/ServiceProviderBuilders.java @@ -1,12 +1,12 @@ -/** +/* * Copyright 2015 Flipkart Internet Pvt. Ltd. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -16,8 +16,7 @@ package com.flipkart.ranger.zookeeper; -import com.flipkart.ranger.zookeeper.zk.ZkServiceProviderBuilder; -import com.flipkart.ranger.core.finder.unsharded.UnshardedClusterInfo; +import com.flipkart.ranger.zookeeper.serviceprovider.ZkServiceProviderBuilder; public class ServiceProviderBuilders { @@ -29,7 +28,7 @@ public static ZkServiceProviderBuilder shardedServiceProviderBuilder() { return new ZkServiceProviderBuilder<>(); } - public static ZkServiceProviderBuilder unshardedServiceProviderBuilder() { + public static ZkServiceProviderBuilder unshardedServiceProviderBuilder() { return new ZkServiceProviderBuilder<>(); } } diff --git a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/zk/ZkNodeDataStoreConnector.java b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/common/ZkNodeDataStoreConnector.java similarity index 85% rename from ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/zk/ZkNodeDataStoreConnector.java rename to ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/common/ZkNodeDataStoreConnector.java index 2ad0c2f4..79261dee 100644 --- a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/zk/ZkNodeDataStoreConnector.java +++ b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/common/ZkNodeDataStoreConnector.java @@ -1,4 +1,19 @@ -package com.flipkart.ranger.zookeeper.zk; +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.zookeeper.common; import com.flipkart.ranger.core.model.NodeDataStoreConnector; import com.flipkart.ranger.core.model.Service; diff --git a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/serde/ZkNodeDataDeserializer.java b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/serde/ZkNodeDataDeserializer.java index 11b22821..2990943a 100644 --- a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/serde/ZkNodeDataDeserializer.java +++ b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/serde/ZkNodeDataDeserializer.java @@ -1,3 +1,18 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.flipkart.ranger.zookeeper.serde; import com.flipkart.ranger.core.model.Deserializer; diff --git a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/serde/ZkNodeDataSerializer.java b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/serde/ZkNodeDataSerializer.java index 182f001b..3a2db411 100644 --- a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/serde/ZkNodeDataSerializer.java +++ b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/serde/ZkNodeDataSerializer.java @@ -1,3 +1,18 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.flipkart.ranger.zookeeper.serde; import com.flipkart.ranger.core.model.Serializer; diff --git a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/zk/ZkNodeDataSource.java b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinder/ZkNodeDataSource.java similarity index 83% rename from ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/zk/ZkNodeDataSource.java rename to ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinder/ZkNodeDataSource.java index 9d701053..d2258dfc 100644 --- a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/zk/ZkNodeDataSource.java +++ b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinder/ZkNodeDataSource.java @@ -1,9 +1,25 @@ -package com.flipkart.ranger.zookeeper.zk; +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.zookeeper.servicefinder; import com.flipkart.ranger.core.model.NodeDataSource; import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.util.FinderUtils; +import com.flipkart.ranger.zookeeper.common.ZkNodeDataStoreConnector; import com.flipkart.ranger.zookeeper.serde.ZkNodeDataDeserializer; import com.flipkart.ranger.zookeeper.util.PathBuilder; import com.google.common.base.Preconditions; diff --git a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/zk/ZkSimpleShardedServiceFinderBuilder.java b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinder/ZkSimpleShardedServiceFinderBuilder.java similarity index 59% rename from ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/zk/ZkSimpleShardedServiceFinderBuilder.java rename to ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinder/ZkSimpleShardedServiceFinderBuilder.java index bfb6345d..fa055fe0 100644 --- a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/zk/ZkSimpleShardedServiceFinderBuilder.java +++ b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinder/ZkSimpleShardedServiceFinderBuilder.java @@ -1,11 +1,28 @@ -package com.flipkart.ranger.zookeeper.zk; +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.zookeeper.servicefinder; -import com.flipkart.ranger.core.finder.sharded.SimpleShardedServiceFinder; -import com.flipkart.ranger.core.finder.sharded.SimpleShardedServiceFinderBuilder; +import com.flipkart.ranger.core.finder.SimpleShardedServiceFinder; +import com.flipkart.ranger.core.finder.SimpleShardedServiceFinderBuilder; +import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.NodeDataSource; import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.core.signals.Signal; import com.flipkart.ranger.zookeeper.serde.ZkNodeDataDeserializer; +import com.flipkart.ranger.zookeeper.servicefinder.signals.ZkWatcherRegistryUpdateSignal; import com.google.common.base.Preconditions; import lombok.extern.slf4j.Slf4j; import org.apache.curator.framework.CuratorFramework; @@ -19,23 +36,22 @@ * */ @Slf4j -public class ZkSimpleShardedServiceFinderBuilder extends SimpleShardedServiceFinderBuilder, ZkNodeDataDeserializer> { +public class ZkSimpleShardedServiceFinderBuilder> extends SimpleShardedServiceFinderBuilder, ZkNodeDataDeserializer, C> { protected CuratorFramework curatorFramework; protected String connectionString; - - public ZkSimpleShardedServiceFinderBuilder withCuratorFramework(CuratorFramework curatorFramework) { + public ZkSimpleShardedServiceFinderBuilder withCuratorFramework(CuratorFramework curatorFramework) { this.curatorFramework = curatorFramework; return this; } - public ZkSimpleShardedServiceFinderBuilder withConnectionString(final String connectionString) { + public ZkSimpleShardedServiceFinderBuilder withConnectionString(final String connectionString) { this.connectionString = connectionString; return this; } @Override - public SimpleShardedServiceFinder build() { + public SimpleShardedServiceFinder build() { boolean curatorProvided = curatorFramework != null; if (!curatorProvided) { Preconditions.checkNotNull(connectionString); diff --git a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/zk/ZkUnshardedFinderBuilder.java b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinder/ZkSimpleUnshardedServiceFinderBuilder.java similarity index 52% rename from ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/zk/ZkUnshardedFinderBuilder.java rename to ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinder/ZkSimpleUnshardedServiceFinderBuilder.java index 1e0259d3..640ebb68 100644 --- a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/zk/ZkUnshardedFinderBuilder.java +++ b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinder/ZkSimpleUnshardedServiceFinderBuilder.java @@ -1,12 +1,28 @@ -package com.flipkart.ranger.zookeeper.zk; +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.zookeeper.servicefinder; -import com.flipkart.ranger.core.finder.unsharded.UnshardedClusterFinder; -import com.flipkart.ranger.core.finder.unsharded.UnshardedClusterInfo; -import com.flipkart.ranger.core.finder.unsharded.UnshardedFinderBuilder; +import com.flipkart.ranger.core.finder.SimpleUnshardedServiceFinder; +import com.flipkart.ranger.core.finder.SimpleUnshardedServiceFinderBuilder; +import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.NodeDataSource; import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.core.signals.Signal; import com.flipkart.ranger.zookeeper.serde.ZkNodeDataDeserializer; +import com.flipkart.ranger.zookeeper.servicefinder.signals.ZkWatcherRegistryUpdateSignal; import com.google.common.base.Preconditions; import lombok.extern.slf4j.Slf4j; import org.apache.curator.framework.CuratorFramework; @@ -20,22 +36,23 @@ * */ @Slf4j -public class ZkUnshardedFinderBuilder extends UnshardedFinderBuilder> { +public class ZkSimpleUnshardedServiceFinderBuilder> + extends SimpleUnshardedServiceFinderBuilder, ZkNodeDataDeserializer, C> { private CuratorFramework curatorFramework; private String connectionString; - public ZkUnshardedFinderBuilder withCuratorFramework(CuratorFramework curatorFramework) { + public ZkSimpleUnshardedServiceFinderBuilder withCuratorFramework(CuratorFramework curatorFramework) { this.curatorFramework = curatorFramework; return this; } - public ZkUnshardedFinderBuilder withConnectionString(final String connectionString) { + public ZkSimpleUnshardedServiceFinderBuilder withConnectionString(final String connectionString) { this.connectionString = connectionString; return this; } @Override - public UnshardedClusterFinder build() { + public SimpleUnshardedServiceFinder build() { boolean curatorProvided = curatorFramework != null; if (!curatorProvided) { Preconditions.checkNotNull(connectionString); @@ -50,15 +67,15 @@ public UnshardedClusterFinder build() { } @Override - protected NodeDataSource> dataSource( + protected NodeDataSource> dataSource( Service service) { return new ZkNodeDataSource<>(service, curatorFramework); } @Override - protected List> implementationSpecificRefreshSignals( - final Service service, final NodeDataSource> nodeDataSource) { + protected List> implementationSpecificRefreshSignals( + final Service service, final NodeDataSource> nodeDataSource) { if (!disablePushUpdaters) { return Collections.singletonList( new ZkWatcherRegistryUpdateSignal<>(service, nodeDataSource, curatorFramework)); diff --git a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/zk/ZkWatcherRegistryUpdateSignal.java b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinder/signals/ZkWatcherRegistryUpdateSignal.java similarity index 75% rename from ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/zk/ZkWatcherRegistryUpdateSignal.java rename to ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinder/signals/ZkWatcherRegistryUpdateSignal.java index e4d9c3dc..8b5c5659 100644 --- a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/zk/ZkWatcherRegistryUpdateSignal.java +++ b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinder/signals/ZkWatcherRegistryUpdateSignal.java @@ -1,9 +1,24 @@ -package com.flipkart.ranger.zookeeper.zk; +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.zookeeper.servicefinder.signals; import com.flipkart.ranger.core.model.NodeDataStoreConnector; import com.flipkart.ranger.core.model.Service; -import com.flipkart.ranger.zookeeper.util.PathBuilder; import com.flipkart.ranger.core.signals.Signal; +import com.flipkart.ranger.zookeeper.util.PathBuilder; import lombok.EqualsAndHashCode; import lombok.ToString; import lombok.extern.slf4j.Slf4j; diff --git a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZKUnshardedServiceFinderFactory.java b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZKUnshardedServiceFinderFactory.java new file mode 100644 index 00000000..8539998b --- /dev/null +++ b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZKUnshardedServiceFinderFactory.java @@ -0,0 +1,78 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.zookeeper.servicefinderhub; + +import com.flipkart.ranger.core.finder.SimpleUnshardedServiceFinder; +import com.flipkart.ranger.core.finder.serviceregistry.ListBasedServiceRegistry; +import com.flipkart.ranger.core.finderhub.ServiceFinderFactory; +import com.flipkart.ranger.core.model.Criteria; +import com.flipkart.ranger.core.model.Service; +import com.flipkart.ranger.core.model.ServiceNodeSelector; +import com.flipkart.ranger.core.model.ShardSelector; +import com.flipkart.ranger.zookeeper.serde.ZkNodeDataDeserializer; +import com.flipkart.ranger.zookeeper.servicefinder.ZkSimpleUnshardedServiceFinderBuilder; +import lombok.Builder; +import lombok.Getter; +import lombok.val; +import org.apache.curator.framework.CuratorFramework; + + +@Getter +public class ZKUnshardedServiceFinderFactory> implements ServiceFinderFactory>{ + + private final CuratorFramework curatorFramework; + private final String connectionString; + private final int nodeRefreshIntervalMs; + private final boolean disablePushUpdaters; + private final ZkNodeDataDeserializer deserializer; + private final ShardSelector> shardSelector; + private final ServiceNodeSelector nodeSelector; + + @Builder + public ZKUnshardedServiceFinderFactory( + CuratorFramework curatorFramework, + String connectionString, + int nodeRefreshIntervalMs, + boolean disablePushUpdaters, + ZkNodeDataDeserializer deserializer, + ShardSelector> shardSelector, + ServiceNodeSelector nodeSelector) { + this.curatorFramework = curatorFramework; + this.connectionString = connectionString; + this.nodeRefreshIntervalMs = nodeRefreshIntervalMs; + this.disablePushUpdaters = disablePushUpdaters; + this.deserializer = deserializer; + this.shardSelector = shardSelector; + this.nodeSelector = nodeSelector; + } + + @Override + public SimpleUnshardedServiceFinder buildFinder(Service service) { + val finder = new ZkSimpleUnshardedServiceFinderBuilder() + .withDeserializer(deserializer) + .withNamespace(service.getNamespace()) + .withServiceName(service.getServiceName()) + .withNodeRefreshIntervalMs(nodeRefreshIntervalMs) + .withDisableWatchers(disablePushUpdaters) + .withShardSelector(shardSelector) + .withNodeSelector(nodeSelector) + .withConnectionString(connectionString) + .withCuratorFramework(curatorFramework) + .build(); + finder.start(); + return finder; + } +} diff --git a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZkServiceDataSource.java b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZkServiceDataSource.java new file mode 100644 index 00000000..e13fcaf1 --- /dev/null +++ b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZkServiceDataSource.java @@ -0,0 +1,86 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.zookeeper.servicefinderhub; + +import com.flipkart.ranger.core.finderhub.ServiceDataSource; +import com.flipkart.ranger.core.model.Service; +import com.google.common.base.Preconditions; +import lombok.extern.slf4j.Slf4j; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.retry.ExponentialBackoffRetry; + +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; + +/** + * + */ +@Slf4j +public class ZkServiceDataSource implements ServiceDataSource { + + private final String namespace; + private String connectionString; + private CuratorFramework curatorFramework; + private boolean curatorProvided; + + public ZkServiceDataSource(String namespace, + String connectionString, + CuratorFramework curatorFramework){ + this.namespace = namespace; + this.connectionString = connectionString; + this.curatorFramework = curatorFramework; + } + + @Override + public Collection services() throws Exception { + final List children = curatorFramework.getChildren() + .forPath("/"); + return children.stream() + .map(child -> new Service(namespace, child)) + .collect(Collectors.toSet()); + } + + @Override + public void start() { + if(null == curatorFramework){ + Preconditions.checkNotNull(connectionString); + log.info("Building custom curator framework"); + curatorFramework = CuratorFrameworkFactory.builder() + .namespace(namespace) + .connectString(connectionString) + .retryPolicy(new ExponentialBackoffRetry(1000, 100)) + .build(); + curatorFramework.start(); + curatorProvided = false; + } + try { + curatorFramework.blockUntilConnected(); + } + catch (InterruptedException e) { + log.error("Curator block interrupted", e); + } + log.info("Service data source started. Curator state is: {}", curatorFramework.getState().name()); + } + + @Override + public void stop() { + log.info("Service data stopped"); + if(!curatorProvided) curatorFramework.close(); + log.info("Service data source stopped"); + } +} diff --git a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/zk/ZkServiceFinderHubBuilder.java b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZkServiceFinderHubBuilder.java similarity index 52% rename from ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/zk/ZkServiceFinderHubBuilder.java rename to ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZkServiceFinderHubBuilder.java index 9fa692b6..aead107e 100644 --- a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/zk/ZkServiceFinderHubBuilder.java +++ b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZkServiceFinderHubBuilder.java @@ -1,6 +1,23 @@ -package com.flipkart.ranger.zookeeper.zk; +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.zookeeper.servicefinderhub; import com.flipkart.ranger.core.finderhub.ServiceFinderHub; +import com.flipkart.ranger.core.finderhub.ServiceFinderHubBuilder; +import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.ServiceRegistry; import com.google.common.base.Preconditions; import lombok.extern.slf4j.Slf4j; @@ -12,22 +29,22 @@ * */ @Slf4j -public class ZkServiceFinderHubBuilder> extends ServiceFinderHubBuilder { +public class ZkServiceFinderHubBuilder, R extends ServiceRegistry> extends ServiceFinderHubBuilder { private String namespace; private CuratorFramework curatorFramework; private String connectionString; - public ZkServiceFinderHubBuilder withNamespace(final String namespace) { + public ZkServiceFinderHubBuilder withNamespace(final String namespace) { this.namespace = namespace; return this; } - public ZkServiceFinderHubBuilder withCuratorFramework(CuratorFramework curatorFramework) { + public ZkServiceFinderHubBuilder withCuratorFramework(CuratorFramework curatorFramework) { this.curatorFramework = curatorFramework; return this; } - public ZkServiceFinderHubBuilder withConnectionString(final String connectionString) { + public ZkServiceFinderHubBuilder withConnectionString(final String connectionString) { this.connectionString = connectionString; return this; } @@ -48,7 +65,7 @@ protected void preBuild() { } @Override - protected void postBuild(ServiceFinderHub serviceFinderHub) { + protected void postBuild(ServiceFinderHub serviceFinderHub) { log.debug("No post build steps necessary"); } } diff --git a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/zk/ZkShardedServiceFinderFactory.java b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZkShardedServiceFinderFactory.java similarity index 58% rename from ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/zk/ZkShardedServiceFinderFactory.java rename to ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZkShardedServiceFinderFactory.java index f6feca8d..d36981ff 100644 --- a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/zk/ZkShardedServiceFinderFactory.java +++ b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZkShardedServiceFinderFactory.java @@ -1,12 +1,29 @@ -package com.flipkart.ranger.zookeeper.zk; +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.zookeeper.servicefinderhub; -import com.flipkart.ranger.core.finder.sharded.MapBasedServiceRegistry; -import com.flipkart.ranger.core.finder.sharded.SimpleShardedServiceFinder; +import com.flipkart.ranger.core.finder.SimpleShardedServiceFinder; +import com.flipkart.ranger.core.finder.serviceregistry.MapBasedServiceRegistry; import com.flipkart.ranger.core.finderhub.ServiceFinderFactory; +import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.core.model.ServiceNodeSelector; import com.flipkart.ranger.core.model.ShardSelector; import com.flipkart.ranger.zookeeper.serde.ZkNodeDataDeserializer; +import com.flipkart.ranger.zookeeper.servicefinder.ZkSimpleShardedServiceFinderBuilder; import lombok.Builder; import lombok.val; import org.apache.curator.framework.CuratorFramework; @@ -14,13 +31,13 @@ /** * */ -public class ZkShardedServiceFinderFactory implements ServiceFinderFactory> { +public class ZkShardedServiceFinderFactory> implements ServiceFinderFactory> { private final CuratorFramework curatorFramework; private final String connectionString; private final int nodeRefreshIntervalMs; private final boolean disablePushUpdaters; private final ZkNodeDataDeserializer deserializer; - private final ShardSelector> shardSelector; + private final ShardSelector> shardSelector; private final ServiceNodeSelector nodeSelector; @Builder @@ -30,7 +47,7 @@ public ZkShardedServiceFinderFactory( int nodeRefreshIntervalMs, boolean disablePushUpdaters, ZkNodeDataDeserializer deserializer, - ShardSelector> shardSelector, + ShardSelector> shardSelector, ServiceNodeSelector nodeSelector) { this.curatorFramework = curatorFramework; this.connectionString = connectionString; @@ -42,8 +59,8 @@ public ZkShardedServiceFinderFactory( } @Override - public SimpleShardedServiceFinder buildFinder(Service service) { - val finder = new ZkSimpleShardedServiceFinderBuilder() + public SimpleShardedServiceFinder buildFinder(Service service) { + val finder = new ZkSimpleShardedServiceFinderBuilder() .withDeserializer(deserializer) .withNamespace(service.getNamespace()) .withServiceName(service.getServiceName()) diff --git a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/zk/ZkNodeDataSink.java b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/serviceprovider/ZkNodeDataSink.java similarity index 79% rename from ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/zk/ZkNodeDataSink.java rename to ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/serviceprovider/ZkNodeDataSink.java index 3e5d27ab..3792e731 100644 --- a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/zk/ZkNodeDataSink.java +++ b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/serviceprovider/ZkNodeDataSink.java @@ -1,9 +1,25 @@ -package com.flipkart.ranger.zookeeper.zk; +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.zookeeper.serviceprovider; import com.flipkart.ranger.core.model.NodeDataSink; import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.util.Exceptions; +import com.flipkart.ranger.zookeeper.common.ZkNodeDataStoreConnector; import com.flipkart.ranger.zookeeper.serde.ZkNodeDataSerializer; import com.flipkart.ranger.zookeeper.util.PathBuilder; import com.google.common.base.Preconditions; diff --git a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/zk/ZkServiceProviderBuilder.java b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/serviceprovider/ZkServiceProviderBuilder.java similarity index 74% rename from ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/zk/ZkServiceProviderBuilder.java rename to ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/serviceprovider/ZkServiceProviderBuilder.java index cb6a47f8..1a838324 100644 --- a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/zk/ZkServiceProviderBuilder.java +++ b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/serviceprovider/ZkServiceProviderBuilder.java @@ -1,4 +1,19 @@ -package com.flipkart.ranger.zookeeper.zk; +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.zookeeper.serviceprovider; import com.flipkart.ranger.core.model.NodeDataSink; import com.flipkart.ranger.core.model.Service; diff --git a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/util/PathBuilder.java b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/util/PathBuilder.java index 02542579..3474c749 100644 --- a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/util/PathBuilder.java +++ b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/util/PathBuilder.java @@ -1,12 +1,12 @@ -/** +/* * Copyright 2015 Flipkart Internet Pvt. Ltd. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/zk/ZkServiceDataSource.java b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/zk/ZkServiceDataSource.java deleted file mode 100644 index 3f3cfd00..00000000 --- a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/zk/ZkServiceDataSource.java +++ /dev/null @@ -1,50 +0,0 @@ -package com.flipkart.ranger.zookeeper.zk; - -import com.flipkart.ranger.core.finderhub.ServiceDataSource; -import com.flipkart.ranger.core.model.Service; -import lombok.extern.slf4j.Slf4j; -import org.apache.curator.framework.CuratorFramework; - -import java.util.Collection; -import java.util.List; -import java.util.stream.Collectors; - -/** - * - */ -@Slf4j -public class ZkServiceDataSource implements ServiceDataSource { - - private final String namespace; - private final CuratorFramework curatorFramework; - - public ZkServiceDataSource(String namespace, CuratorFramework curatorFramework) { - this.namespace = namespace; - this.curatorFramework = curatorFramework; - } - - @Override - public Collection services() throws Exception { - final List children = curatorFramework.getChildren() - .forPath("/"); - return children.stream() - .map(child -> new Service(namespace, child)) - .collect(Collectors.toSet()); - } - - @Override - public void start() { - try { - curatorFramework.blockUntilConnected(); - } - catch (InterruptedException e) { - log.error("Curator block interrupted", e); - } - log.info("Service data source started. Curator state is: {}", curatorFramework.getState().name()); - } - - @Override - public void stop() { - log.info("Service data source stopped"); - } -} diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/ServiceHealthAggregatorTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/ServiceHealthAggregatorTest.java index 6f52e2eb..bc217329 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/ServiceHealthAggregatorTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/ServiceHealthAggregatorTest.java @@ -1,27 +1,26 @@ -/** - * Copyright 2016 Flipkart Internet Pvt. Ltd. - * +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ - package com.flipkart.ranger.zookeeper.healthservice; +import com.flipkart.ranger.core.TestUtils; import com.flipkart.ranger.core.healthcheck.HealthcheckStatus; import com.flipkart.ranger.core.healthservice.ServiceHealthAggregator; import com.flipkart.ranger.core.healthservice.TimeEntity; import com.flipkart.ranger.core.healthservice.monitor.IsolatedHealthMonitor; import com.flipkart.ranger.core.healthservice.monitor.Monitor; -import com.flipkart.ranger.core.utils.TestUtils; import org.junit.After; import org.junit.Assert; import org.junit.Before; diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/ServiceProviderIntegrationTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/ServiceProviderIntegrationTest.java index 64ed9401..3219233f 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/ServiceProviderIntegrationTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/ServiceProviderIntegrationTest.java @@ -1,12 +1,12 @@ -/** - * Copyright 2016 Flipkart Internet Pvt. Ltd. - * +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,14 +19,14 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import com.flipkart.ranger.core.finder.unsharded.UnshardedClusterFinder; -import com.flipkart.ranger.core.finder.unsharded.UnshardedClusterInfo; +import com.flipkart.ranger.core.TestUtils; +import com.flipkart.ranger.core.finder.SimpleUnshardedServiceFinder; import com.flipkart.ranger.core.healthcheck.Healthchecks; import com.flipkart.ranger.core.healthservice.TimeEntity; import com.flipkart.ranger.core.healthservice.monitor.sample.RotationStatusMonitor; +import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.util.Exceptions; -import com.flipkart.ranger.core.utils.TestUtils; import com.flipkart.ranger.zookeeper.ServiceFinderBuilders; import com.flipkart.ranger.zookeeper.ServiceProviderBuilders; import lombok.val; @@ -50,8 +50,19 @@ public class ServiceProviderIntegrationTest { private TestingCluster testingCluster; private ObjectMapper objectMapper; - UnshardedClusterFinder serviceFinder; + SimpleUnshardedServiceFinder serviceFinder; + private static final class UnshardedClusterInfo { + @Override + public int hashCode() { + return 0; + } + + @Override + public boolean equals(Object obj) { + return super.equals(obj); + } + } @Before public void startTestCluster() throws Exception { objectMapper = new ObjectMapper(); @@ -65,7 +76,7 @@ public void startTestCluster() throws Exception { registerService("localhost-4", 9000, 2, anotherFile); - serviceFinder = ServiceFinderBuilders.unshardedFinderBuilder() + serviceFinder = ServiceFinderBuilders.>unshardedFinderBuilder() .withConnectionString(testingCluster.getConnectString()) .withNamespace("test") .withServiceName("test-service") diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/monitor/RollingWindowHealthQueueTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/monitor/RollingWindowHealthQueueTest.java index 065cc7a3..34b58db5 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/monitor/RollingWindowHealthQueueTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/monitor/RollingWindowHealthQueueTest.java @@ -1,19 +1,18 @@ -/** - * Copyright 2016 Flipkart Internet Pvt. Ltd. - * +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ - package com.flipkart.ranger.zookeeper.healthservice.monitor; import com.flipkart.ranger.core.healthcheck.HealthcheckStatus; diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/monitor/sample/DiskSpaceMonitorTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/monitor/sample/DiskSpaceMonitorTest.java index 54ea08d5..be3c5e07 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/monitor/sample/DiskSpaceMonitorTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/monitor/sample/DiskSpaceMonitorTest.java @@ -1,12 +1,12 @@ -/** - * Copyright 2016 Flipkart Internet Pvt. Ltd. - * +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/monitor/sample/PingCheckMonitorTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/monitor/sample/PingCheckMonitorTest.java index 2a4cb14f..d0e0a9cc 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/monitor/sample/PingCheckMonitorTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/monitor/sample/PingCheckMonitorTest.java @@ -1,19 +1,18 @@ -/** - * Copyright 2016 Flipkart Internet Pvt. Ltd. - * +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ - package com.flipkart.ranger.zookeeper.healthservice.monitor.sample; import com.flipkart.ranger.core.healthcheck.HealthcheckStatus; diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/monitor/sample/RotationStatusMonitorTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/monitor/sample/RotationStatusMonitorTest.java index 22457cbb..63c4264a 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/monitor/sample/RotationStatusMonitorTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/monitor/sample/RotationStatusMonitorTest.java @@ -1,19 +1,18 @@ -/** - * Copyright 2016 Flipkart Internet Pvt. Ltd. - * +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ - package com.flipkart.ranger.zookeeper.healthservice.monitor.sample; import com.flipkart.ranger.core.healthcheck.HealthcheckStatus; diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/CustomShardSelectorTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/CustomShardSelectorTest.java index 36587d52..d489e6b3 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/CustomShardSelectorTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/CustomShardSelectorTest.java @@ -1,27 +1,27 @@ -/** +/* * Copyright 2015 Flipkart Internet Pvt. Ltd. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ - package com.flipkart.ranger.zookeeper.model; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import com.flipkart.ranger.core.finder.sharded.MapBasedServiceRegistry; -import com.flipkart.ranger.core.finder.sharded.SimpleShardedServiceFinder; +import com.flipkart.ranger.core.finder.SimpleShardedServiceFinder; +import com.flipkart.ranger.core.finder.serviceregistry.MapBasedServiceRegistry; import com.flipkart.ranger.core.healthcheck.Healthchecks; +import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.model.ShardSelector; import com.flipkart.ranger.core.serviceprovider.ServiceProvider; @@ -40,6 +40,7 @@ import java.util.List; import java.util.Map; + @Slf4j public class CustomShardSelectorTest { private TestingCluster testingCluster; @@ -111,16 +112,20 @@ public int hashCode() { result = 31 * result + b; return result; } + + private static Criteria getCriteria(int a, int b){ + return nodeData -> nodeData.getA() == a && nodeData.getB() == b; + } } - private static final class TestShardSelector implements ShardSelector> { + private static final class TestShardSelector implements ShardSelector, MapBasedServiceRegistry> { @Override - public List> nodes(TestShardInfo criteria, MapBasedServiceRegistry serviceRegistry) { + public List> nodes(Criteria criteria, MapBasedServiceRegistry serviceRegistry) { List> nodes = Lists.newArrayList(); for(Map.Entry> entry : serviceRegistry.nodes().entries()) { TestShardInfo shardInfo = entry.getKey(); - if((shardInfo.getA() + shardInfo.getB()) == (criteria.getA() + criteria.getB())) { + if(criteria.apply(shardInfo)){ nodes.add(entry.getValue()); } } @@ -130,7 +135,7 @@ public List> nodes(TestShardInfo criteria, MapBasedSe @Test public void testBasicDiscovery() throws Exception { - SimpleShardedServiceFinder serviceFinder = ServiceFinderBuilders.shardedFinderBuilder() + SimpleShardedServiceFinder> serviceFinder = ServiceFinderBuilders.>shardedFinderBuilder() .withConnectionString(testingCluster.getConnectString()) .withNamespace("test") .withServiceName("test-service") @@ -147,18 +152,18 @@ public void testBasicDiscovery() throws Exception { .build(); serviceFinder.start(); { - ServiceNode node = serviceFinder.get(new TestShardInfo(1, 10)); + ServiceNode node = serviceFinder.get(TestShardInfo.getCriteria(1, 10)); Assert.assertNull(node); } { - ServiceNode node = serviceFinder.get(new TestShardInfo(1, 2)); + ServiceNode node = serviceFinder.get(TestShardInfo.getCriteria(1, 2)); Assert.assertNotNull(node); Assert.assertEquals(new TestShardInfo(1, 2), node.getNodeData()); } serviceFinder.stop(); } - private void registerService(String host, int port, int a, int b) throws Exception { + private void registerService(String host, int port, int a, int b) { final ServiceProvider> serviceProvider = ServiceProviderBuilders.shardedServiceProviderBuilder() .withConnectionString(testingCluster.getConnectString()) .withNamespace("test") diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceNoProviderTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceNoProviderTest.java index 0bb56679..ebee0623 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceNoProviderTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceNoProviderTest.java @@ -1,4 +1,4 @@ -/** +/* * Copyright 2015 Flipkart Internet Pvt. Ltd. *

* Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,13 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package com.flipkart.ranger.zookeeper.model; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import com.flipkart.ranger.core.finder.RoundRobinServiceNodeSelector; -import com.flipkart.ranger.core.finder.sharded.SimpleShardedServiceFinder; +import com.flipkart.ranger.core.finder.SimpleShardedServiceFinder; +import com.flipkart.ranger.core.finder.nodeselector.RoundRobinServiceNodeSelector; +import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.zookeeper.ServiceFinderBuilders; import lombok.val; @@ -93,11 +93,16 @@ public boolean equals(Object o) { public int hashCode() { return shardId; } + + private static Criteria getCriteria(int shardId){ + return nodeData -> shardId == nodeData.getShardId(); + } } + @Test public void testBasicDiscovery() throws Exception { - SimpleShardedServiceFinder serviceFinder = ServiceFinderBuilders.shardedFinderBuilder() + SimpleShardedServiceFinder> serviceFinder = ServiceFinderBuilders.>shardedFinderBuilder() .withConnectionString(testingCluster.getConnectString()) .withNamespace("test") .withServiceName("test-service") @@ -114,15 +119,15 @@ public void testBasicDiscovery() throws Exception { }) .build(); serviceFinder.start(); - ServiceNode node = serviceFinder.get(new TestShardInfo(1)); + ServiceNode node = serviceFinder.get(TestShardInfo.getCriteria(1)); Assert.assertNull(node); serviceFinder.stop(); } @Test - public void testBasicDiscoveryRR() throws Exception { - val serviceFinder = ServiceFinderBuilders.shardedFinderBuilder() + public void testBasicDiscoveryRR() { + val serviceFinder = ServiceFinderBuilders.>shardedFinderBuilder() .withConnectionString(testingCluster.getConnectString()) .withNamespace("test") .withServiceName("test-service") @@ -140,7 +145,7 @@ public void testBasicDiscoveryRR() throws Exception { }) .build(); serviceFinder.start(); - ServiceNode node = serviceFinder.get(new TestShardInfo(1)); + ServiceNode node = serviceFinder.get(TestShardInfo.getCriteria(1)); Assert.assertNull(node); serviceFinder.stop(); } diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderExtCuratorTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderExtCuratorTest.java index 5c8b1183..68fbb4f6 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderExtCuratorTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderExtCuratorTest.java @@ -1,26 +1,26 @@ -/** +/* * Copyright 2015 Flipkart Internet Pvt. Ltd. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ - package com.flipkart.ranger.zookeeper.model; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import com.flipkart.ranger.core.finder.sharded.SimpleShardedServiceFinder; +import com.flipkart.ranger.core.finder.SimpleShardedServiceFinder; import com.flipkart.ranger.core.healthcheck.Healthchecks; +import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.serviceprovider.ServiceProvider; import com.flipkart.ranger.zookeeper.ServiceFinderBuilders; @@ -108,11 +108,16 @@ public boolean equals(Object o) { public int hashCode() { return shardId; } + + private static Criteria getCriteria(int shardId){ + return nodeData -> nodeData.getShardId() == shardId; + } + } @Test - public void testBasicDiscovery() throws Exception { - SimpleShardedServiceFinder serviceFinder = ServiceFinderBuilders.shardedFinderBuilder() + public void testBasicDiscovery() { + SimpleShardedServiceFinder> serviceFinder = ServiceFinderBuilders.>shardedFinderBuilder() .withCuratorFramework(curatorFramework) .withNamespace("test") .withServiceName("test-service") @@ -129,25 +134,25 @@ public void testBasicDiscovery() throws Exception { .build(); serviceFinder.start(); { - ServiceNode node = serviceFinder.get(new TestShardInfo(1)); + ServiceNode node = serviceFinder.get(TestShardInfo.getCriteria(1)); Assert.assertNotNull(node); Assert.assertEquals(1, node.getNodeData().getShardId()); } { - ServiceNode node = serviceFinder.get(new TestShardInfo(1)); + ServiceNode node = serviceFinder.get(TestShardInfo.getCriteria(1)); Assert.assertNotNull(node); Assert.assertEquals(1, node.getNodeData().getShardId()); } long startTime = System.currentTimeMillis(); for(long i = 0; i <1000000; i++) { - ServiceNode node = serviceFinder.get(new TestShardInfo(2)); + ServiceNode node = serviceFinder.get(TestShardInfo.getCriteria(2)); Assert.assertNotNull(node); Assert.assertEquals(2, node.getNodeData().getShardId()); } log.info("PERF::RandomSelector::Took (ms):" + (System.currentTimeMillis() - startTime)); { - ServiceNode node = serviceFinder.get(new TestShardInfo(99)); + ServiceNode node = serviceFinder.get(TestShardInfo.getCriteria(99)); Assert.assertNull(node); } serviceFinder.stop(); diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderHealthcheckTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderHealthcheckTest.java index 3e31e1e9..bbeef818 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderHealthcheckTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderHealthcheckTest.java @@ -1,12 +1,12 @@ -/** +/* * Copyright 2015 Flipkart Internet Pvt. Ltd. - *

+ *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

+ *

* http://www.apache.org/licenses/LICENSE-2.0 - *

+ *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,12 +19,13 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import com.flipkart.ranger.core.finder.sharded.SimpleShardedServiceFinder; +import com.flipkart.ranger.core.TestUtils; +import com.flipkart.ranger.core.finder.SimpleShardedServiceFinder; import com.flipkart.ranger.core.healthcheck.Healthcheck; import com.flipkart.ranger.core.healthcheck.HealthcheckStatus; +import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.serviceprovider.ServiceProvider; -import com.flipkart.ranger.core.utils.TestUtils; import com.flipkart.ranger.zookeeper.ServiceFinderBuilders; import com.flipkart.ranger.zookeeper.ServiceProviderBuilders; import com.flipkart.ranger.zookeeper.serde.ZkNodeDataSerializer; @@ -95,11 +96,15 @@ public boolean equals(Object o) { public int hashCode() { return shardId; } + + private static Criteria getCriteria(int shardId){ + return nodeData -> nodeData.getShardId() == shardId; + } } @Test - public void testBasicDiscovery() throws Exception { - SimpleShardedServiceFinder serviceFinder = ServiceFinderBuilders.shardedFinderBuilder() + public void testBasicDiscovery() { + SimpleShardedServiceFinder> serviceFinder = ServiceFinderBuilders.>shardedFinderBuilder() .withConnectionString(testingCluster.getConnectString()) .withNamespace("test") .withServiceName("test-service") @@ -116,13 +121,13 @@ public void testBasicDiscovery() throws Exception { .withNodeRefreshIntervalMs(1000) .build(); serviceFinder.start(); - ServiceNode node = serviceFinder.get(new TestShardInfo(1)); + ServiceNode node = serviceFinder.get(TestShardInfo.getCriteria(1)); Assert.assertNotNull(node); Assert.assertEquals("localhost-1", node.getHost()); TestServiceProvider testServiceProvider = serviceProviders.get(node.getHost()); testServiceProvider.oor(); TestUtils.sleepForSeconds(6); - Assert.assertNull(serviceFinder.get(new TestShardInfo(1))); + Assert.assertNull(serviceFinder.get(TestShardInfo.getCriteria(1))); serviceFinder.stop(); } diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderTest.java index 23432125..55041fd0 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderTest.java @@ -1,4 +1,4 @@ -/** +/* * Copyright 2015 Flipkart Internet Pvt. Ltd. *

* Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,9 +19,10 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import com.flipkart.ranger.core.finder.RoundRobinServiceNodeSelector; -import com.flipkart.ranger.core.finder.sharded.SimpleShardedServiceFinder; +import com.flipkart.ranger.core.finder.SimpleShardedServiceFinder; +import com.flipkart.ranger.core.finder.nodeselector.RoundRobinServiceNodeSelector; import com.flipkart.ranger.core.healthcheck.Healthchecks; +import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.serviceprovider.ServiceProvider; import com.flipkart.ranger.zookeeper.ServiceFinderBuilders; @@ -106,11 +107,15 @@ public boolean equals(Object o) { public int hashCode() { return shardId; } + + private static Criteria getCriteria(int shardId){ + return nodeData -> nodeData.getShardId() == shardId; + } } @Test public void testBasicDiscovery() throws Exception { - SimpleShardedServiceFinder serviceFinder = ServiceFinderBuilders.shardedFinderBuilder() + SimpleShardedServiceFinder> serviceFinder = ServiceFinderBuilders.>shardedFinderBuilder() .withConnectionString(testingCluster.getConnectString()) .withNamespace("test") .withServiceName("test-service") @@ -126,24 +131,24 @@ public void testBasicDiscovery() throws Exception { .build(); serviceFinder.start(); { - ServiceNode node = serviceFinder.get(new TestShardInfo(1)); + ServiceNode node = serviceFinder.get(TestShardInfo.getCriteria(1)); Assert.assertNotNull(node); Assert.assertEquals(1, node.getNodeData().getShardId()); } { - ServiceNode node = serviceFinder.get(new TestShardInfo(1)); + ServiceNode node = serviceFinder.get(TestShardInfo.getCriteria(1)); Assert.assertNotNull(node); Assert.assertEquals(1, node.getNodeData().getShardId()); } long startTime = System.currentTimeMillis(); for (long i = 0; i < 1000000; i++) { - ServiceNode node = serviceFinder.get(new TestShardInfo(2)); + ServiceNode node = serviceFinder.get(TestShardInfo.getCriteria(2)); Assert.assertNotNull(node); Assert.assertEquals(2, node.getNodeData().getShardId()); } log.info("PERF::RandomSelector::Took (ms):" + (System.currentTimeMillis() - startTime)); { - ServiceNode node = serviceFinder.get(new TestShardInfo(99)); + ServiceNode node = serviceFinder.get(TestShardInfo.getCriteria(99)); Assert.assertNull(node); } serviceFinder.stop(); @@ -152,8 +157,8 @@ public void testBasicDiscovery() throws Exception { @Test public void testBasicDiscoveryRR() throws Exception { - SimpleShardedServiceFinder serviceFinder - = ServiceFinderBuilders.shardedFinderBuilder() + SimpleShardedServiceFinder> serviceFinder + = ServiceFinderBuilders.>shardedFinderBuilder() .withConnectionString(testingCluster.getConnectString()) .withNamespace("test") .withServiceName("test-service") @@ -172,24 +177,24 @@ public void testBasicDiscoveryRR() throws Exception { .build(); serviceFinder.start(); { - ServiceNode node = serviceFinder.get(new TestShardInfo(1)); + ServiceNode node = serviceFinder.get(TestShardInfo.getCriteria(1)); Assert.assertNotNull(node); Assert.assertEquals(1, node.getNodeData().getShardId()); } { - ServiceNode node = serviceFinder.get(new TestShardInfo(1)); + ServiceNode node = serviceFinder.get(TestShardInfo.getCriteria(1)); Assert.assertNotNull(node); Assert.assertEquals(1, node.getNodeData().getShardId()); } long startTime = System.currentTimeMillis(); for (long i = 0; i < 1000000; i++) { - ServiceNode node = serviceFinder.get(new TestShardInfo(2)); + ServiceNode node = serviceFinder.get(TestShardInfo.getCriteria(2)); Assert.assertNotNull(node); Assert.assertEquals(2, node.getNodeData().getShardId()); } log.info("PERF::RoundRobinSelector::Took (ms):" + (System.currentTimeMillis() - startTime)); { - ServiceNode node = serviceFinder.get(new TestShardInfo(99)); + ServiceNode node = serviceFinder.get(TestShardInfo.getCriteria(99)); Assert.assertNull(node); } serviceFinder.stop(); @@ -197,8 +202,9 @@ public void testBasicDiscoveryRR() throws Exception { } @Test - public void testVisibility() throws Exception { - SimpleShardedServiceFinder serviceFinder = ServiceFinderBuilders.shardedFinderBuilder() + public void testVisibility() { + SimpleShardedServiceFinder> serviceFinder = ServiceFinderBuilders. + >shardedFinderBuilder() .withConnectionString(testingCluster.getConnectString()) .withNamespace("test") .withServiceName("test-service") @@ -215,7 +221,7 @@ public void testVisibility() throws Exception { }) .build(); serviceFinder.start(); - List> all = serviceFinder.getAll(new TestShardInfo(1)); + List> all = serviceFinder.getAll(TestShardInfo.getCriteria(1)); log.info("Testing ServiceFinder.getAll()"); for (ServiceNode serviceNode : all) { log.info("node = " + serviceNode.getHost() + ":" + serviceNode.getPort() + " " + serviceNode.getHealthcheckStatus() + " " + serviceNode @@ -225,7 +231,7 @@ public void testVisibility() throws Exception { serviceFinder.stop(); } - private void registerService(String host, int port, int shardId) throws Exception { + private void registerService(String host, int port, int shardId) { final ServiceProvider> serviceProvider = ServiceProviderBuilders.shardedServiceProviderBuilder() .withConnectionString(testingCluster.getConnectString()) .withNamespace("test") diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/SimpleServiceProviderTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/SimpleServiceProviderTest.java index 71d163bb..e875c186 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/SimpleServiceProviderTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/SimpleServiceProviderTest.java @@ -1,12 +1,12 @@ -/** +/* * Copyright 2015 Flipkart Internet Pvt. Ltd. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -19,9 +19,9 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import com.flipkart.ranger.core.finder.unsharded.UnshardedClusterFinder; -import com.flipkart.ranger.core.finder.unsharded.UnshardedClusterInfo; +import com.flipkart.ranger.core.finder.SimpleUnshardedServiceFinder; import com.flipkart.ranger.core.healthcheck.Healthchecks; +import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.zookeeper.ServiceFinderBuilders; import com.flipkart.ranger.zookeeper.ServiceProviderBuilders; @@ -58,23 +58,36 @@ public void stopTestCluster() throws Exception { } } + private static class UnshardedInfo { + + @Override + public int hashCode() { + return 0; + } + + @Override + public boolean equals(Object obj) { + return super.equals(obj); + } + } + @Test - public void testBasicDiscovery() throws Exception { - UnshardedClusterFinder serviceFinder = ServiceFinderBuilders.unshardedFinderBuilder() + public void testBasicDiscovery() { + SimpleUnshardedServiceFinder> serviceFinder = ServiceFinderBuilders.>unshardedFinderBuilder() .withConnectionString(testingCluster.getConnectString()) .withNamespace("test") .withServiceName("test-service") + .withDisableWatchers() .withDeserializer(data -> { try { return objectMapper.readValue(data, - new TypeReference>() { + new TypeReference>() { }); } catch (IOException e) { e.printStackTrace(); } return null; }) - .withDisableWatchers() .build(); serviceFinder.start(); { @@ -95,8 +108,8 @@ public void testBasicDiscovery() throws Exception { //while (true); } - private void registerService(String host, int port, int shardId) throws Exception { - val serviceProvider = ServiceProviderBuilders.unshardedServiceProviderBuilder() + private void registerService(String host, int port, int shardId) { + val serviceProvider = ServiceProviderBuilders.unshardedServiceProviderBuilder() .withConnectionString(testingCluster.getConnectString()) .withNamespace("test") .withServiceName("test-service") diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/servicehub/ServiceHubTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/servicehub/ServiceHubTest.java index 6292b505..e563d2e2 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/servicehub/ServiceHubTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/servicehub/ServiceHubTest.java @@ -1,20 +1,36 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.flipkart.ranger.zookeeper.servicehub; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import com.flipkart.ranger.core.finder.sharded.MapBasedServiceRegistry; +import com.flipkart.ranger.core.TestUtils; +import com.flipkart.ranger.core.finder.serviceregistry.MapBasedServiceRegistry; import com.flipkart.ranger.core.healthcheck.HealthcheckResult; import com.flipkart.ranger.core.healthcheck.HealthcheckStatus; +import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.signals.ExternalTriggeredSignal; import com.flipkart.ranger.core.util.Exceptions; -import com.flipkart.ranger.core.utils.TestUtils; import com.flipkart.ranger.zookeeper.ServiceProviderBuilders; -import com.flipkart.ranger.zookeeper.zk.ZkServiceDataSource; -import com.flipkart.ranger.zookeeper.zk.ZkShardedServiceFinderFactory; -import com.flipkart.ranger.zookeeper.zk.ZkServiceFinderHubBuilder; +import com.flipkart.ranger.zookeeper.servicefinderhub.ZkServiceDataSource; +import com.flipkart.ranger.zookeeper.servicefinderhub.ZkServiceFinderHubBuilder; +import com.flipkart.ranger.zookeeper.servicefinderhub.ZkShardedServiceFinderFactory; import lombok.Data; import lombok.extern.slf4j.Slf4j; import lombok.val; @@ -68,14 +84,14 @@ public void stopTestCluster() throws Exception { } @Test - public void testHub() throws InterruptedException { + public void testHub() { ExternalTriggeredSignal refreshHubSignal = new ExternalTriggeredSignal<>(() -> null, Collections.emptyList()); - val hub = new ZkServiceFinderHubBuilder>() + val hub = new ZkServiceFinderHubBuilder, MapBasedServiceRegistry>() .withCuratorFramework(curatorFramework) .withNamespace("test") .withRefreshFrequencyMs(1000) - .withServiceDataSource(new ZkServiceDataSource("test", curatorFramework)) - .withServiceFinderFactory(ZkShardedServiceFinderFactory.builder() + .withServiceDataSource(new ZkServiceDataSource("test", testingCluster.getConnectString(), curatorFramework)) + .withServiceFinderFactory(ZkShardedServiceFinderFactory.>builder() .curatorFramework(curatorFramework) .deserializer(this::read) .build()) @@ -96,7 +112,7 @@ public void testHub() throws InterruptedException { .withSerializer(this::write) .withNodeData(new TestShardInfo("prod")) .withHealthcheck(() -> HealthcheckStatus.healthy) - .withExtraRefreshSignal(refreshProviderSignal) + .withadditionalRefreshSignal(refreshProviderSignal) .withCuratorFramework(curatorFramework) .build(); provider1.start(); @@ -106,7 +122,7 @@ public void testHub() throws InterruptedException { TestUtils.sleepForSeconds(3); val node = hub.finder(new Service(NAMESPACE, "s1")) - .map(finder -> finder.get(new TestShardInfo("prod"))) + .map(finder -> finder.get(nodeData -> nodeData.getEnvironment().equalsIgnoreCase("prod"))) .orElse(null); Assert.assertNotNull(node); hub.stop(); diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/serviceprovider/BaseServiceProviderBuilderTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/serviceprovider/BaseServiceProviderBuilderTest.java index 110f9dd7..06bb8f49 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/serviceprovider/BaseServiceProviderBuilderTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/serviceprovider/BaseServiceProviderBuilderTest.java @@ -1,12 +1,12 @@ -/** +/* * Copyright 2015 Flipkart Internet Pvt. Ltd. - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + *

* http://www.apache.org/licenses/LICENSE-2.0 - * + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -52,7 +52,7 @@ public void stopTestCluster() throws Exception { } @Test - public void testbuilder() throws Exception { + public void testbuilder() { final String host = "localhost"; final int port = 9000; Exception exception = null; @@ -95,5 +95,6 @@ public void testbuilder() throws Exception { .withPort(port) .withHealthUpdateIntervalMs(1000) .build(); + serviceProvider.start(); } } \ No newline at end of file From 9bd12b9dc0c426364e2dfd857ce4546ab56a1205 Mon Sep 17 00:00:00 2001 From: Koushik R Date: Mon, 18 Oct 2021 15:23:29 +0530 Subject: [PATCH 02/69] a) Usage of val where it could be used. Retained type initialisation for when it has to be latter passed down as an arg to a function. b) .equalsIgnoreCase in selector tests has crept in during the previous sonar lints, corrected the same. c) Addressed sonar lint issues on license block --- .../nodeselector/RoundRobinServiceNodeSelectorTest.java | 8 ++++---- .../core/finder/shardselector/ListShardSelectorTest.java | 2 +- .../ranger/http/common/HttpNodeDataStoreConnector.java | 2 +- .../http/common/HttpNodeDataStoreConnectorTest.java | 3 ++- .../flipkart/ranger/http/config/HttpClientConfigTest.java | 2 +- .../ranger/http/model/ServiceNodeResponseTest.java | 2 +- .../http/model/ServiceRegistrationResponseTest.java | 2 +- .../HttpShardedServiceFinderBuilderTest.java | 2 +- .../http/servicefinderhub/HttpServiceDataSourceTest.java | 2 +- .../HttpShardedServiceProviderBuilderTest.java | 8 ++++---- 10 files changed, 17 insertions(+), 16 deletions(-) diff --git a/ranger-core/src/test/java/com/flipkart/ranger/core/finder/nodeselector/RoundRobinServiceNodeSelectorTest.java b/ranger-core/src/test/java/com/flipkart/ranger/core/finder/nodeselector/RoundRobinServiceNodeSelectorTest.java index 70563099..cb00c712 100644 --- a/ranger-core/src/test/java/com/flipkart/ranger/core/finder/nodeselector/RoundRobinServiceNodeSelectorTest.java +++ b/ranger-core/src/test/java/com/flipkart/ranger/core/finder/nodeselector/RoundRobinServiceNodeSelectorTest.java @@ -27,15 +27,15 @@ public class RoundRobinServiceNodeSelectorTest { @Test public void testRandomNodeSelector(){ final RoundRobinServiceNodeSelector roundRobinSelector = new RoundRobinServiceNodeSelector(); - List> serviceNodes = Lists.newArrayList(); + final List> serviceNodes = Lists.newArrayList(); serviceNodes.add(new ServiceNode<>("localhost-1", 9000, TestNodeData.builder().nodeId(1).build())); serviceNodes.add(new ServiceNode<>("localhost-2", 9001, TestNodeData.builder().nodeId(2).build())); serviceNodes.add(new ServiceNode<>("localhost-3", 9002, TestNodeData.builder().nodeId(3).build())); ServiceNode select = roundRobinSelector.select(serviceNodes); - Assert.assertTrue(select.getHost().equalsIgnoreCase("localhost-2")); + Assert.assertEquals("localhost-2", select.getHost()); select = roundRobinSelector.select(serviceNodes); - Assert.assertTrue(select.getHost().equalsIgnoreCase("localhost-3")); + Assert.assertEquals("localhost-3", select.getHost()); select = roundRobinSelector.select(serviceNodes); - Assert.assertTrue(select.getHost().equalsIgnoreCase("localhost-1")); + Assert.assertEquals("localhost-1", select.getHost()); } } diff --git a/ranger-core/src/test/java/com/flipkart/ranger/core/finder/shardselector/ListShardSelectorTest.java b/ranger-core/src/test/java/com/flipkart/ranger/core/finder/shardselector/ListShardSelectorTest.java index 1f747c24..614c493b 100644 --- a/ranger-core/src/test/java/com/flipkart/ranger/core/finder/shardselector/ListShardSelectorTest.java +++ b/ranger-core/src/test/java/com/flipkart/ranger/core/finder/shardselector/ListShardSelectorTest.java @@ -30,7 +30,7 @@ public class ListShardSelectorTest { @Test public void testNoOpShardSelector(){ - final ListBasedServiceRegistry serviceRegistry = RegistryTestUtils.getUnshardedRegistry(); + final ListBasedServiceRegistry serviceRegistry = RegistryTestUtils.getUnshardedRegistry(); final ListShardSelector> shardSelector = new ListShardSelector<>(); final List> nodes = shardSelector.nodes(CriteriaUtils.getCriteria(1), serviceRegistry); Assert.assertFalse(nodes.isEmpty()); diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/common/HttpNodeDataStoreConnector.java b/ranger-http/src/main/java/com/flipkart/ranger/http/common/HttpNodeDataStoreConnector.java index 4cb3bd7e..bbaf51a3 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/common/HttpNodeDataStoreConnector.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/common/HttpNodeDataStoreConnector.java @@ -48,7 +48,7 @@ public HttpNodeDataStoreConnector( .connectionPool(new ConnectionPool(1, 30, TimeUnit.SECONDS)) .build(); this.config = config; - this.mapper = null != mapper ? mapper : new ObjectMapper(); + this.mapper = mapper; } diff --git a/ranger-http/src/test/java/com/flipkart/ranger/http/common/HttpNodeDataStoreConnectorTest.java b/ranger-http/src/test/java/com/flipkart/ranger/http/common/HttpNodeDataStoreConnectorTest.java index caf6d5fc..71e2916e 100644 --- a/ranger-http/src/test/java/com/flipkart/ranger/http/common/HttpNodeDataStoreConnectorTest.java +++ b/ranger-http/src/test/java/com/flipkart/ranger/http/common/HttpNodeDataStoreConnectorTest.java @@ -1,4 +1,4 @@ -/** +/* * Copyright 2015 Flipkart Internet Pvt. Ltd. * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,6 +16,7 @@ package com.flipkart.ranger.http.common; import com.fasterxml.jackson.databind.ObjectMapper; +import com.flipkart.ranger.core.units.TestNodeData; import com.flipkart.ranger.http.config.HttpClientConfig; import org.junit.Assert; import org.junit.Test; diff --git a/ranger-http/src/test/java/com/flipkart/ranger/http/config/HttpClientConfigTest.java b/ranger-http/src/test/java/com/flipkart/ranger/http/config/HttpClientConfigTest.java index 336fb925..b1b0f68f 100644 --- a/ranger-http/src/test/java/com/flipkart/ranger/http/config/HttpClientConfigTest.java +++ b/ranger-http/src/test/java/com/flipkart/ranger/http/config/HttpClientConfigTest.java @@ -1,4 +1,4 @@ -/** +/* * Copyright 2015 Flipkart Internet Pvt. Ltd. * * Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/ranger-http/src/test/java/com/flipkart/ranger/http/model/ServiceNodeResponseTest.java b/ranger-http/src/test/java/com/flipkart/ranger/http/model/ServiceNodeResponseTest.java index 8c1f6f40..c3887954 100644 --- a/ranger-http/src/test/java/com/flipkart/ranger/http/model/ServiceNodeResponseTest.java +++ b/ranger-http/src/test/java/com/flipkart/ranger/http/model/ServiceNodeResponseTest.java @@ -1,4 +1,4 @@ -/** +/* * Copyright 2015 Flipkart Internet Pvt. Ltd. * * Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/ranger-http/src/test/java/com/flipkart/ranger/http/model/ServiceRegistrationResponseTest.java b/ranger-http/src/test/java/com/flipkart/ranger/http/model/ServiceRegistrationResponseTest.java index 42025c66..9949a859 100644 --- a/ranger-http/src/test/java/com/flipkart/ranger/http/model/ServiceRegistrationResponseTest.java +++ b/ranger-http/src/test/java/com/flipkart/ranger/http/model/ServiceRegistrationResponseTest.java @@ -1,4 +1,4 @@ -/** +/* * Copyright 2015 Flipkart Internet Pvt. Ltd. * * Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java b/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java index e01acac9..368c8549 100644 --- a/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java +++ b/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java @@ -1,4 +1,4 @@ -/** +/* * Copyright 2015 Flipkart Internet Pvt. Ltd. * * Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSourceTest.java b/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSourceTest.java index d5420f68..49a081a4 100644 --- a/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSourceTest.java +++ b/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSourceTest.java @@ -1,4 +1,4 @@ -/** +/* * Copyright 2015 Flipkart Internet Pvt. Ltd. * * Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/ranger-http/src/test/java/com/flipkart/ranger/http/serviceprovider/HttpShardedServiceProviderBuilderTest.java b/ranger-http/src/test/java/com/flipkart/ranger/http/serviceprovider/HttpShardedServiceProviderBuilderTest.java index 3c325024..16c48fce 100644 --- a/ranger-http/src/test/java/com/flipkart/ranger/http/serviceprovider/HttpShardedServiceProviderBuilderTest.java +++ b/ranger-http/src/test/java/com/flipkart/ranger/http/serviceprovider/HttpShardedServiceProviderBuilderTest.java @@ -1,4 +1,4 @@ -/** +/* * Copyright 2015 Flipkart Internet Pvt. Ltd. * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -51,8 +51,8 @@ public TestNodeData(@JsonProperty("farmId") String farmId) { @Test public void testProvider() throws Exception { - TestNodeData nm5NodeData = TestNodeData.builder().farmId("nm5").build(); - ServiceNode testNode = new ServiceNode<>("localhost-1", 80, nm5NodeData); + val nm5NodeData = TestNodeData.builder().farmId("nm5").build(); + val testNode = new ServiceNode<>("localhost-1", 80, nm5NodeData); val response = MAPPER.writeValueAsBytes( ServiceRegistrationResponse.builder() .success(true) @@ -69,7 +69,7 @@ public void testProvider() throws Exception { .connectionTimeoutMs(30_000) .operationTimeoutMs(30_000) .build(); - ServiceProvider> serviceProvider = new HttpShardedServiceProviderBuilder() + val serviceProvider = new HttpShardedServiceProviderBuilder() .withNamespace("testns") .withServiceName("test") .withHostname("localhost-1") From 77981eae653757deef8bb5227d3060e6c8dd79dd Mon Sep 17 00:00:00 2001 From: Koushik R Date: Mon, 18 Oct 2021 16:05:03 +0530 Subject: [PATCH 03/69] 1) Added error to httpResponses 2) Fixed formatting to align with source formatting c) Reverted the version change to have the APIs begin with a version --- .../http/model/ServiceDataSourceResponse.java | 3 +++ .../http/model/ServiceNodesResponse.java | 25 +++++++++---------- .../model/ServiceRegistrationResponse.java | 23 ++++++++--------- .../servicefinder/HttpNodeDataSource.java | 23 ++++++++--------- .../HttpServiceDataSource.java | 5 ++-- .../serviceprovider/HttpNodeDataSink.java | 10 +++++--- .../HttpShardedServiceFinderBuilderTest.java | 2 +- .../HttpServiceDataSourceTest.java | 2 +- ...HttpShardedServiceProviderBuilderTest.java | 2 +- 9 files changed, 49 insertions(+), 46 deletions(-) diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceDataSourceResponse.java b/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceDataSourceResponse.java index 59964417..33b378a8 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceDataSourceResponse.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceDataSourceResponse.java @@ -15,6 +15,7 @@ */ package com.flipkart.ranger.http.model; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.flipkart.ranger.core.model.Service; import lombok.*; @@ -25,7 +26,9 @@ @Getter @Setter @Builder +@JsonIgnoreProperties(ignoreUnknown = true) public class ServiceDataSourceResponse { private boolean success; + private String error; private List data; } diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceNodesResponse.java b/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceNodesResponse.java index 0d713cff..ef55a3a1 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceNodesResponse.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceNodesResponse.java @@ -15,26 +15,25 @@ */ package com.flipkart.ranger.http.model; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; import com.flipkart.ranger.core.model.ServiceNode; -import lombok.Builder; -import lombok.Data; +import lombok.*; +import java.util.Collections; import java.util.List; /** * */ -@Data +@AllArgsConstructor +@NoArgsConstructor +@Getter +@Setter +@Builder +@JsonIgnoreProperties(ignoreUnknown = true) public class ServiceNodesResponse { - private final boolean success; - final List> data; - - @Builder - public ServiceNodesResponse( - @JsonProperty("success") boolean success, - @JsonProperty("data") List> data) { - this.success = success; - this.data = data; - } + private boolean success; + private String error; + private List> data = Collections.emptyList(); } diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceRegistrationResponse.java b/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceRegistrationResponse.java index 897d760e..61594913 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceRegistrationResponse.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceRegistrationResponse.java @@ -15,18 +15,17 @@ */ package com.flipkart.ranger.http.model; -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Builder; -import lombok.Data; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import lombok.*; -@Data -public class ServiceRegistrationResponse { - private final boolean success; - @Builder - public ServiceRegistrationResponse( - @JsonProperty("success") boolean success - ) { - this.success = success; - } +@AllArgsConstructor +@NoArgsConstructor +@Getter +@Setter +@Builder +@JsonIgnoreProperties(ignoreUnknown = true) +public class ServiceRegistrationResponse { + private boolean success; + private String error; } diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpNodeDataSource.java b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpNodeDataSource.java index 7ca0bfc0..66b1450b 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpNodeDataSource.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpNodeDataSource.java @@ -33,6 +33,7 @@ import okhttp3.ResponseBody; import java.io.IOException; +import java.util.Collections; import java.util.List; import java.util.Optional; @@ -64,13 +65,13 @@ public Optional>> refresh(D deserializer) { .port(config.getPort() == 0 ? defaultPort() : config.getPort()) - .encodedPath(String.format("/v1/ranger/nodes/%s/%s", service.getNamespace(), service.getServiceName())) + .encodedPath(String.format("/ranger/nodes/v1/%s/%s", service.getNamespace(), service.getServiceName())) .build(); val request = new Request.Builder() .url(httpUrl) .get() .build(); - ServiceNodesResponse serviceNodesResponse = null; + try (val response = httpClient.newCall(request).execute()) { if (response.isSuccessful()) { try (final ResponseBody body = response.body()) { @@ -78,7 +79,14 @@ public Optional>> refresh(D deserializer) { log.warn("HTTP call to {} returned empty body", httpUrl); } else { final byte[] bytes = body.bytes(); - serviceNodesResponse = deserializer.deserialize(bytes); + val serviceNodesResponse = deserializer.deserialize(bytes); + if(serviceNodesResponse.isSuccess()){ + return Optional.of(FinderUtils.filterValidNodes( + service, + serviceNodesResponse.getData(), + healthcheckZombieCheckThresholdTime(service))); + } + log.warn("Http call to {} returned a failure response with error {}", httpUrl, serviceNodesResponse.getError()); } } } else { @@ -87,20 +95,11 @@ public Optional>> refresh(D deserializer) { } catch (IOException e) { Exceptions.illegalState("Error fetching data from server: " + httpUrl, e); } - - if (null != serviceNodesResponse && null != serviceNodesResponse.getData() && - !serviceNodesResponse.getData().isEmpty()) { - return Optional.of(FinderUtils.filterValidNodes( - service, - serviceNodesResponse.getData(), - healthcheckZombieCheckThresholdTime(service))); - } return Optional.empty(); } @Override public boolean isActive() { -// return httpClient.connectionPool().connectionCount() > 0; return true; } } diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSource.java b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSource.java index cb0af3f7..9268c84e 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSource.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSource.java @@ -53,7 +53,7 @@ public Collection services() { .port(config.getPort() == 0 ? defaultPort() : config.getPort()) - .encodedPath("/v1/ranger/services") + .encodedPath("/ranger/services/v1") .build(); val request = new Request.Builder() .url(httpUrl) @@ -72,11 +72,12 @@ public Collection services() { if(serviceDataSourceResponse.isSuccess()){ return serviceDataSourceResponse.getData(); } + log.warn("Http call to {} returned a failure response with error {}", httpUrl, serviceDataSourceResponse.getError()); } } } else { - log.warn("HTTP call to {} returned: {}", httpUrl, response.code()); + log.warn("HTTP call to {} returned code: {}", httpUrl, response.code()); } } catch (IOException e) { diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/serviceprovider/HttpNodeDataSink.java b/ranger-http/src/main/java/com/flipkart/ranger/http/serviceprovider/HttpNodeDataSink.java index 92263bf8..c5df26a6 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/serviceprovider/HttpNodeDataSink.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/serviceprovider/HttpNodeDataSink.java @@ -58,7 +58,7 @@ public void updateState(S serializer, ServiceNode serviceNode) { .port(config.getPort() == 0 ? defaultPort() : config.getPort()) - .encodedPath(String.format("/v1/ranger/nodes/add/%s/%s", service.getNamespace(), service.getServiceName())) + .encodedPath(String.format("/ranger/nodes/v1/add/%s/%s", service.getNamespace(), service.getServiceName())) .build(); val requestBody = RequestBody.create(serializer.serialize(serviceNode)); val serviceRegistrationResponse = registerService(httpUrl, requestBody); @@ -80,9 +80,11 @@ private Optional registerService(HttpUrl httpUrl, R } else { final byte[] bytes = body.bytes(); - return Optional.of( - mapper.readValue(bytes, ServiceRegistrationResponse.class) - ); + val serviceRegistrationResponse = mapper.readValue(bytes, ServiceRegistrationResponse.class); + if(serviceRegistrationResponse.isSuccess()) { + return Optional.of(serviceRegistrationResponse); + } + log.warn("Http call to {} returned a failure response with code {} and error {}", httpUrl, response.code(), serviceRegistrationResponse.getError()); } } } diff --git a/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java b/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java index 368c8549..fb9f5202 100644 --- a/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java +++ b/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java @@ -67,7 +67,7 @@ public void testFinder() throws Exception { .data(Lists.newArrayList(node)) .success(true) .build()); - server.stubFor(get(urlEqualTo("/v1/ranger/nodes/testns/test")) + server.stubFor(get(urlEqualTo("/ranger/nodes/v1/testns/test")) .willReturn(aResponse() .withBody(payload) .withStatus(200))); diff --git a/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSourceTest.java b/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSourceTest.java index 49a081a4..5da90812 100644 --- a/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSourceTest.java +++ b/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSourceTest.java @@ -62,7 +62,7 @@ public void testServiceDataSource() throws IOException { )) .build(); val response = MAPPER.writeValueAsBytes(responseObj); - server.stubFor(get(urlEqualTo("/v1/ranger/services")) + server.stubFor(get(urlEqualTo("/ranger/services/v1")) .willReturn(aResponse() .withBody(response) .withStatus(200))); diff --git a/ranger-http/src/test/java/com/flipkart/ranger/http/serviceprovider/HttpShardedServiceProviderBuilderTest.java b/ranger-http/src/test/java/com/flipkart/ranger/http/serviceprovider/HttpShardedServiceProviderBuilderTest.java index 16c48fce..f83832b6 100644 --- a/ranger-http/src/test/java/com/flipkart/ranger/http/serviceprovider/HttpShardedServiceProviderBuilderTest.java +++ b/ranger-http/src/test/java/com/flipkart/ranger/http/serviceprovider/HttpShardedServiceProviderBuilderTest.java @@ -58,7 +58,7 @@ public void testProvider() throws Exception { .success(true) .build()); byte[] requestBytes = MAPPER.writeValueAsBytes(testNode); - server.stubFor(post(urlEqualTo("/v1/ranger/nodes/add/testns/test")) + server.stubFor(post(urlEqualTo("/ranger/nodes/v1/add/testns/test")) .withRequestBody(binaryEqualTo(requestBytes)) .willReturn(aResponse() .withBody(response) From 7fdeb8f657e56c1023dbdc14233b2adb7a59c5b7 Mon Sep 17 00:00:00 2001 From: Koushik R Date: Mon, 18 Oct 2021 16:39:01 +0530 Subject: [PATCH 04/69] 1. Added a ranger-client module to contain the client creation abstractions and added http and zk clients respectively with required test cases. This will help end-clients not replicate the same boiler plate code again and again --- pom.xml | 4 +- {ranger-server => ranger-client}/pom.xml | 16 +- .../client/AbstractRangerHubClient.java | 111 +++++++++++ .../flipkart/ranger/client/RangerClient.java | 39 ++++ .../ranger/client/RangerClientConstants.java | 24 +++ .../ranger/client/RangerHubClient.java | 39 ++++ .../client/AbstractRangerHubClientTest.java | 183 ++++++++++++++++++ ranger-http-client/pom.xml | 39 ++++ .../http/ShardedRangerHttpHubClient.java | 92 +++++++++ .../client/http/SimpleRangerHttpClient.java | 99 ++++++++++ .../http/UnshardedRangerHttpHubClient.java | 88 +++++++++ .../client/http/BaseRangerHttpClientTest.java | 101 ++++++++++ .../http/ShardedRangerHttpClientTest.java | 56 ++++++ .../http/SimpleRangerHttpClientTest.java | 56 ++++++ .../http/UnshardedRangerHttpClientTest.java | 56 ++++++ .../HttpShardedServiceFinderFactory.java | 2 +- ranger-zk-client/pom.xml | 38 ++++ .../client/zk/AbstractRangerZKHubClient.java | 95 +++++++++ .../client/zk/ShardedRangerZKHubClient.java | 74 +++++++ .../client/zk/SimpleRangerZKClient.java | 136 +++++++++++++ .../client/zk/UnshardedRangerZKHubClient.java | 74 +++++++ .../client/zk/BaseRangerZKClientTest.java | 106 ++++++++++ .../client/zk/ShardedZKRangerClientTest.java | 41 ++++ .../client/zk/SimpleRangerZKClientTest.java | 36 ++++ .../zk/UnshardedZKRangerClientTest.java | 41 ++++ 25 files changed, 1641 insertions(+), 5 deletions(-) rename {ranger-server => ranger-client}/pom.xml (55%) create mode 100644 ranger-client/src/main/java/com/flipkart/ranger/client/AbstractRangerHubClient.java create mode 100644 ranger-client/src/main/java/com/flipkart/ranger/client/RangerClient.java create mode 100644 ranger-client/src/main/java/com/flipkart/ranger/client/RangerClientConstants.java create mode 100644 ranger-client/src/main/java/com/flipkart/ranger/client/RangerHubClient.java create mode 100644 ranger-client/src/test/java/com/flipkart/ranger/client/AbstractRangerHubClientTest.java create mode 100644 ranger-http-client/pom.xml create mode 100644 ranger-http-client/src/main/java/com/flipkart/ranger/client/http/ShardedRangerHttpHubClient.java create mode 100644 ranger-http-client/src/main/java/com/flipkart/ranger/client/http/SimpleRangerHttpClient.java create mode 100644 ranger-http-client/src/main/java/com/flipkart/ranger/client/http/UnshardedRangerHttpHubClient.java create mode 100644 ranger-http-client/src/test/java/com/flipkart/ranger/client/http/BaseRangerHttpClientTest.java create mode 100644 ranger-http-client/src/test/java/com/flipkart/ranger/client/http/ShardedRangerHttpClientTest.java create mode 100644 ranger-http-client/src/test/java/com/flipkart/ranger/client/http/SimpleRangerHttpClientTest.java create mode 100644 ranger-http-client/src/test/java/com/flipkart/ranger/client/http/UnshardedRangerHttpClientTest.java create mode 100644 ranger-zk-client/pom.xml create mode 100644 ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/AbstractRangerZKHubClient.java create mode 100644 ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/ShardedRangerZKHubClient.java create mode 100644 ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/SimpleRangerZKClient.java create mode 100644 ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/UnshardedRangerZKHubClient.java create mode 100644 ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/BaseRangerZKClientTest.java create mode 100644 ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/ShardedZKRangerClientTest.java create mode 100644 ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/SimpleRangerZKClientTest.java create mode 100644 ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/UnshardedZKRangerClientTest.java diff --git a/pom.xml b/pom.xml index 7573ca59..5e8c5b9c 100644 --- a/pom.xml +++ b/pom.xml @@ -12,7 +12,9 @@ ranger-core ranger-zookeeper ranger-http - ranger-server + ranger-client + ranger-zk-client + ranger-http-client diff --git a/ranger-server/pom.xml b/ranger-client/pom.xml similarity index 55% rename from ranger-server/pom.xml rename to ranger-client/pom.xml index 8743e241..bc563c1a 100644 --- a/ranger-server/pom.xml +++ b/ranger-client/pom.xml @@ -9,14 +9,24 @@ 4.0.0 - ranger-server + ranger-client com.flipkart.ranger - ranger-zookeeper + ranger-core ${project.version} + + com.flipkart.ranger + ranger-core + ${project.version} + test-jar + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + - \ No newline at end of file diff --git a/ranger-client/src/main/java/com/flipkart/ranger/client/AbstractRangerHubClient.java b/ranger-client/src/main/java/com/flipkart/ranger/client/AbstractRangerHubClient.java new file mode 100644 index 00000000..8bb5e42a --- /dev/null +++ b/ranger-client/src/main/java/com/flipkart/ranger/client/AbstractRangerHubClient.java @@ -0,0 +1,111 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.client; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.flipkart.ranger.core.finderhub.ServiceDataSource; +import com.flipkart.ranger.core.finderhub.ServiceFinderFactory; +import com.flipkart.ranger.core.finderhub.ServiceFinderHub; +import com.flipkart.ranger.core.model.*; +import com.google.common.base.Preconditions; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import lombok.val; + +import java.util.List; +import java.util.Optional; + +@Slf4j +@Getter +public abstract class AbstractRangerHubClient, R extends ServiceRegistry, D extends Deserializer> implements RangerHubClient { + + private final String namespace; + private final ObjectMapper mapper; + private final C criteria; + private final D deserializer; + + private int nodeRefreshTimeMs; + private ServiceFinderHub hub; + + public AbstractRangerHubClient( + String namespace, + ObjectMapper mapper, + int nodeRefreshTimeMs, + C criteria, + D deserializer + ){ + this.namespace = namespace; + this.mapper = mapper; + this.nodeRefreshTimeMs = nodeRefreshTimeMs; + this.criteria = criteria; + this.deserializer = deserializer; + } + + public void start(){ + Preconditions.checkNotNull(mapper, "Mapper can't be null"); + Preconditions.checkNotNull(namespace, "namespace can't be null"); + Preconditions.checkNotNull(deserializer, "deserializer can't be null"); + + if (this.nodeRefreshTimeMs < RangerClientConstants.MINIMUM_REFRESH_TIME) { + log.warn("Node info update interval too low: {} ms. Has been upgraded to {} ms ", + this.nodeRefreshTimeMs, + RangerClientConstants.MINIMUM_REFRESH_TIME); + } + this.nodeRefreshTimeMs = Math.max(RangerClientConstants.MINIMUM_REFRESH_TIME, this.nodeRefreshTimeMs); + this.hub = buildHub(); + this.hub.start(); + } + + public void stop(){ + hub.stop(); + } + + public Optional> getNode( + final Service service + ){ + return getNode(service, criteria); + } + + public Optional>> getAllNodes( + final Service service + ){ + return getAllNodes(service, criteria); + } + + public Optional> getNode( + final Service service, + final C criteria + ){ + val optionalFinder = this.getHub().finder(service); + return optionalFinder.map(trcServiceFinder -> trcServiceFinder.get(criteria)); + } + + public Optional>> getAllNodes( + final Service service, + final C criteria + ){ + val optionalFinder = this.getHub().finder(service); + return optionalFinder.map(trcServiceFinder -> trcServiceFinder.getAll(criteria)); + } + + protected abstract ServiceFinderHub buildHub(); + + protected abstract ServiceDataSource buildServiceDataSource(); + + protected abstract ServiceFinderFactory buildFinderFactory(); + +} + diff --git a/ranger-client/src/main/java/com/flipkart/ranger/client/RangerClient.java b/ranger-client/src/main/java/com/flipkart/ranger/client/RangerClient.java new file mode 100644 index 00000000..57738a69 --- /dev/null +++ b/ranger-client/src/main/java/com/flipkart/ranger/client/RangerClient.java @@ -0,0 +1,39 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.flipkart.ranger.client; + +import com.flipkart.ranger.core.model.Criteria; +import com.flipkart.ranger.core.model.Deserializer; +import com.flipkart.ranger.core.model.ServiceNode; + +import java.util.List; +import java.util.Optional; + +public interface RangerClient> { + + void start(); + + void stop(); + + Optional> getNode(); + + Optional> getNode(final C criteria); + + Optional>> getAllNodes(); + + Optional>> getAllNodes(final C criteria); +} diff --git a/ranger-client/src/main/java/com/flipkart/ranger/client/RangerClientConstants.java b/ranger-client/src/main/java/com/flipkart/ranger/client/RangerClientConstants.java new file mode 100644 index 00000000..5f9fa9c0 --- /dev/null +++ b/ranger-client/src/main/java/com/flipkart/ranger/client/RangerClientConstants.java @@ -0,0 +1,24 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.client; + +public class RangerClientConstants { + + private RangerClientConstants(){} + + public static final int CONNECTION_RETRY_TIME = 5000; + public static final int MINIMUM_REFRESH_TIME = 5000; +} diff --git a/ranger-client/src/main/java/com/flipkart/ranger/client/RangerHubClient.java b/ranger-client/src/main/java/com/flipkart/ranger/client/RangerHubClient.java new file mode 100644 index 00000000..f9a2534f --- /dev/null +++ b/ranger-client/src/main/java/com/flipkart/ranger/client/RangerHubClient.java @@ -0,0 +1,39 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.client; + +import com.flipkart.ranger.core.model.Criteria; +import com.flipkart.ranger.core.model.Service; +import com.flipkart.ranger.core.model.ServiceNode; + +import java.util.List; +import java.util.Optional; + +public interface RangerHubClient> { + + void start(); + + void stop(); + + Optional> getNode(final Service service); + + Optional> getNode(final Service service, final C criteria); + + Optional>> getAllNodes(final Service service); + + Optional>> getAllNodes(final Service service, final C criteria); + +} diff --git a/ranger-client/src/test/java/com/flipkart/ranger/client/AbstractRangerHubClientTest.java b/ranger-client/src/test/java/com/flipkart/ranger/client/AbstractRangerHubClientTest.java new file mode 100644 index 00000000..6324974a --- /dev/null +++ b/ranger-client/src/test/java/com/flipkart/ranger/client/AbstractRangerHubClientTest.java @@ -0,0 +1,183 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.client; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.flipkart.ranger.core.TestUtils; +import com.flipkart.ranger.core.finder.ServiceFinder; +import com.flipkart.ranger.core.finder.SimpleUnshardedServiceFinder; +import com.flipkart.ranger.core.finder.SimpleUnshardedServiceFinderBuilder; +import com.flipkart.ranger.core.finder.serviceregistry.ListBasedServiceRegistry; +import com.flipkart.ranger.core.finderhub.*; +import com.flipkart.ranger.core.model.*; +import com.google.common.collect.Lists; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.junit.Assert; +import org.junit.Test; + +import java.util.List; +import java.util.Optional; + +@Slf4j +public class AbstractRangerHubClientTest { + + private static final ObjectMapper mapper = new ObjectMapper(); + private static final Service service = new Service("test-ns", "test-s"); + + @Data + @Builder + @AllArgsConstructor + private static class TestShardInfo{ + private int shardId; + } + + private static class TestCriteria implements Criteria{ + @Override + public boolean apply(TestShardInfo nodeData) { + return nodeData.getShardId() == 1; + } + } + + + @Builder + private static class TestSimpleUnshardedServiceFinder> + extends SimpleUnshardedServiceFinderBuilder, Deserializer, Criteria> { + + @Override + public SimpleUnshardedServiceFinder> build() { + return buildFinder(); + } + + @Override + protected NodeDataSource> dataSource(Service service) { + return new NodeDataSource>() { + @Override + public Optional>> refresh(Deserializer deserializer) { + return Optional.of( + Lists.newArrayList( + new ServiceNode<>("localhost", 9200, TestShardInfo.builder().shardId(1).build()) + ) + ); + } + + @Override + public void start() { + + } + + @Override + public void ensureConnected() { + + } + + @Override + public void stop() { + + } + + @Override + public boolean isActive() { + return true; + } + }; + } + } + + private static class TestServiceFinderFactory implements ServiceFinderFactory, ListBasedServiceRegistry>{ + + @Override + public ServiceFinder, ListBasedServiceRegistry> buildFinder(Service service) { + SimpleUnshardedServiceFinder> finder = new TestSimpleUnshardedServiceFinder>() + .withNamespace(service.getNamespace()) + .withServiceName(service.getServiceName()) + .withDeserializer(new Deserializer() { + @Override + public int hashCode() { + return super.hashCode(); + } + }) + .build(); + finder.start(); + return finder; + } + } + + private static class TestDeserilizer implements Deserializer{ + + } + + private static class TestAbstractRangerHub extends AbstractRangerHubClient, ListBasedServiceRegistry, TestDeserilizer> { + + public TestAbstractRangerHub(String namespace, ObjectMapper mapper, int nodeRefreshTimeMs, Criteria criteria, TestDeserilizer deserilizer) { + super(namespace, mapper, nodeRefreshTimeMs, criteria, deserilizer); + } + + @Override + protected ServiceFinderHub, ListBasedServiceRegistry> buildHub() { + return new ServiceFinderHubBuilder, ListBasedServiceRegistry>() { + @Override + protected void preBuild() { + + } + + @Override + protected void postBuild(ServiceFinderHub, ListBasedServiceRegistry> serviceFinderHub) { + + } + }.withServiceDataSource(buildServiceDataSource()) + .withServiceFinderFactory(buildFinderFactory()) + .build(); + } + + @Override + protected ServiceDataSource buildServiceDataSource() { + return new StaticDataSource(Lists.newArrayList(service)); + } + + @Override + protected ServiceFinderFactory, ListBasedServiceRegistry> buildFinderFactory() { + return new TestServiceFinderFactory(); + } + } + + @Test + public void testAbstractHubClient() { + TestAbstractRangerHub testAbstractHub = new TestAbstractRangerHub(service.getNamespace(), mapper, 1000, new TestCriteria(), new TestDeserilizer<>()); + testAbstractHub.start(); + + TestUtils.sleepForSeconds(3); + + Optional> node = testAbstractHub.getNode(service); + Assert.assertTrue(node.isPresent()); + Assert.assertTrue(node.get().getHost().equalsIgnoreCase("localhost")); + Assert.assertEquals(9200, node.get().getPort()); + Assert.assertEquals(1, node.get().getNodeData().getShardId()); + + node = testAbstractHub.getNode(new Service("test", "test")); + Assert.assertFalse(node.isPresent()); + + node = testAbstractHub.getNode(service, nodeData -> nodeData.getShardId() == 2); + Assert.assertFalse(node.isPresent()); + + node = testAbstractHub.getNode(new Service("test", "test"), nodeData -> nodeData.getShardId() == 1); + Assert.assertFalse(node.isPresent()); + + testAbstractHub.start(); + } +} diff --git a/ranger-http-client/pom.xml b/ranger-http-client/pom.xml new file mode 100644 index 00000000..9041de0a --- /dev/null +++ b/ranger-http-client/pom.xml @@ -0,0 +1,39 @@ + + + + ranger + com.flipkart.ranger + 1.0-SNAPSHOT + + 4.0.0 + + ranger-http-client + + + + com.flipkart.ranger + ranger-http + ${project.version} + + + com.flipkart.ranger + ranger-client + ${project.version} + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + + + com.github.tomakehurst + wiremock + 2.25.1 + test + + + + + \ No newline at end of file diff --git a/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/ShardedRangerHttpHubClient.java b/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/ShardedRangerHttpHubClient.java new file mode 100644 index 00000000..1cf4d081 --- /dev/null +++ b/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/ShardedRangerHttpHubClient.java @@ -0,0 +1,92 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.client.http; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.flipkart.ranger.client.AbstractRangerHubClient; +import com.flipkart.ranger.core.finder.nodeselector.RoundRobinServiceNodeSelector; +import com.flipkart.ranger.core.finder.serviceregistry.MapBasedServiceRegistry; +import com.flipkart.ranger.core.finder.shardselector.MatchingShardSelector; +import com.flipkart.ranger.core.finderhub.ServiceDataSource; +import com.flipkart.ranger.core.finderhub.ServiceFinderFactory; +import com.flipkart.ranger.core.finderhub.ServiceFinderHub; +import com.flipkart.ranger.core.finderhub.StaticDataSource; +import com.flipkart.ranger.core.model.Criteria; +import com.flipkart.ranger.core.model.Service; +import com.flipkart.ranger.http.config.HttpClientConfig; +import com.flipkart.ranger.http.serde.HTTPResponseDataDeserializer; +import com.flipkart.ranger.http.servicefinderhub.HttpServiceDataSource; +import com.flipkart.ranger.http.servicefinderhub.HttpServiceFinderHubBuilder; +import com.flipkart.ranger.http.servicefinderhub.HttpShardedServiceFinderFactory; +import lombok.Builder; +import lombok.extern.slf4j.Slf4j; + +import java.util.List; + +@Slf4j +public class ShardedRangerHttpHubClient> extends AbstractRangerHubClient, HTTPResponseDataDeserializer> { + + private final List services; + private final HttpClientConfig clientConfig; + + @Builder + public ShardedRangerHttpHubClient( + String namespace, + ObjectMapper mapper, + int nodeRefreshIntervalMs, + C criteria, + HTTPResponseDataDeserializer deserializer, + HttpClientConfig clientConfig, + List services + ) { + super(namespace, mapper, nodeRefreshIntervalMs, criteria, deserializer); + this.clientConfig = clientConfig; + this.services = services; + } + + @Override + protected ServiceFinderHub> buildHub() { + return new HttpServiceFinderHubBuilder>() + .withServiceDataSource(buildServiceDataSource()) + .withServiceFinderFactory(buildFinderFactory()) + .withRefreshFrequencyMs(getNodeRefreshTimeMs()) + .build(); + } + + /* + If the client knows the services for which a refresh ought to be run. Use that + information instead of a data source! Handy during service integrations! + */ + @Override + protected ServiceDataSource buildServiceDataSource() { + return null != services && !services.isEmpty() ? + new StaticDataSource(services) : + new HttpServiceDataSource<>(clientConfig, getMapper()); + } + + @Override + protected ServiceFinderFactory> buildFinderFactory() { + return HttpShardedServiceFinderFactory.builder() + .httpClientConfig(clientConfig) + .nodeRefreshIntervalMs(getNodeRefreshTimeMs()) + .deserializer(getDeserializer()) + .shardSelector(new MatchingShardSelector<>()) + .nodeSelector(new RoundRobinServiceNodeSelector<>()) + .mapper(getMapper()) + .build(); + } + +} diff --git a/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/SimpleRangerHttpClient.java b/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/SimpleRangerHttpClient.java new file mode 100644 index 00000000..8efb5b2b --- /dev/null +++ b/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/SimpleRangerHttpClient.java @@ -0,0 +1,99 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.client.http; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.flipkart.ranger.client.RangerClient; +import com.flipkart.ranger.core.finder.SimpleUnshardedServiceFinder; +import com.flipkart.ranger.core.finder.shardselector.ListShardSelector; +import com.flipkart.ranger.core.model.Criteria; +import com.flipkart.ranger.core.model.ServiceNode; +import com.flipkart.ranger.http.HttpServiceFinderBuilders; +import com.flipkart.ranger.http.config.HttpClientConfig; +import com.flipkart.ranger.http.serde.HTTPResponseDataDeserializer; +import com.google.common.base.Preconditions; +import lombok.Builder; +import lombok.extern.slf4j.Slf4j; + +import java.util.List; +import java.util.Optional; + +@Slf4j +public class SimpleRangerHttpClient, D extends HTTPResponseDataDeserializer> implements RangerClient { + + private final C criteria; + private final SimpleUnshardedServiceFinder serviceFinder; + + @Builder + public SimpleRangerHttpClient( + String namespace, + String serviceName, + ObjectMapper mapper, + int nodeRefreshIntervalMs, + HttpClientConfig clientConfig, + C criteria, + D deserializer + ) { + Preconditions.checkNotNull(mapper, "Mapper can't be null"); + Preconditions.checkNotNull(namespace, "namespace can't be null"); + Preconditions.checkNotNull(deserializer, "deserializer can't be null"); + + this.criteria = criteria; + + this.serviceFinder = HttpServiceFinderBuilders.httpUnshardedServiceFinderBuilider() + .withClientConfig(clientConfig) + .withServiceName(serviceName) + .withNamespace(namespace) + .withObjectMapper(mapper) + .withNodeRefreshIntervalMs(nodeRefreshIntervalMs) + .withDeserializer(deserializer) + .withShardSelector(new ListShardSelector<>()) + .build(); + } + + @Override + public void start() { + log.info("Starting the service finder"); + this.serviceFinder.start(); + } + + @Override + public void stop() { + log.info("Stopping the service finder"); + this.serviceFinder.stop(); + } + + @Override + public Optional> getNode() { + return getNode(criteria); + } + + @Override + public Optional>> getAllNodes() { + return getAllNodes(criteria); + } + + @Override + public Optional> getNode(C criteria) { + return Optional.ofNullable(this.serviceFinder.get(criteria)); + } + + @Override + public Optional>> getAllNodes(C criteria) { + return Optional.ofNullable(this.serviceFinder.getAll(criteria)); + } +} + diff --git a/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/UnshardedRangerHttpHubClient.java b/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/UnshardedRangerHttpHubClient.java new file mode 100644 index 00000000..7765f3a3 --- /dev/null +++ b/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/UnshardedRangerHttpHubClient.java @@ -0,0 +1,88 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.client.http; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.flipkart.ranger.client.AbstractRangerHubClient; +import com.flipkart.ranger.core.finder.nodeselector.RoundRobinServiceNodeSelector; +import com.flipkart.ranger.core.finder.serviceregistry.ListBasedServiceRegistry; +import com.flipkart.ranger.core.finder.shardselector.ListShardSelector; +import com.flipkart.ranger.core.finderhub.ServiceDataSource; +import com.flipkart.ranger.core.finderhub.ServiceFinderFactory; +import com.flipkart.ranger.core.finderhub.ServiceFinderHub; +import com.flipkart.ranger.core.finderhub.StaticDataSource; +import com.flipkart.ranger.core.model.Criteria; +import com.flipkart.ranger.core.model.Service; +import com.flipkart.ranger.http.config.HttpClientConfig; +import com.flipkart.ranger.http.serde.HTTPResponseDataDeserializer; +import com.flipkart.ranger.http.servicefinderhub.HttpServiceDataSource; +import com.flipkart.ranger.http.servicefinderhub.HttpServiceFinderHubBuilder; +import com.flipkart.ranger.http.servicefinderhub.HttpUnshardedServiceFinderFactory; +import lombok.Builder; +import lombok.extern.slf4j.Slf4j; + +import java.util.List; + +@Slf4j +public class UnshardedRangerHttpHubClient> extends AbstractRangerHubClient, HTTPResponseDataDeserializer> { + + private final List services; + private final HttpClientConfig clientConfig; + + @Builder + public UnshardedRangerHttpHubClient( + String namespace, + ObjectMapper mapper, + int nodeRefreshIntervalMs, + C criteria, + HTTPResponseDataDeserializer deserializer, + HttpClientConfig clientConfig, + List services + ) { + super(namespace, mapper, nodeRefreshIntervalMs, criteria, deserializer); + this.clientConfig = clientConfig; + this.services = services; + } + + @Override + protected ServiceFinderHub> buildHub() { + return new HttpServiceFinderHubBuilder>() + .withServiceDataSource(buildServiceDataSource()) + .withServiceFinderFactory(buildFinderFactory()) + .withRefreshFrequencyMs(getNodeRefreshTimeMs()) + .build(); + } + + @Override + protected ServiceDataSource buildServiceDataSource() { + return null != services && !services.isEmpty() ? + new StaticDataSource(services) : + new HttpServiceDataSource<>(clientConfig, getMapper()); + } + + @Override + protected ServiceFinderFactory> buildFinderFactory() { + return HttpUnshardedServiceFinderFactory.builder() + .httpClientConfig(clientConfig) + .nodeRefreshIntervalMs(getNodeRefreshTimeMs()) + .deserializer(getDeserializer()) + .shardSelector(new ListShardSelector<>()) + .nodeSelector(new RoundRobinServiceNodeSelector<>()) + .mapper(getMapper()) + .build(); + } + +} diff --git a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/BaseRangerHttpClientTest.java b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/BaseRangerHttpClientTest.java new file mode 100644 index 00000000..f1e92621 --- /dev/null +++ b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/BaseRangerHttpClientTest.java @@ -0,0 +1,101 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.client.http; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.flipkart.ranger.core.healthcheck.HealthcheckStatus; +import com.flipkart.ranger.core.model.Service; +import com.flipkart.ranger.core.model.ServiceNode; +import com.flipkart.ranger.core.units.TestNodeData; +import com.flipkart.ranger.core.util.Exceptions; +import com.flipkart.ranger.http.config.HttpClientConfig; +import com.flipkart.ranger.http.model.ServiceDataSourceResponse; +import com.flipkart.ranger.http.model.ServiceNodesResponse; +import com.github.tomakehurst.wiremock.junit.WireMockRule; +import com.google.common.collect.Lists; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import lombok.val; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; + +import java.io.IOException; + +import static com.github.tomakehurst.wiremock.client.WireMock.*; + +@Slf4j +@Getter +public abstract class BaseRangerHttpClientTest { + + private final ObjectMapper objectMapper = new ObjectMapper(); + @Rule + public WireMockRule server = new WireMockRule(8888); + private HttpClientConfig httpClientConfig; + + @Before + public void startTestCluster() throws Exception { + final TestNodeData testNode = new TestNodeData(1); + final ServiceNode node = new ServiceNode<>("127.0.0.1", 80, testNode); + node.setHealthcheckStatus(HealthcheckStatus.healthy); + node.setLastUpdatedTimeStamp(System.currentTimeMillis()); + val payload = objectMapper.writeValueAsBytes( + ServiceNodesResponse.builder() + .data(Lists.newArrayList(node)) + .success(true) + .build()); + server.stubFor(get(urlEqualTo("/ranger/nodes/v1/test-n/test-s")) + .willReturn(aResponse() + .withBody(payload) + .withStatus(200))); + + val responseObj = ServiceDataSourceResponse.builder() + .success(true) + .data(Lists.newArrayList( + new Service("test-n", "test-s") + )) + .build(); + val response = objectMapper.writeValueAsBytes(responseObj); + server.stubFor(get(urlEqualTo("/ranger/services/v1")) + .willReturn(aResponse() + .withBody(response) + .withStatus(200))); + + httpClientConfig = HttpClientConfig.builder() + .host("127.0.0.1") + .port(server.port()) + .connectionTimeoutMs(30_000) + .operationTimeoutMs(30_000) + .build(); + log.debug("Started http subsystem"); + } + + @After + public void stopTestCluster() { + log.debug("Stopping http subsystem"); + } + + protected ServiceNodesResponse read(final byte[] data) { + try { + return getObjectMapper().readValue(data, new TypeReference>() {}); + } + catch (IOException e) { + Exceptions.illegalState(e); + } + return null; + } +} diff --git a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/ShardedRangerHttpClientTest.java b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/ShardedRangerHttpClientTest.java new file mode 100644 index 00000000..93596836 --- /dev/null +++ b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/ShardedRangerHttpClientTest.java @@ -0,0 +1,56 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.client.http; + +import com.flipkart.ranger.core.TestUtils; +import com.flipkart.ranger.core.model.Criteria; +import com.flipkart.ranger.core.model.Service; +import com.flipkart.ranger.core.model.ServiceNode; +import com.flipkart.ranger.core.units.TestNodeData; +import lombok.val; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Optional; + +public class ShardedRangerHttpClientTest extends BaseRangerHttpClientTest { + + @Test + public void testShardedHttpHubClient(){ + val client = ShardedRangerHttpHubClient.>builder() + .clientConfig(getHttpClientConfig()) + .namespace("test-n") + .deserializer(this::read) + .mapper(getObjectMapper()) + .nodeRefreshIntervalMs(5000) + .build(); + client.start(); + + TestUtils.sleepForSeconds(6); + + val service = new Service("test-n", "test-s"); + Optional> node = client.getNode(service); + Assert.assertTrue(node.isPresent()); + + node = client.getNode(service, nodeData -> nodeData.getNodeId() == 1); + Assert.assertTrue(node.isPresent()); + + node = client.getNode(service, nodeData -> nodeData.getNodeId() == 2); + Assert.assertFalse(node.isPresent()); + + client.stop(); + } +} diff --git a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/SimpleRangerHttpClientTest.java b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/SimpleRangerHttpClientTest.java new file mode 100644 index 00000000..c55b4971 --- /dev/null +++ b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/SimpleRangerHttpClientTest.java @@ -0,0 +1,56 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.client.http; + +import com.flipkart.ranger.core.TestUtils; +import com.flipkart.ranger.core.model.Criteria; +import com.flipkart.ranger.core.model.ServiceNode; +import com.flipkart.ranger.core.units.TestNodeData; +import com.flipkart.ranger.http.serde.HTTPResponseDataDeserializer; +import lombok.val; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Optional; + +public class SimpleRangerHttpClientTest extends BaseRangerHttpClientTest{ + + @Test + public void testSimpleHttpRangerClient(){ + val client = SimpleRangerHttpClient., HTTPResponseDataDeserializer>builder() + .clientConfig(getHttpClientConfig()) + .mapper(getObjectMapper()) + .deserializer(this::read) + .namespace("test-n") + .serviceName("test-s") + .nodeRefreshIntervalMs(5000) + .build(); + client.start(); + + TestUtils.sleepForSeconds(6); + + Optional> node = client.getNode(); + Assert.assertTrue(node.isPresent()); + + node = client.getNode(nodeData -> nodeData.getNodeId() == 1); + Assert.assertTrue(node.isPresent()); + + node = client.getNode(nodeData -> nodeData.getNodeId() == 2); + Assert.assertFalse(node.isPresent()); + + client.stop(); + } +} diff --git a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/UnshardedRangerHttpClientTest.java b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/UnshardedRangerHttpClientTest.java new file mode 100644 index 00000000..10fd6017 --- /dev/null +++ b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/UnshardedRangerHttpClientTest.java @@ -0,0 +1,56 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.client.http; + +import com.flipkart.ranger.core.TestUtils; +import com.flipkart.ranger.core.model.Criteria; +import com.flipkart.ranger.core.model.Service; +import com.flipkart.ranger.core.model.ServiceNode; +import com.flipkart.ranger.core.units.TestNodeData; +import lombok.val; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Optional; + +public class UnshardedRangerHttpClientTest extends BaseRangerHttpClientTest { + + @Test + public void testUnshardedRangerHubClient(){ + val client = UnshardedRangerHttpHubClient.>builder() + .clientConfig(getHttpClientConfig()) + .namespace("test-n") + .deserializer(this::read) + .mapper(getObjectMapper()) + .nodeRefreshIntervalMs(5000) + .build(); + client.start(); + + TestUtils.sleepForSeconds(6); + + val service = new Service("test-n", "test-s"); + Optional> node = client.getNode(service); + Assert.assertTrue(node.isPresent()); + + node = client.getNode(service, nodeData -> nodeData.getNodeId() == 1); + Assert.assertTrue(node.isPresent()); + + node = client.getNode(service, nodeData -> nodeData.getNodeId() == 2); + Assert.assertFalse(node.isPresent()); + + client.stop(); + } +} diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpShardedServiceFinderFactory.java b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpShardedServiceFinderFactory.java index c5f7c340..b4805de1 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpShardedServiceFinderFactory.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpShardedServiceFinderFactory.java @@ -31,7 +31,7 @@ import lombok.val; @Getter -public class HttpShardedServiceFinderFactory > implements ServiceFinderFactory> { +public class HttpShardedServiceFinderFactory > implements ServiceFinderFactory> { private final HttpClientConfig clientConfig; private final ObjectMapper mapper; diff --git a/ranger-zk-client/pom.xml b/ranger-zk-client/pom.xml new file mode 100644 index 00000000..768fbb7e --- /dev/null +++ b/ranger-zk-client/pom.xml @@ -0,0 +1,38 @@ + + + + ranger + com.flipkart.ranger + 1.0-SNAPSHOT + + 4.0.0 + + ranger-zk-client + + + + com.flipkart.ranger + ranger-zookeeper + ${project.version} + + + com.flipkart.ranger + ranger-client + ${project.version} + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + + + org.apache.curator + curator-test + ${curator.version} + test + + + + \ No newline at end of file diff --git a/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/AbstractRangerZKHubClient.java b/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/AbstractRangerZKHubClient.java new file mode 100644 index 00000000..f3e4a96a --- /dev/null +++ b/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/AbstractRangerZKHubClient.java @@ -0,0 +1,95 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.client.zk; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.flipkart.ranger.client.AbstractRangerHubClient; +import com.flipkart.ranger.core.finderhub.ServiceDataSource; +import com.flipkart.ranger.core.finderhub.ServiceFinderHub; +import com.flipkart.ranger.core.finderhub.StaticDataSource; +import com.flipkart.ranger.core.model.Criteria; +import com.flipkart.ranger.core.model.Service; +import com.flipkart.ranger.core.model.ServiceRegistry; +import com.flipkart.ranger.zookeeper.serde.ZkNodeDataDeserializer; +import com.flipkart.ranger.zookeeper.servicefinderhub.ZkServiceDataSource; +import com.flipkart.ranger.zookeeper.servicefinderhub.ZkServiceFinderHubBuilder; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.retry.RetryForever; + +import java.util.List; + +@Slf4j +@Getter +public abstract class AbstractRangerZKHubClient, R extends ServiceRegistry, D extends ZkNodeDataDeserializer> extends AbstractRangerHubClient { + + private final boolean disablePushUpdaters; + private final String connectionString; + private final CuratorFramework curatorFramework; + /* + Use this if you don't want the datasource to fetch the entire list of services; + but know beforehand about the services you are going to build the hub for. + */ + private final List services; + + public AbstractRangerZKHubClient( + String namespace, + ObjectMapper mapper, + int nodeRefreshIntervalMs, + boolean disablePushUpdaters, + String connectionString, + CuratorFramework curatorFramework, + C criteria, + D deserializer, + List services + ) { + super(namespace, mapper, nodeRefreshIntervalMs, criteria, deserializer); + this.disablePushUpdaters = disablePushUpdaters; + this.connectionString = connectionString; + this.services = services; + + this.curatorFramework = null != curatorFramework ? curatorFramework : + CuratorFrameworkFactory.builder() + .namespace(namespace) + .connectString(this.connectionString) + .retryPolicy(new RetryForever(5000)) + .build(); + + } + + @Override + protected ServiceFinderHub buildHub() { + return new ZkServiceFinderHubBuilder() + .withCuratorFramework(curatorFramework) + .withConnectionString(connectionString) + .withNamespace(getNamespace()) + .withRefreshFrequencyMs(getNodeRefreshTimeMs()) + .withServiceDataSource(buildServiceDataSource()) + .withServiceFinderFactory(buildFinderFactory()) + .build(); + } + + @Override + protected ServiceDataSource buildServiceDataSource() { + return null != services && !services.isEmpty() ? + new StaticDataSource(services) : + new ZkServiceDataSource(getNamespace(), connectionString, curatorFramework); + } + +} + diff --git a/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/ShardedRangerZKHubClient.java b/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/ShardedRangerZKHubClient.java new file mode 100644 index 00000000..3f9236ac --- /dev/null +++ b/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/ShardedRangerZKHubClient.java @@ -0,0 +1,74 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.client.zk; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.flipkart.ranger.core.finder.nodeselector.RoundRobinServiceNodeSelector; +import com.flipkart.ranger.core.finder.serviceregistry.MapBasedServiceRegistry; +import com.flipkart.ranger.core.finder.shardselector.MatchingShardSelector; +import com.flipkart.ranger.core.finderhub.ServiceFinderFactory; +import com.flipkart.ranger.core.model.Criteria; +import com.flipkart.ranger.core.model.Service; +import com.flipkart.ranger.zookeeper.serde.ZkNodeDataDeserializer; +import com.flipkart.ranger.zookeeper.servicefinderhub.ZkShardedServiceFinderFactory; +import lombok.Builder; +import lombok.extern.slf4j.Slf4j; +import org.apache.curator.framework.CuratorFramework; + +import java.util.List; + +@Slf4j +public class ShardedRangerZKHubClient> extends AbstractRangerZKHubClient, ZkNodeDataDeserializer> { + + @Builder + public ShardedRangerZKHubClient( + String namespace, + ObjectMapper mapper, + int refreshTimeMs, + boolean disablePushUpdaters, + String connectionString, + CuratorFramework curatorFramework, + C criteria, + ZkNodeDataDeserializer deserializer, + List services + ) { + super( + namespace, + mapper, + refreshTimeMs, + disablePushUpdaters, + connectionString, + curatorFramework, + criteria, + deserializer, + services + ); + } + + @Override + protected ServiceFinderFactory> buildFinderFactory() { + return ZkShardedServiceFinderFactory.builder() + .curatorFramework(getCuratorFramework()) + .connectionString(getConnectionString()) + .nodeRefreshIntervalMs(getNodeRefreshTimeMs()) + .disablePushUpdaters(isDisablePushUpdaters()) + .deserializer(getDeserializer()) + .shardSelector(new MatchingShardSelector<>()) + .nodeSelector(new RoundRobinServiceNodeSelector<>()) + .build(); + } +} + diff --git a/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/SimpleRangerZKClient.java b/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/SimpleRangerZKClient.java new file mode 100644 index 00000000..58cf61a3 --- /dev/null +++ b/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/SimpleRangerZKClient.java @@ -0,0 +1,136 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.client.zk; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.flipkart.ranger.client.RangerClientConstants; +import com.flipkart.ranger.client.RangerClient; +import com.flipkart.ranger.core.finder.SimpleShardedServiceFinder; +import com.flipkart.ranger.core.finder.shardselector.MatchingShardSelector; +import com.flipkart.ranger.core.model.Criteria; +import com.flipkart.ranger.core.model.ServiceNode; +import com.flipkart.ranger.zookeeper.ServiceFinderBuilders; +import com.flipkart.ranger.zookeeper.serde.ZkNodeDataDeserializer; +import com.google.common.base.Preconditions; +import lombok.Builder; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.retry.RetryForever; + +import java.util.List; +import java.util.Optional; + +@Slf4j +@Getter +public class SimpleRangerZKClient, D extends ZkNodeDataDeserializer> implements RangerClient { + + private final SimpleShardedServiceFinder serviceFinder; + private final C criteria; + private final D deserializer; + + @Builder(builderMethodName = "fromConnectionString", builderClassName = "FromConnectionStringBuilder") + public SimpleRangerZKClient( + String namespace, + String serviceName, + ObjectMapper mapper, + int nodeRefreshIntervalMs, + boolean disableWatchers, + String connectionString, + C criteria, + D deserializer + ){ + this( + namespace, + serviceName, + mapper, + nodeRefreshIntervalMs, + disableWatchers, + CuratorFrameworkFactory.newClient(connectionString, new RetryForever(RangerClientConstants.CONNECTION_RETRY_TIME)), + criteria, + deserializer + ); + } + + @Builder(builderMethodName = "fromCurator", builderClassName = "FromCuratorBuilder") + public SimpleRangerZKClient( + String namespace, + String serviceName, + ObjectMapper mapper, + int nodeRefreshIntervalMs, + boolean disableWatchers, + CuratorFramework curatorFramework, + C criteria, + D deserializer + ){ + Preconditions.checkNotNull(mapper, "Mapper can't be null"); + Preconditions.checkNotNull(namespace, "namespace can't be null"); + Preconditions.checkNotNull(deserializer, "deserializer can't be null"); + + int effectiveRefreshTime = nodeRefreshIntervalMs; + if (effectiveRefreshTime < RangerClientConstants.MINIMUM_REFRESH_TIME) { + effectiveRefreshTime = RangerClientConstants.MINIMUM_REFRESH_TIME; + log.warn("Node info update interval too low: {} ms. Has been upgraded to {} ms ", + nodeRefreshIntervalMs, + RangerClientConstants.MINIMUM_REFRESH_TIME); + } + + this.criteria = criteria; + this.deserializer = deserializer; + this.serviceFinder = ServiceFinderBuilders.shardedFinderBuilder() + .withCuratorFramework(curatorFramework) + .withNamespace(namespace) + .withServiceName(serviceName) + .withDeserializer(deserializer) + .withNodeRefreshIntervalMs(effectiveRefreshTime) + .withDisableWatchers(disableWatchers) + .withShardSelector(new MatchingShardSelector<>()) + .build(); + } + + @Override + public void start() { + log.info("Starting the service finder"); + this.serviceFinder.start(); + } + + @Override + public void stop() { + log.info("Stopping the service finder"); + this.serviceFinder.stop(); + } + + @Override + public Optional> getNode() { + return getNode(criteria); + } + + @Override + public Optional>> getAllNodes() { + return getAllNodes(criteria); + } + + @Override + public Optional> getNode(C criteria) { + return Optional.ofNullable(this.serviceFinder.get(criteria)); + } + + @Override + public Optional>> getAllNodes(C criteria) { + return Optional.ofNullable(this.serviceFinder.getAll(criteria)); + } +} diff --git a/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/UnshardedRangerZKHubClient.java b/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/UnshardedRangerZKHubClient.java new file mode 100644 index 00000000..187fa909 --- /dev/null +++ b/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/UnshardedRangerZKHubClient.java @@ -0,0 +1,74 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.client.zk; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.flipkart.ranger.core.finder.nodeselector.RoundRobinServiceNodeSelector; +import com.flipkart.ranger.core.finder.serviceregistry.ListBasedServiceRegistry; +import com.flipkart.ranger.core.finder.shardselector.ListShardSelector; +import com.flipkart.ranger.core.finderhub.ServiceFinderFactory; +import com.flipkart.ranger.core.model.Criteria; +import com.flipkart.ranger.core.model.Service; +import com.flipkart.ranger.zookeeper.serde.ZkNodeDataDeserializer; +import com.flipkart.ranger.zookeeper.servicefinderhub.ZKUnshardedServiceFinderFactory; +import lombok.Builder; +import lombok.extern.slf4j.Slf4j; +import org.apache.curator.framework.CuratorFramework; + +import java.util.List; + +@Slf4j +public class UnshardedRangerZKHubClient> extends AbstractRangerZKHubClient, ZkNodeDataDeserializer> { + + @Builder + public UnshardedRangerZKHubClient( + String namespace, + ObjectMapper mapper, + int refreshTimeMs, + boolean disablePushUpdaters, + String connectionString, + CuratorFramework curatorFramework, + C criteria, + ZkNodeDataDeserializer deserializer, + List services + ) { + super( + namespace, + mapper, + refreshTimeMs, + disablePushUpdaters, + connectionString, + curatorFramework, + criteria, + deserializer, + services + ); + } + + @Override + protected ServiceFinderFactory> buildFinderFactory() { + return ZKUnshardedServiceFinderFactory.builder() + .curatorFramework(getCuratorFramework()) + .connectionString(getConnectionString()) + .nodeRefreshIntervalMs(getNodeRefreshTimeMs()) + .disablePushUpdaters(isDisablePushUpdaters()) + .deserializer(getDeserializer()) + .shardSelector(new ListShardSelector<>()) + .nodeSelector(new RoundRobinServiceNodeSelector<>()) + .build(); + } + +} diff --git a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/BaseRangerZKClientTest.java b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/BaseRangerZKClientTest.java new file mode 100644 index 00000000..40b74f08 --- /dev/null +++ b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/BaseRangerZKClientTest.java @@ -0,0 +1,106 @@ +package com.flipkart.ranger.client.zk; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.flipkart.ranger.core.healthcheck.HealthcheckResult; +import com.flipkart.ranger.core.healthcheck.HealthcheckStatus; +import com.flipkart.ranger.core.model.ServiceNode; +import com.flipkart.ranger.core.serviceprovider.ServiceProvider; +import com.flipkart.ranger.core.signals.ExternalTriggeredSignal; +import com.flipkart.ranger.core.units.TestNodeData; +import com.flipkart.ranger.core.util.Exceptions; +import com.flipkart.ranger.zookeeper.ServiceProviderBuilders; +import com.flipkart.ranger.zookeeper.serde.ZkNodeDataSerializer; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.retry.ExponentialBackoffRetry; +import org.apache.curator.test.TestingCluster; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.util.Collections; +import java.util.Date; + +@Slf4j +@Getter +public abstract class BaseRangerZKClientTest { + + private TestingCluster testingCluster; + private ObjectMapper objectMapper = new ObjectMapper(); + private CuratorFramework curatorFramework; + private ServiceProvider> provider; + + @Before + public void startTestCluster() throws Exception { + objectMapper = new ObjectMapper(); + testingCluster = new TestingCluster(3); + testingCluster.start(); + curatorFramework = CuratorFrameworkFactory.builder() + .namespace("test-n") + .connectString(testingCluster.getConnectString()) + .retryPolicy(new ExponentialBackoffRetry(1000, 100)) + .build(); + curatorFramework.start(); + curatorFramework.blockUntilConnected(); + initilizeProvider(); + log.debug("Started zk subsystem"); + } + + @After + public void stopTestCluster() throws Exception { + log.debug("Stopping zk subsystem"); + curatorFramework.close(); + if (null != testingCluster) { + testingCluster.close(); + } + if(null != provider){ + provider.stop(); + } + } + + protected ServiceNode read(final byte[] data) { + try { + return getObjectMapper().readValue(data, new TypeReference>() {}); + } + catch (IOException e) { + Exceptions.illegalState(e); + } + return null; + } + + protected byte[] write(final ServiceNode node) { + try { + return getObjectMapper().writeValueAsBytes(node); + } + catch (IOException e) { + Exceptions.illegalState(e); + } + return null; + } + + protected void initilizeProvider(){ + ExternalTriggeredSignal refreshProviderSignal = new ExternalTriggeredSignal<>( + () -> HealthcheckResult.builder() + .status(HealthcheckStatus.healthy) + .updatedTime(new Date().getTime()) + .build(), Collections.emptyList()); + provider = ServiceProviderBuilders.shardedServiceProviderBuilder() + .withHostname("localhost") + .withPort(1080) + .withNamespace("test-n") + .withServiceName("s1") + .withSerializer(this::write) + .withNodeData(new TestNodeData(1)) + .withHealthcheck(() -> HealthcheckStatus.healthy) + .withadditionalRefreshSignal(refreshProviderSignal) + .withCuratorFramework(getCuratorFramework()) + .build(); + provider.start(); + refreshProviderSignal.trigger(); + } + + +} diff --git a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/ShardedZKRangerClientTest.java b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/ShardedZKRangerClientTest.java new file mode 100644 index 00000000..a67fda0c --- /dev/null +++ b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/ShardedZKRangerClientTest.java @@ -0,0 +1,41 @@ +package com.flipkart.ranger.client.zk; + +import com.flipkart.ranger.core.TestUtils; +import com.flipkart.ranger.core.model.Criteria; +import com.flipkart.ranger.core.model.Service; +import com.flipkart.ranger.core.model.ServiceNode; +import com.flipkart.ranger.core.units.TestNodeData; +import lombok.extern.slf4j.Slf4j; +import lombok.val; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Optional; + +@Slf4j +public class ShardedZKRangerClientTest extends BaseRangerZKClientTest { + + @Test + public void testShardedHub(){ + val zkHubClient = ShardedRangerZKHubClient.>builder() + .namespace("test-n") + .connectionString(getTestingCluster().getConnectString()) + .curatorFramework(getCuratorFramework()) + .disablePushUpdaters(true) + .mapper(getObjectMapper()) + .deserializer(this::read) + .build(); + + zkHubClient.start(); + TestUtils.sleepForSeconds(6); + + val service = new Service("test-n", "s1"); + Optional> node = zkHubClient.getNode(new Service("test-n", "s1")); + Assert.assertTrue(node.isPresent()); + + node = zkHubClient.getNode(service, nodeData -> nodeData.getNodeId() == 1); + Assert.assertTrue(node.isPresent()); + + zkHubClient.stop(); + } +} diff --git a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/SimpleRangerZKClientTest.java b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/SimpleRangerZKClientTest.java new file mode 100644 index 00000000..f2fa998e --- /dev/null +++ b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/SimpleRangerZKClientTest.java @@ -0,0 +1,36 @@ +package com.flipkart.ranger.client.zk; + +import com.flipkart.ranger.core.model.Criteria; +import com.flipkart.ranger.core.model.ServiceNode; +import com.flipkart.ranger.core.units.TestNodeData; +import com.flipkart.ranger.zookeeper.serde.ZkNodeDataDeserializer; +import lombok.val; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Optional; + +public class SimpleRangerZKClientTest extends BaseRangerZKClientTest { + + @Test + public void testBaseClient(){ + val client = SimpleRangerZKClient., ZkNodeDataDeserializer>fromCurator() + .curatorFramework(getCuratorFramework()) + .deserializer(this::read) + .namespace("test-n") + .serviceName("s1") + .disableWatchers(true) + .mapper(getObjectMapper()) + .build(); + client.start(); + + Optional> node = client.getNode(); + Assert.assertTrue(node.isPresent()); + + node = client.getNode(c -> c.getNodeId() == 1); + Assert.assertTrue(node.isPresent()); + + node = client.getNode(c -> c.getNodeId() == 2); + Assert.assertFalse(node.isPresent()); + } +} diff --git a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/UnshardedZKRangerClientTest.java b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/UnshardedZKRangerClientTest.java new file mode 100644 index 00000000..5dfe5e56 --- /dev/null +++ b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/UnshardedZKRangerClientTest.java @@ -0,0 +1,41 @@ +package com.flipkart.ranger.client.zk; + +import com.flipkart.ranger.core.TestUtils; +import com.flipkart.ranger.core.model.Criteria; +import com.flipkart.ranger.core.model.Service; +import com.flipkart.ranger.core.model.ServiceNode; +import com.flipkart.ranger.core.units.TestNodeData; +import lombok.extern.slf4j.Slf4j; +import lombok.val; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Optional; + +@Slf4j +public class UnshardedZKRangerClientTest extends BaseRangerZKClientTest { + + @Test + public void testShardedHub(){ + val zkHubClient =UnshardedRangerZKHubClient.>builder() + .namespace("test-n") + .connectionString(getTestingCluster().getConnectString()) + .curatorFramework(getCuratorFramework()) + .disablePushUpdaters(true) + .mapper(getObjectMapper()) + .deserializer(this::read) + .build(); + + zkHubClient.start(); + TestUtils.sleepForSeconds(6); + + val service = new Service("test-n", "s1"); + Optional> node = zkHubClient.getNode(new Service("test-n", "s1")); + Assert.assertTrue(node.isPresent()); + + node = zkHubClient.getNode(service, nodeData -> nodeData.getNodeId() == 1); + Assert.assertTrue(node.isPresent()); + + zkHubClient.stop(); + } +} From 6a402979ec5b47fbcd3fd431f13ab3e1dac9ffbb Mon Sep 17 00:00:00 2001 From: Koushik R Date: Mon, 18 Oct 2021 19:49:17 +0530 Subject: [PATCH 05/69] a. Added a ranger-server module with zk as the backend. b. Optimized imports --- pom.xml | 1 + .../client/AbstractRangerHubClient.java | 5 + .../ranger/client/RangerHubClient.java | 3 + .../http/ShardedRangerHttpHubClient.java | 3 +- .../http/UnshardedRangerHttpHubClient.java | 3 +- ranger-server/pom.xml | 42 ++++++++ .../java/com/flipkart/ranger/server/App.java | 49 +++++++++ .../ranger/server/AppConfiguration.java | 38 +++++++ .../server/healthcheck/RangerHealthCheck.java | 25 +++++ .../server/manager/RangerClientManager.java | 99 +++++++++++++++++++ .../ranger/server/model/GenericResponse.java | 30 ++++++ .../ranger/server/model/ShardInfo.java | 54 ++++++++++ .../server/resources/RangerResource.java | 76 ++++++++++++++ .../ranger/server/rotation/BirTask.java | 25 +++++ .../ranger/server/rotation/OorTask.java | 26 +++++ .../server/rotation/RotationStatus.java | 23 +++++ .../client/zk/AbstractRangerZKHubClient.java | 3 +- .../client/zk/ShardedRangerZKHubClient.java | 3 +- .../client/zk/UnshardedRangerZKHubClient.java | 3 +- 19 files changed, 506 insertions(+), 5 deletions(-) create mode 100644 ranger-server/pom.xml create mode 100644 ranger-server/src/main/java/com/flipkart/ranger/server/App.java create mode 100644 ranger-server/src/main/java/com/flipkart/ranger/server/AppConfiguration.java create mode 100644 ranger-server/src/main/java/com/flipkart/ranger/server/healthcheck/RangerHealthCheck.java create mode 100644 ranger-server/src/main/java/com/flipkart/ranger/server/manager/RangerClientManager.java create mode 100644 ranger-server/src/main/java/com/flipkart/ranger/server/model/GenericResponse.java create mode 100644 ranger-server/src/main/java/com/flipkart/ranger/server/model/ShardInfo.java create mode 100644 ranger-server/src/main/java/com/flipkart/ranger/server/resources/RangerResource.java create mode 100644 ranger-server/src/main/java/com/flipkart/ranger/server/rotation/BirTask.java create mode 100644 ranger-server/src/main/java/com/flipkart/ranger/server/rotation/OorTask.java create mode 100644 ranger-server/src/main/java/com/flipkart/ranger/server/rotation/RotationStatus.java diff --git a/pom.xml b/pom.xml index 5e8c5b9c..7cc0da22 100644 --- a/pom.xml +++ b/pom.xml @@ -15,6 +15,7 @@ ranger-client ranger-zk-client ranger-http-client + ranger-server diff --git a/ranger-client/src/main/java/com/flipkart/ranger/client/AbstractRangerHubClient.java b/ranger-client/src/main/java/com/flipkart/ranger/client/AbstractRangerHubClient.java index 8bb5e42a..0f610873 100644 --- a/ranger-client/src/main/java/com/flipkart/ranger/client/AbstractRangerHubClient.java +++ b/ranger-client/src/main/java/com/flipkart/ranger/client/AbstractRangerHubClient.java @@ -25,6 +25,7 @@ import lombok.extern.slf4j.Slf4j; import lombok.val; +import java.util.Collection; import java.util.List; import java.util.Optional; @@ -101,6 +102,10 @@ public Optional>> getAllNodes( return optionalFinder.map(trcServiceFinder -> trcServiceFinder.getAll(criteria)); } + public Collection getServices() throws Exception { + return this.getHub().getServiceDataSource().services(); + } + protected abstract ServiceFinderHub buildHub(); protected abstract ServiceDataSource buildServiceDataSource(); diff --git a/ranger-client/src/main/java/com/flipkart/ranger/client/RangerHubClient.java b/ranger-client/src/main/java/com/flipkart/ranger/client/RangerHubClient.java index f9a2534f..f95b0f9c 100644 --- a/ranger-client/src/main/java/com/flipkart/ranger/client/RangerHubClient.java +++ b/ranger-client/src/main/java/com/flipkart/ranger/client/RangerHubClient.java @@ -19,6 +19,7 @@ import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.core.model.ServiceNode; +import java.util.Collection; import java.util.List; import java.util.Optional; @@ -28,6 +29,8 @@ public interface RangerHubClient> { void stop(); + Collection getServices() throws Exception; + Optional> getNode(final Service service); Optional> getNode(final Service service, final C criteria); diff --git a/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/ShardedRangerHttpHubClient.java b/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/ShardedRangerHttpHubClient.java index 1cf4d081..6f999046 100644 --- a/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/ShardedRangerHttpHubClient.java +++ b/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/ShardedRangerHttpHubClient.java @@ -37,7 +37,8 @@ import java.util.List; @Slf4j -public class ShardedRangerHttpHubClient> extends AbstractRangerHubClient, HTTPResponseDataDeserializer> { +public class ShardedRangerHttpHubClient> + extends AbstractRangerHubClient, HTTPResponseDataDeserializer> { private final List services; private final HttpClientConfig clientConfig; diff --git a/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/UnshardedRangerHttpHubClient.java b/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/UnshardedRangerHttpHubClient.java index 7765f3a3..8897830f 100644 --- a/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/UnshardedRangerHttpHubClient.java +++ b/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/UnshardedRangerHttpHubClient.java @@ -37,7 +37,8 @@ import java.util.List; @Slf4j -public class UnshardedRangerHttpHubClient> extends AbstractRangerHubClient, HTTPResponseDataDeserializer> { +public class UnshardedRangerHttpHubClient> + extends AbstractRangerHubClient, HTTPResponseDataDeserializer> { private final List services; private final HttpClientConfig clientConfig; diff --git a/ranger-server/pom.xml b/ranger-server/pom.xml new file mode 100644 index 00000000..7f09fd9a --- /dev/null +++ b/ranger-server/pom.xml @@ -0,0 +1,42 @@ + + + + ranger + com.flipkart.ranger + 1.0-SNAPSHOT + + 4.0.0 + + ranger-server + + + 2.0.23 + + + + + com.flipkart.ranger + ranger-zk-client + ${project.version} + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + + + io.dropwizard + dropwizard-core + ${dropwizard.version} + + + org.mockito + mockito-core + 3.6.0 + test + + + + \ No newline at end of file diff --git a/ranger-server/src/main/java/com/flipkart/ranger/server/App.java b/ranger-server/src/main/java/com/flipkart/ranger/server/App.java new file mode 100644 index 00000000..5d662148 --- /dev/null +++ b/ranger-server/src/main/java/com/flipkart/ranger/server/App.java @@ -0,0 +1,49 @@ +package com.flipkart.ranger.server; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.flipkart.ranger.client.RangerClientConstants; +import com.flipkart.ranger.client.RangerHubClient; +import com.flipkart.ranger.client.zk.UnshardedRangerZKHubClient; +import com.flipkart.ranger.core.model.Criteria; +import com.flipkart.ranger.core.model.ServiceNode; +import com.flipkart.ranger.server.healthcheck.RangerHealthCheck; +import com.flipkart.ranger.server.manager.RangerClientManager; +import com.flipkart.ranger.server.resources.RangerResource; +import com.flipkart.ranger.server.rotation.BirTask; +import com.flipkart.ranger.server.rotation.OorTask; +import com.flipkart.ranger.server.rotation.RotationStatus; +import io.dropwizard.Application; +import io.dropwizard.setup.Bootstrap; +import io.dropwizard.setup.Environment; +import lombok.extern.slf4j.Slf4j; +import lombok.val; + +@Slf4j +public class App extends Application { + + private RotationStatus rotationStatus; + + public static void main(String[] args) throws Exception { + new App().run(args); + } + + @Override + public void initialize(Bootstrap bootstrap) { + + } + + @Override + public void run(AppConfiguration appConfiguration, Environment environment) { + rotationStatus = new RotationStatus(appConfiguration.isInitialRotationStatus()); + val rangerClientManager = new RangerClientManager(appConfiguration, environment.getObjectMapper()); + environment.lifecycle().manage(rangerClientManager); + environment.jersey().register(new RangerResource(rangerClientManager)); + environment.admin() + .addTask(new OorTask(rotationStatus)); + environment.admin() + .addTask(new BirTask(rotationStatus)); + environment.healthChecks().register( + "ranger-healthcheck", new RangerHealthCheck(rangerClientManager.getCuratorFramework())); + } +} diff --git a/ranger-server/src/main/java/com/flipkart/ranger/server/AppConfiguration.java b/ranger-server/src/main/java/com/flipkart/ranger/server/AppConfiguration.java new file mode 100644 index 00000000..b32c71d5 --- /dev/null +++ b/ranger-server/src/main/java/com/flipkart/ranger/server/AppConfiguration.java @@ -0,0 +1,38 @@ +package com.flipkart.ranger.server; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.flipkart.ranger.client.RangerClientConstants; +import com.flipkart.ranger.core.model.Service; +import io.dropwizard.Configuration; +import lombok.*; + +import javax.validation.constraints.Min; +import javax.validation.constraints.NotEmpty; +import javax.validation.constraints.NotNull; +import java.util.List; + +@Getter +@Setter +@AllArgsConstructor +@NoArgsConstructor +@JsonIgnoreProperties(ignoreUnknown = true) +public class AppConfiguration extends Configuration { + + @NotEmpty + @NotNull + private String zookeeper; + + @NotEmpty + @NotNull + private String namespace; + + @Min(1000) + private int refreshTimeMs = RangerClientConstants.MINIMUM_REFRESH_TIME; + + private boolean disablePushUpdaters = true; + + // Specify these if you don't want to use the data source! + private List services; + + private boolean initialRotationStatus; +} diff --git a/ranger-server/src/main/java/com/flipkart/ranger/server/healthcheck/RangerHealthCheck.java b/ranger-server/src/main/java/com/flipkart/ranger/server/healthcheck/RangerHealthCheck.java new file mode 100644 index 00000000..000c6252 --- /dev/null +++ b/ranger-server/src/main/java/com/flipkart/ranger/server/healthcheck/RangerHealthCheck.java @@ -0,0 +1,25 @@ +package com.flipkart.ranger.server.healthcheck; + +import com.codahale.metrics.health.HealthCheck; +import lombok.extern.slf4j.Slf4j; +import org.apache.curator.framework.CuratorFramework; + +import javax.inject.Singleton; + +@Singleton +@Slf4j +public class RangerHealthCheck extends HealthCheck { + + private final CuratorFramework curatorFramework; + + public RangerHealthCheck(CuratorFramework curatorFramework){ + this.curatorFramework = curatorFramework; + } + + + @Override + protected Result check() { + return curatorFramework.getZookeeperClient().isConnected() ? + Result.healthy("Service is healthy") : Result.unhealthy("Can't connect to zookeeper"); + } +} diff --git a/ranger-server/src/main/java/com/flipkart/ranger/server/manager/RangerClientManager.java b/ranger-server/src/main/java/com/flipkart/ranger/server/manager/RangerClientManager.java new file mode 100644 index 00000000..e54eca2f --- /dev/null +++ b/ranger-server/src/main/java/com/flipkart/ranger/server/manager/RangerClientManager.java @@ -0,0 +1,99 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.server.manager; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.flipkart.ranger.client.RangerClientConstants; +import com.flipkart.ranger.client.zk.UnshardedRangerZKHubClient; +import com.flipkart.ranger.core.model.Criteria; +import com.flipkart.ranger.core.model.ServiceNode; +import com.flipkart.ranger.server.AppConfiguration; +import com.flipkart.ranger.server.model.ShardInfo; +import com.google.common.base.Preconditions; +import io.dropwizard.lifecycle.Managed; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.retry.RetryForever; + +import javax.inject.Inject; +import javax.inject.Singleton; +import java.io.IOException; + +@Singleton +@Getter +@Slf4j +public class RangerClientManager implements Managed { + + private final AppConfiguration appConfiguration; + private final CuratorFramework curatorFramework; + private final UnshardedRangerZKHubClient> hubClient; + + @Inject + public RangerClientManager(AppConfiguration appConfiguration, ObjectMapper mapper){ + Preconditions.checkNotNull(appConfiguration, "Ranger configuration can't be null"); + Preconditions.checkNotNull(appConfiguration.getZookeeper(), "Zookeeper can't be null"); + Preconditions.checkNotNull(appConfiguration.getNamespace(), "Namespace can't be null"); + Preconditions.checkNotNull(mapper, "Mapper can't be null"); + + this.appConfiguration = appConfiguration; + this.curatorFramework = CuratorFrameworkFactory.newClient( + appConfiguration.getZookeeper(), + new RetryForever(RangerClientConstants.CONNECTION_RETRY_TIME) + ); + this.hubClient = UnshardedRangerZKHubClient.>builder() + .connectionString(appConfiguration.getZookeeper()) + .curatorFramework(curatorFramework) + .disablePushUpdaters(appConfiguration.isDisablePushUpdaters()) + .mapper(mapper) + .services(appConfiguration.getServices()) + .refreshTimeMs(appConfiguration.getRefreshTimeMs()) + .deserializer(data -> { + try { + mapper.readValue(data, new TypeReference>() { + }); + } catch (IOException e) { + log.warn("Error parsing node data"); + } + return null; + }) + .build(); + } + + @Override + public void start() { + log.info("Starting the ranger client manager"); + curatorFramework.start(); + try { + curatorFramework.blockUntilConnected(); + } + catch (InterruptedException e) { + log.error("Curator block interrupted", e); + } + hubClient.start(); + log.info("Started the ranger client manager"); + } + + @Override + public void stop() { + log.info("Stopping the ranger client manager"); + hubClient.stop(); + curatorFramework.close(); + log.info("Stopped the ranger client manager"); + } +} diff --git a/ranger-server/src/main/java/com/flipkart/ranger/server/model/GenericResponse.java b/ranger-server/src/main/java/com/flipkart/ranger/server/model/GenericResponse.java new file mode 100644 index 00000000..58ef067a --- /dev/null +++ b/ranger-server/src/main/java/com/flipkart/ranger/server/model/GenericResponse.java @@ -0,0 +1,30 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.server.model; + +import lombok.*; + +@Builder +@Getter +@Setter +@AllArgsConstructor +@NoArgsConstructor +public class GenericResponse { + + private boolean success; + private String error; + private T data; +} diff --git a/ranger-server/src/main/java/com/flipkart/ranger/server/model/ShardInfo.java b/ranger-server/src/main/java/com/flipkart/ranger/server/model/ShardInfo.java new file mode 100644 index 00000000..df7e22f6 --- /dev/null +++ b/ranger-server/src/main/java/com/flipkart/ranger/server/model/ShardInfo.java @@ -0,0 +1,54 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.server.model; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import lombok.*; +import lombok.extern.slf4j.Slf4j; + +import java.util.Objects; + +@AllArgsConstructor +@NoArgsConstructor +@Getter +@Setter +@Builder +@JsonIgnoreProperties(ignoreUnknown = true) +public class ShardInfo { + private String environment; + private String region; + + @Override + public boolean equals(Object o) { + if (o == this) + return true; + + if (!(o instanceof ShardInfo)) + return false; + + ShardInfo other = (ShardInfo) o; + return Objects.equals(this.getEnvironment(), other.getEnvironment()) + && Objects.equals(this.getRegion(), other.getRegion()); + } + + @Override + public int hashCode() { + int hash = 5; + hash = hash + (this.environment != null ? this.environment.hashCode() : 0); + hash = hash + (this.region != null ? this.region.hashCode() : 0); + return hash; + } +} diff --git a/ranger-server/src/main/java/com/flipkart/ranger/server/resources/RangerResource.java b/ranger-server/src/main/java/com/flipkart/ranger/server/resources/RangerResource.java new file mode 100644 index 00000000..5647c796 --- /dev/null +++ b/ranger-server/src/main/java/com/flipkart/ranger/server/resources/RangerResource.java @@ -0,0 +1,76 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.server.resources; + +import com.codahale.metrics.annotation.Metered; +import com.flipkart.ranger.core.model.Criteria; +import com.flipkart.ranger.core.model.Service; +import com.flipkart.ranger.core.model.ServiceNode; +import com.flipkart.ranger.server.manager.RangerClientManager; +import com.flipkart.ranger.server.model.GenericResponse; +import com.flipkart.ranger.server.model.ShardInfo; +import lombok.extern.slf4j.Slf4j; +import lombok.val; + +import javax.inject.Inject; +import javax.validation.constraints.NotEmpty; +import javax.validation.constraints.NotNull; +import javax.ws.rs.*; +import javax.ws.rs.core.MediaType; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +@Slf4j +@Consumes(MediaType.APPLICATION_JSON) +@Produces(MediaType.APPLICATION_JSON) +@Path("/ranger") +public class RangerResource { + + private final RangerClientManager clientManager; + + @Inject + public RangerResource(RangerClientManager clientManager){ + this.clientManager = clientManager; + } + + @GET + @Path("/services/v1") + @Metered + public GenericResponse> getServices() { + return GenericResponse.>builder() + .success(true) + .data(clientManager.getHubClient().getServices()) + .build(); + } + + @GET + @Path("/nodes/v1/{namespace}/{serviceName}") + @Metered + public GenericResponse>> getNodes( + @NotNull @NotEmpty @PathParam("namespace") final String namespace, + @NotNull @NotEmpty @PathParam("serviceName") final String serviceName + ){ + val service = new Service(namespace, serviceName); + Optional>> nodeList = clientManager.getHubClient().getAllNodes( + service, null); + return GenericResponse.>>builder() + .success(true) + .data(nodeList.orElse(Collections.emptyList())) + .build(); + } +} diff --git a/ranger-server/src/main/java/com/flipkart/ranger/server/rotation/BirTask.java b/ranger-server/src/main/java/com/flipkart/ranger/server/rotation/BirTask.java new file mode 100644 index 00000000..05ffb6cb --- /dev/null +++ b/ranger-server/src/main/java/com/flipkart/ranger/server/rotation/BirTask.java @@ -0,0 +1,25 @@ +package com.flipkart.ranger.server.rotation; + +import io.dropwizard.servlets.tasks.Task; +import lombok.extern.slf4j.Slf4j; + +import java.io.PrintWriter; +import java.util.List; +import java.util.Map; + +@Slf4j +public class BirTask extends Task { + + private RotationStatus rotationStatus; + + public BirTask(RotationStatus rotationStatus) { + super("ranger-bir"); + this.rotationStatus = rotationStatus; + } + + @Override + public void execute(Map> map, PrintWriter printWriter) { + rotationStatus.bir(); + log.info("Taking node back into rotation on ranger"); + } +} diff --git a/ranger-server/src/main/java/com/flipkart/ranger/server/rotation/OorTask.java b/ranger-server/src/main/java/com/flipkart/ranger/server/rotation/OorTask.java new file mode 100644 index 00000000..c7496e33 --- /dev/null +++ b/ranger-server/src/main/java/com/flipkart/ranger/server/rotation/OorTask.java @@ -0,0 +1,26 @@ +package com.flipkart.ranger.server.rotation; + + +import io.dropwizard.servlets.tasks.Task; +import lombok.extern.slf4j.Slf4j; + +import java.io.PrintWriter; +import java.util.List; +import java.util.Map; + +@Slf4j +public class OorTask extends Task { + + private RotationStatus rotationStatus; + + public OorTask(RotationStatus rotationStatus) { + super("ranger-oor"); + this.rotationStatus = rotationStatus; + } + + @Override + public void execute(Map> map, PrintWriter printWriter) { + rotationStatus.oor(); + log.info("Taking node out of rotation on ranger"); + } +} diff --git a/ranger-server/src/main/java/com/flipkart/ranger/server/rotation/RotationStatus.java b/ranger-server/src/main/java/com/flipkart/ranger/server/rotation/RotationStatus.java new file mode 100644 index 00000000..1a32e4cf --- /dev/null +++ b/ranger-server/src/main/java/com/flipkart/ranger/server/rotation/RotationStatus.java @@ -0,0 +1,23 @@ +package com.flipkart.ranger.server.rotation; + +import java.util.concurrent.atomic.AtomicBoolean; + +public class RotationStatus { + private AtomicBoolean rotationStatus; + + public RotationStatus(boolean initialStatus) { + rotationStatus = new AtomicBoolean(initialStatus); + } + + public void oor() { + rotationStatus.set(false); + } + + public void bir() { + rotationStatus.set(true); + } + + public boolean status() { + return rotationStatus.get(); + } +} diff --git a/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/AbstractRangerZKHubClient.java b/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/AbstractRangerZKHubClient.java index f3e4a96a..e10ab1a5 100644 --- a/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/AbstractRangerZKHubClient.java +++ b/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/AbstractRangerZKHubClient.java @@ -36,7 +36,8 @@ @Slf4j @Getter -public abstract class AbstractRangerZKHubClient, R extends ServiceRegistry, D extends ZkNodeDataDeserializer> extends AbstractRangerHubClient { +public abstract class AbstractRangerZKHubClient, R extends ServiceRegistry, D extends ZkNodeDataDeserializer> + extends AbstractRangerHubClient { private final boolean disablePushUpdaters; private final String connectionString; diff --git a/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/ShardedRangerZKHubClient.java b/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/ShardedRangerZKHubClient.java index 3f9236ac..6b0141a8 100644 --- a/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/ShardedRangerZKHubClient.java +++ b/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/ShardedRangerZKHubClient.java @@ -31,7 +31,8 @@ import java.util.List; @Slf4j -public class ShardedRangerZKHubClient> extends AbstractRangerZKHubClient, ZkNodeDataDeserializer> { +public class ShardedRangerZKHubClient> + extends AbstractRangerZKHubClient, ZkNodeDataDeserializer> { @Builder public ShardedRangerZKHubClient( diff --git a/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/UnshardedRangerZKHubClient.java b/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/UnshardedRangerZKHubClient.java index 187fa909..d5ce3722 100644 --- a/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/UnshardedRangerZKHubClient.java +++ b/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/UnshardedRangerZKHubClient.java @@ -31,7 +31,8 @@ import java.util.List; @Slf4j -public class UnshardedRangerZKHubClient> extends AbstractRangerZKHubClient, ZkNodeDataDeserializer> { +public class UnshardedRangerZKHubClient> + extends AbstractRangerZKHubClient, ZkNodeDataDeserializer> { @Builder public UnshardedRangerZKHubClient( From a449ee60387b535966b8962ad90e1892796fd392 Mon Sep 17 00:00:00 2001 From: Koushik R Date: Mon, 18 Oct 2021 20:12:17 +0530 Subject: [PATCH 06/69] a. Added a local.yml for the server to test out the resources --- .../ranger/core/util/FinderUtils.java | 2 ++ ranger-server/config/local.yml | 34 +++++++++++++++++++ .../ranger/server/AppConfiguration.java | 4 +++ .../server/manager/RangerClientManager.java | 3 +- 4 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 ranger-server/config/local.yml diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/util/FinderUtils.java b/ranger-core/src/main/java/com/flipkart/ranger/core/util/FinderUtils.java index 54fa9545..d0d7be50 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/util/FinderUtils.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/util/FinderUtils.java @@ -45,6 +45,8 @@ public static boolean isValidNode( Service service, long healthcheckZombieCheckThresholdTime, ServiceNode serviceNode) { + if(null == serviceNode) return false; + if(HealthcheckStatus.healthy != serviceNode.getHealthcheckStatus()) { log.debug("Unhealthy node [{}:{}] found for [{}]", serviceNode.getHost(), serviceNode.getPort(), service.getServiceName()); diff --git a/ranger-server/config/local.yml b/ranger-server/config/local.yml new file mode 100644 index 00000000..ff20fde8 --- /dev/null +++ b/ranger-server/config/local.yml @@ -0,0 +1,34 @@ +name: ranger-server +zookeeper: localhost:2181 +namespace: test +disablePushUpdaters: true +initialRotationStatus: false + +server: + maxThreads: 128 + minThreads: 128 + applicationConnectors: + - type: http + port: 8080 + adminConnectors: + - type: http + port: 8081 + applicationContextPath: / + requestLog: + appenders: + - type: console + timeZone: IST + +logging: + level: INFO + + loggers: + com.flipkart.ranger.server: DEBUG + + appenders: + - type: console + threshold: INFO + timeZone: IST + logFormat: "%(%-5level) [%date] [%thread] [%logger{0}]: %message%n" + + diff --git a/ranger-server/src/main/java/com/flipkart/ranger/server/AppConfiguration.java b/ranger-server/src/main/java/com/flipkart/ranger/server/AppConfiguration.java index b32c71d5..8b896fb6 100644 --- a/ranger-server/src/main/java/com/flipkart/ranger/server/AppConfiguration.java +++ b/ranger-server/src/main/java/com/flipkart/ranger/server/AppConfiguration.java @@ -18,6 +18,10 @@ @JsonIgnoreProperties(ignoreUnknown = true) public class AppConfiguration extends Configuration { + @NotEmpty + @NotNull + private String name; + @NotEmpty @NotNull private String zookeeper; diff --git a/ranger-server/src/main/java/com/flipkart/ranger/server/manager/RangerClientManager.java b/ranger-server/src/main/java/com/flipkart/ranger/server/manager/RangerClientManager.java index e54eca2f..ceb2b48c 100644 --- a/ranger-server/src/main/java/com/flipkart/ranger/server/manager/RangerClientManager.java +++ b/ranger-server/src/main/java/com/flipkart/ranger/server/manager/RangerClientManager.java @@ -57,6 +57,7 @@ public RangerClientManager(AppConfiguration appConfiguration, ObjectMapper mappe new RetryForever(RangerClientConstants.CONNECTION_RETRY_TIME) ); this.hubClient = UnshardedRangerZKHubClient.>builder() + .namespace(appConfiguration.getNamespace()) .connectionString(appConfiguration.getZookeeper()) .curatorFramework(curatorFramework) .disablePushUpdaters(appConfiguration.isDisablePushUpdaters()) @@ -68,7 +69,7 @@ public RangerClientManager(AppConfiguration appConfiguration, ObjectMapper mappe mapper.readValue(data, new TypeReference>() { }); } catch (IOException e) { - log.warn("Error parsing node data"); + log.warn("Error parsing node data with value {}", new String(data)); } return null; }) From a16376699a14bf1f9f51fccd79dece70d3470136 Mon Sep 17 00:00:00 2001 From: Koushik R Date: Mon, 18 Oct 2021 22:30:52 +0530 Subject: [PATCH 07/69] Split the tests in the ranger-client into stubs so that the testjar can be used across for various modules --- pom.xml | 1 + .../flipkart/ranger/client/RangerClient.java | 1 - .../client/AbstractRangerHubClientTest.java | 136 +----------------- .../ranger/client/stubs/RangerTestHub.java | 45 ++++++ .../ranger/client/stubs/TestCriteria.java | 9 ++ .../ranger/client/stubs/TestDeserializer.java | 6 + .../stubs/TestServiceFinderFactory.java | 29 ++++ .../ranger/client/stubs/TestShardInfo.java | 12 ++ .../TestSimpleUnshardedServiceFinder.java | 55 +++++++ .../client/utils/RangerHubTestUtils.java | 19 +++ .../http/model/ServiceNodesResponse.java | 1 - .../servicefinder/HttpNodeDataSource.java | 2 - .../HttpNodeDataStoreConnectorTest.java | 1 - ...HttpShardedServiceProviderBuilderTest.java | 2 - ranger-server-bundle/pom.xml | 48 +++++++ .../java/com/flipkart/ranger/server/App.java | 9 +- .../ranger/server/AppConfiguration.java | 5 +- .../ranger/server/model/ShardInfo.java | 1 - .../server/resources/RangerResource.java | 1 - .../client/zk/SimpleRangerZKClient.java | 2 +- .../servicefinder/ZkNodeDataSource.java | 2 + 21 files changed, 238 insertions(+), 149 deletions(-) create mode 100644 ranger-client/src/test/java/com/flipkart/ranger/client/stubs/RangerTestHub.java create mode 100644 ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestCriteria.java create mode 100644 ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestDeserializer.java create mode 100644 ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestServiceFinderFactory.java create mode 100644 ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestShardInfo.java create mode 100644 ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestSimpleUnshardedServiceFinder.java create mode 100644 ranger-client/src/test/java/com/flipkart/ranger/client/utils/RangerHubTestUtils.java create mode 100644 ranger-server-bundle/pom.xml diff --git a/pom.xml b/pom.xml index 7cc0da22..6165389f 100644 --- a/pom.xml +++ b/pom.xml @@ -16,6 +16,7 @@ ranger-zk-client ranger-http-client ranger-server + ranger-server-bundle diff --git a/ranger-client/src/main/java/com/flipkart/ranger/client/RangerClient.java b/ranger-client/src/main/java/com/flipkart/ranger/client/RangerClient.java index 57738a69..ac2d99e3 100644 --- a/ranger-client/src/main/java/com/flipkart/ranger/client/RangerClient.java +++ b/ranger-client/src/main/java/com/flipkart/ranger/client/RangerClient.java @@ -17,7 +17,6 @@ package com.flipkart.ranger.client; import com.flipkart.ranger.core.model.Criteria; -import com.flipkart.ranger.core.model.Deserializer; import com.flipkart.ranger.core.model.ServiceNode; import java.util.List; diff --git a/ranger-client/src/test/java/com/flipkart/ranger/client/AbstractRangerHubClientTest.java b/ranger-client/src/test/java/com/flipkart/ranger/client/AbstractRangerHubClientTest.java index 6324974a..a0453524 100644 --- a/ranger-client/src/test/java/com/flipkart/ranger/client/AbstractRangerHubClientTest.java +++ b/ranger-client/src/test/java/com/flipkart/ranger/client/AbstractRangerHubClientTest.java @@ -16,22 +16,16 @@ package com.flipkart.ranger.client; import com.fasterxml.jackson.databind.ObjectMapper; +import com.flipkart.ranger.client.stubs.RangerTestHub; +import com.flipkart.ranger.client.stubs.TestShardInfo; +import com.flipkart.ranger.client.utils.RangerHubTestUtils; import com.flipkart.ranger.core.TestUtils; -import com.flipkart.ranger.core.finder.ServiceFinder; -import com.flipkart.ranger.core.finder.SimpleUnshardedServiceFinder; -import com.flipkart.ranger.core.finder.SimpleUnshardedServiceFinderBuilder; -import com.flipkart.ranger.core.finder.serviceregistry.ListBasedServiceRegistry; -import com.flipkart.ranger.core.finderhub.*; -import com.flipkart.ranger.core.model.*; -import com.google.common.collect.Lists; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; +import com.flipkart.ranger.core.model.Service; +import com.flipkart.ranger.core.model.ServiceNode; import lombok.extern.slf4j.Slf4j; import org.junit.Assert; import org.junit.Test; -import java.util.List; import java.util.Optional; @Slf4j @@ -40,129 +34,11 @@ public class AbstractRangerHubClientTest { private static final ObjectMapper mapper = new ObjectMapper(); private static final Service service = new Service("test-ns", "test-s"); - @Data - @Builder - @AllArgsConstructor - private static class TestShardInfo{ - private int shardId; - } - - private static class TestCriteria implements Criteria{ - @Override - public boolean apply(TestShardInfo nodeData) { - return nodeData.getShardId() == 1; - } - } - - - @Builder - private static class TestSimpleUnshardedServiceFinder> - extends SimpleUnshardedServiceFinderBuilder, Deserializer, Criteria> { - - @Override - public SimpleUnshardedServiceFinder> build() { - return buildFinder(); - } - - @Override - protected NodeDataSource> dataSource(Service service) { - return new NodeDataSource>() { - @Override - public Optional>> refresh(Deserializer deserializer) { - return Optional.of( - Lists.newArrayList( - new ServiceNode<>("localhost", 9200, TestShardInfo.builder().shardId(1).build()) - ) - ); - } - - @Override - public void start() { - - } - - @Override - public void ensureConnected() { - - } - - @Override - public void stop() { - - } - - @Override - public boolean isActive() { - return true; - } - }; - } - } - - private static class TestServiceFinderFactory implements ServiceFinderFactory, ListBasedServiceRegistry>{ - - @Override - public ServiceFinder, ListBasedServiceRegistry> buildFinder(Service service) { - SimpleUnshardedServiceFinder> finder = new TestSimpleUnshardedServiceFinder>() - .withNamespace(service.getNamespace()) - .withServiceName(service.getServiceName()) - .withDeserializer(new Deserializer() { - @Override - public int hashCode() { - return super.hashCode(); - } - }) - .build(); - finder.start(); - return finder; - } - } - - private static class TestDeserilizer implements Deserializer{ - - } - - private static class TestAbstractRangerHub extends AbstractRangerHubClient, ListBasedServiceRegistry, TestDeserilizer> { - - public TestAbstractRangerHub(String namespace, ObjectMapper mapper, int nodeRefreshTimeMs, Criteria criteria, TestDeserilizer deserilizer) { - super(namespace, mapper, nodeRefreshTimeMs, criteria, deserilizer); - } - - @Override - protected ServiceFinderHub, ListBasedServiceRegistry> buildHub() { - return new ServiceFinderHubBuilder, ListBasedServiceRegistry>() { - @Override - protected void preBuild() { - - } - - @Override - protected void postBuild(ServiceFinderHub, ListBasedServiceRegistry> serviceFinderHub) { - - } - }.withServiceDataSource(buildServiceDataSource()) - .withServiceFinderFactory(buildFinderFactory()) - .build(); - } - - @Override - protected ServiceDataSource buildServiceDataSource() { - return new StaticDataSource(Lists.newArrayList(service)); - } - - @Override - protected ServiceFinderFactory, ListBasedServiceRegistry> buildFinderFactory() { - return new TestServiceFinderFactory(); - } - } - @Test public void testAbstractHubClient() { - TestAbstractRangerHub testAbstractHub = new TestAbstractRangerHub(service.getNamespace(), mapper, 1000, new TestCriteria(), new TestDeserilizer<>()); + RangerTestHub testAbstractHub = RangerHubTestUtils.getTestHub(); testAbstractHub.start(); - TestUtils.sleepForSeconds(3); - Optional> node = testAbstractHub.getNode(service); Assert.assertTrue(node.isPresent()); Assert.assertTrue(node.get().getHost().equalsIgnoreCase("localhost")); diff --git a/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/RangerTestHub.java b/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/RangerTestHub.java new file mode 100644 index 00000000..e7e62e88 --- /dev/null +++ b/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/RangerTestHub.java @@ -0,0 +1,45 @@ +package com.flipkart.ranger.client.stubs; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.flipkart.ranger.client.AbstractRangerHubClient; +import com.flipkart.ranger.client.utils.RangerHubTestUtils; +import com.flipkart.ranger.core.finder.serviceregistry.ListBasedServiceRegistry; +import com.flipkart.ranger.core.finderhub.*; +import com.flipkart.ranger.core.model.Criteria; +import com.google.common.collect.Lists; + +public class RangerTestHub extends AbstractRangerHubClient, ListBasedServiceRegistry, TestDeserializer> { + + public RangerTestHub(String namespace, ObjectMapper mapper, int nodeRefreshTimeMs, Criteria criteria, TestDeserializer deserilizer) { + super(namespace, mapper, nodeRefreshTimeMs, criteria, deserilizer); + } + + @Override + protected ServiceFinderHub, ListBasedServiceRegistry> buildHub() { + return new ServiceFinderHubBuilder, ListBasedServiceRegistry>() { + @Override + protected void preBuild() { + + } + + @Override + protected void postBuild(ServiceFinderHub, ListBasedServiceRegistry> serviceFinderHub) { + + } + }.withServiceDataSource(buildServiceDataSource()) + .withServiceFinderFactory(buildFinderFactory()) + .build(); + } + + @Override + protected ServiceDataSource buildServiceDataSource() { + return new StaticDataSource(Lists.newArrayList(RangerHubTestUtils.service)); + } + + @Override + protected ServiceFinderFactory, ListBasedServiceRegistry> buildFinderFactory() { + return new TestServiceFinderFactory(); + } +} + + diff --git a/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestCriteria.java b/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestCriteria.java new file mode 100644 index 00000000..4071b655 --- /dev/null +++ b/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestCriteria.java @@ -0,0 +1,9 @@ +package com.flipkart.ranger.client.stubs; + +import com.flipkart.ranger.core.model.Criteria; + +public class TestCriteria implements Criteria { + public boolean apply(TestShardInfo nodeData) { + return nodeData.getShardId() == 1; + } +} diff --git a/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestDeserializer.java b/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestDeserializer.java new file mode 100644 index 00000000..72d6bc98 --- /dev/null +++ b/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestDeserializer.java @@ -0,0 +1,6 @@ +package com.flipkart.ranger.client.stubs; + +import com.flipkart.ranger.core.model.Deserializer; + +public class TestDeserializer implements Deserializer { +} diff --git a/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestServiceFinderFactory.java b/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestServiceFinderFactory.java new file mode 100644 index 00000000..bbbe8b9a --- /dev/null +++ b/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestServiceFinderFactory.java @@ -0,0 +1,29 @@ +package com.flipkart.ranger.client.stubs; + +import com.flipkart.ranger.core.finder.ServiceFinder; +import com.flipkart.ranger.core.finder.SimpleUnshardedServiceFinder; +import com.flipkart.ranger.core.finder.serviceregistry.ListBasedServiceRegistry; +import com.flipkart.ranger.core.finderhub.ServiceFinderFactory; +import com.flipkart.ranger.core.model.Criteria; +import com.flipkart.ranger.core.model.Deserializer; +import com.flipkart.ranger.core.model.Service; + +public class TestServiceFinderFactory implements ServiceFinderFactory, ListBasedServiceRegistry> { + + @Override + public ServiceFinder, ListBasedServiceRegistry> buildFinder(Service service) { + SimpleUnshardedServiceFinder> finder = new TestSimpleUnshardedServiceFinder>() + .withNamespace(service.getNamespace()) + .withServiceName(service.getServiceName()) + .withDeserializer(new Deserializer() { + @Override + public int hashCode() { + return super.hashCode(); + } + }) + .build(); + finder.start(); + return finder; + } +} + diff --git a/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestShardInfo.java b/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestShardInfo.java new file mode 100644 index 00000000..9f26e566 --- /dev/null +++ b/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestShardInfo.java @@ -0,0 +1,12 @@ +package com.flipkart.ranger.client.stubs; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; + +@Data +@Builder +@AllArgsConstructor +public class TestShardInfo { + private int shardId; +} diff --git a/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestSimpleUnshardedServiceFinder.java b/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestSimpleUnshardedServiceFinder.java new file mode 100644 index 00000000..570975ad --- /dev/null +++ b/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestSimpleUnshardedServiceFinder.java @@ -0,0 +1,55 @@ +package com.flipkart.ranger.client.stubs; + +import com.flipkart.ranger.core.finder.SimpleUnshardedServiceFinder; +import com.flipkart.ranger.core.finder.SimpleUnshardedServiceFinderBuilder; +import com.flipkart.ranger.core.model.*; +import com.google.common.collect.Lists; +import lombok.Builder; + +import java.util.List; +import java.util.Optional; + +@Builder +public class TestSimpleUnshardedServiceFinder > + extends SimpleUnshardedServiceFinderBuilder, Deserializer, Criteria> { + + @Override + public SimpleUnshardedServiceFinder> build() { + return buildFinder(); + } + + @Override + protected NodeDataSource> dataSource(Service service) { + return new NodeDataSource>() { + @Override + public Optional>> refresh(Deserializer deserializer) { + return Optional.of( + Lists.newArrayList( + new ServiceNode<>("localhost", 9200, TestShardInfo.builder().shardId(1).build()) + ) + ); + } + + @Override + public void start() { + + } + + @Override + public void ensureConnected() { + + } + + @Override + public void stop() { + + } + + @Override + public boolean isActive() { + return true; + } + }; + } +} + diff --git a/ranger-client/src/test/java/com/flipkart/ranger/client/utils/RangerHubTestUtils.java b/ranger-client/src/test/java/com/flipkart/ranger/client/utils/RangerHubTestUtils.java new file mode 100644 index 00000000..76ada370 --- /dev/null +++ b/ranger-client/src/test/java/com/flipkart/ranger/client/utils/RangerHubTestUtils.java @@ -0,0 +1,19 @@ +package com.flipkart.ranger.client.utils; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.flipkart.ranger.client.stubs.RangerTestHub; +import com.flipkart.ranger.client.stubs.TestCriteria; +import com.flipkart.ranger.client.stubs.TestDeserializer; +import com.flipkart.ranger.core.model.Service; + +public class RangerHubTestUtils { + + public static final Service service = new Service("test-ns", "test-s"); + private static final ObjectMapper mapper = new ObjectMapper(); + + private RangerHubTestUtils(){} + + public static RangerTestHub getTestHub(){ + return new RangerTestHub(service.getNamespace(), mapper, 1000, new TestCriteria(), new TestDeserializer<>()); + } +} diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceNodesResponse.java b/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceNodesResponse.java index ef55a3a1..5404217c 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceNodesResponse.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceNodesResponse.java @@ -16,7 +16,6 @@ package com.flipkart.ranger.http.model; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; import com.flipkart.ranger.core.model.ServiceNode; import lombok.*; diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpNodeDataSource.java b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpNodeDataSource.java index 66b1450b..bef6aaa9 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpNodeDataSource.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpNodeDataSource.java @@ -23,7 +23,6 @@ import com.flipkart.ranger.core.util.FinderUtils; import com.flipkart.ranger.http.common.HttpNodeDataStoreConnector; import com.flipkart.ranger.http.config.HttpClientConfig; -import com.flipkart.ranger.http.model.ServiceNodesResponse; import com.flipkart.ranger.http.serde.HTTPResponseDataDeserializer; import com.google.common.base.Preconditions; import lombok.extern.slf4j.Slf4j; @@ -33,7 +32,6 @@ import okhttp3.ResponseBody; import java.io.IOException; -import java.util.Collections; import java.util.List; import java.util.Optional; diff --git a/ranger-http/src/test/java/com/flipkart/ranger/http/common/HttpNodeDataStoreConnectorTest.java b/ranger-http/src/test/java/com/flipkart/ranger/http/common/HttpNodeDataStoreConnectorTest.java index 71e2916e..7a24e70b 100644 --- a/ranger-http/src/test/java/com/flipkart/ranger/http/common/HttpNodeDataStoreConnectorTest.java +++ b/ranger-http/src/test/java/com/flipkart/ranger/http/common/HttpNodeDataStoreConnectorTest.java @@ -16,7 +16,6 @@ package com.flipkart.ranger.http.common; import com.fasterxml.jackson.databind.ObjectMapper; -import com.flipkart.ranger.core.units.TestNodeData; import com.flipkart.ranger.http.config.HttpClientConfig; import org.junit.Assert; import org.junit.Test; diff --git a/ranger-http/src/test/java/com/flipkart/ranger/http/serviceprovider/HttpShardedServiceProviderBuilderTest.java b/ranger-http/src/test/java/com/flipkart/ranger/http/serviceprovider/HttpShardedServiceProviderBuilderTest.java index f83832b6..347cc6fd 100644 --- a/ranger-http/src/test/java/com/flipkart/ranger/http/serviceprovider/HttpShardedServiceProviderBuilderTest.java +++ b/ranger-http/src/test/java/com/flipkart/ranger/http/serviceprovider/HttpShardedServiceProviderBuilderTest.java @@ -19,10 +19,8 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.flipkart.ranger.core.healthcheck.Healthchecks; import com.flipkart.ranger.core.model.ServiceNode; -import com.flipkart.ranger.core.serviceprovider.ServiceProvider; import com.flipkart.ranger.http.config.HttpClientConfig; import com.flipkart.ranger.http.model.ServiceRegistrationResponse; -import com.flipkart.ranger.http.serde.HttpRequestDataSerializer; import com.github.tomakehurst.wiremock.junit.WireMockRule; import lombok.Builder; import lombok.Data; diff --git a/ranger-server-bundle/pom.xml b/ranger-server-bundle/pom.xml new file mode 100644 index 00000000..caebca1c --- /dev/null +++ b/ranger-server-bundle/pom.xml @@ -0,0 +1,48 @@ + + + + ranger + com.flipkart.ranger + 1.0-SNAPSHOT + + 4.0.0 + + ranger-server-bundle + + + 2.0.23 + + + + + com.flipkart.ranger + ranger-zk-client + ${project.version} + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + + + io.dropwizard + dropwizard-core + ${dropwizard.version} + + + org.mockito + mockito-core + 3.6.0 + test + + + com.flipkart.ranger + ranger-zk-client + ${project.version} + test-jar + + + + \ No newline at end of file diff --git a/ranger-server/src/main/java/com/flipkart/ranger/server/App.java b/ranger-server/src/main/java/com/flipkart/ranger/server/App.java index 5d662148..1f594014 100644 --- a/ranger-server/src/main/java/com/flipkart/ranger/server/App.java +++ b/ranger-server/src/main/java/com/flipkart/ranger/server/App.java @@ -1,12 +1,5 @@ package com.flipkart.ranger.server; -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.flipkart.ranger.client.RangerClientConstants; -import com.flipkart.ranger.client.RangerHubClient; -import com.flipkart.ranger.client.zk.UnshardedRangerZKHubClient; -import com.flipkart.ranger.core.model.Criteria; -import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.server.healthcheck.RangerHealthCheck; import com.flipkart.ranger.server.manager.RangerClientManager; import com.flipkart.ranger.server.resources.RangerResource; @@ -42,7 +35,7 @@ public void run(AppConfiguration appConfiguration, Environment environment) { environment.admin() .addTask(new OorTask(rotationStatus)); environment.admin() - .addTask(new BirTask(rotationStatus)); + .addTask(new BirTask(rotationStatus)); environment.healthChecks().register( "ranger-healthcheck", new RangerHealthCheck(rangerClientManager.getCuratorFramework())); } diff --git a/ranger-server/src/main/java/com/flipkart/ranger/server/AppConfiguration.java b/ranger-server/src/main/java/com/flipkart/ranger/server/AppConfiguration.java index 8b896fb6..200730c9 100644 --- a/ranger-server/src/main/java/com/flipkart/ranger/server/AppConfiguration.java +++ b/ranger-server/src/main/java/com/flipkart/ranger/server/AppConfiguration.java @@ -4,7 +4,10 @@ import com.flipkart.ranger.client.RangerClientConstants; import com.flipkart.ranger.core.model.Service; import io.dropwizard.Configuration; -import lombok.*; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; import javax.validation.constraints.Min; import javax.validation.constraints.NotEmpty; diff --git a/ranger-server/src/main/java/com/flipkart/ranger/server/model/ShardInfo.java b/ranger-server/src/main/java/com/flipkart/ranger/server/model/ShardInfo.java index df7e22f6..74a1c0af 100644 --- a/ranger-server/src/main/java/com/flipkart/ranger/server/model/ShardInfo.java +++ b/ranger-server/src/main/java/com/flipkart/ranger/server/model/ShardInfo.java @@ -17,7 +17,6 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import lombok.*; -import lombok.extern.slf4j.Slf4j; import java.util.Objects; diff --git a/ranger-server/src/main/java/com/flipkart/ranger/server/resources/RangerResource.java b/ranger-server/src/main/java/com/flipkart/ranger/server/resources/RangerResource.java index 5647c796..ec559fbb 100644 --- a/ranger-server/src/main/java/com/flipkart/ranger/server/resources/RangerResource.java +++ b/ranger-server/src/main/java/com/flipkart/ranger/server/resources/RangerResource.java @@ -16,7 +16,6 @@ package com.flipkart.ranger.server.resources; import com.codahale.metrics.annotation.Metered; -import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.server.manager.RangerClientManager; diff --git a/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/SimpleRangerZKClient.java b/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/SimpleRangerZKClient.java index 58cf61a3..bf38c933 100644 --- a/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/SimpleRangerZKClient.java +++ b/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/SimpleRangerZKClient.java @@ -16,8 +16,8 @@ package com.flipkart.ranger.client.zk; import com.fasterxml.jackson.databind.ObjectMapper; -import com.flipkart.ranger.client.RangerClientConstants; import com.flipkart.ranger.client.RangerClient; +import com.flipkart.ranger.client.RangerClientConstants; import com.flipkart.ranger.core.finder.SimpleShardedServiceFinder; import com.flipkart.ranger.core.finder.shardselector.MatchingShardSelector; import com.flipkart.ranger.core.model.Criteria; diff --git a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinder/ZkNodeDataSource.java b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinder/ZkNodeDataSource.java index d2258dfc..b71780f7 100644 --- a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinder/ZkNodeDataSource.java +++ b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinder/ZkNodeDataSource.java @@ -18,6 +18,7 @@ import com.flipkart.ranger.core.model.NodeDataSource; import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.core.model.ServiceNode; +import com.flipkart.ranger.core.util.Exceptions; import com.flipkart.ranger.core.util.FinderUtils; import com.flipkart.ranger.zookeeper.common.ZkNodeDataStoreConnector; import com.flipkart.ranger.zookeeper.serde.ZkNodeDataDeserializer; @@ -87,6 +88,7 @@ private Optional>> checkForUpdateOnZookeeper(D deserializer) } catch (Exception e) { log.error("Error getting service data from zookeeper: ", e); + Exceptions.illegalState(e); } return Optional.empty(); } From a95814cf8df3b7c76db8ff9221ce11bc4b23e575 Mon Sep 17 00:00:00 2001 From: Koushik R Date: Tue, 19 Oct 2021 00:04:05 +0530 Subject: [PATCH 08/69] a) Separated ranger-server from ranger-server-bundle and have written working server with a NodeData in ranger-server --- .../client/AbstractRangerHubClientTest.java | 2 - ranger-server-bundle/pom.xml | 4 +- .../server/bundle/RangerServerBundle.java | 78 ++++++++++++++ .../server/bundle}/model/GenericResponse.java | 2 +- .../bundle}/resources/RangerResource.java | 26 ++--- .../server/bundle/rotation/BirTask.java | 40 +++++++ .../server/bundle/rotation/OorTask.java | 40 +++++++ .../bundle/rotation/RotationStatus.java | 38 +++++++ .../server/bundle/RangerServerBundleTest.java | 99 +++++++++++++++++ ranger-server/config/local.yml | 9 +- ranger-server/pom.xml | 5 + .../java/com/flipkart/ranger/server/App.java | 60 ++++++++--- .../ranger/server/AppConfiguration.java | 38 +++---- .../server/config/RangerConfiguration.java | 30 ++++++ .../server/healthcheck/RangerHealthCheck.java | 15 +++ .../server/manager/RangerBundleManager.java | 67 ++++++++++++ .../server/manager/RangerClientManager.java | 100 ------------------ .../ranger/server/rotation/BirTask.java | 25 ----- .../ranger/server/rotation/OorTask.java | 26 ----- .../server/rotation/RotationStatus.java | 23 ---- .../ranger/server/util/RangerServerUtils.java | 64 +++++++++++ .../client/zk/AbstractRangerZKHubClient.java | 10 +- 22 files changed, 564 insertions(+), 237 deletions(-) create mode 100644 ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/RangerServerBundle.java rename {ranger-server/src/main/java/com/flipkart/ranger/server => ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle}/model/GenericResponse.java (94%) rename {ranger-server/src/main/java/com/flipkart/ranger/server => ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle}/resources/RangerResource.java (70%) create mode 100644 ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/rotation/BirTask.java create mode 100644 ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/rotation/OorTask.java create mode 100644 ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/rotation/RotationStatus.java create mode 100644 ranger-server-bundle/src/test/java/com/flipkart/ranger/server/bundle/RangerServerBundleTest.java create mode 100644 ranger-server/src/main/java/com/flipkart/ranger/server/config/RangerConfiguration.java create mode 100644 ranger-server/src/main/java/com/flipkart/ranger/server/manager/RangerBundleManager.java delete mode 100644 ranger-server/src/main/java/com/flipkart/ranger/server/manager/RangerClientManager.java delete mode 100644 ranger-server/src/main/java/com/flipkart/ranger/server/rotation/BirTask.java delete mode 100644 ranger-server/src/main/java/com/flipkart/ranger/server/rotation/OorTask.java delete mode 100644 ranger-server/src/main/java/com/flipkart/ranger/server/rotation/RotationStatus.java create mode 100644 ranger-server/src/main/java/com/flipkart/ranger/server/util/RangerServerUtils.java diff --git a/ranger-client/src/test/java/com/flipkart/ranger/client/AbstractRangerHubClientTest.java b/ranger-client/src/test/java/com/flipkart/ranger/client/AbstractRangerHubClientTest.java index a0453524..94ea0ee4 100644 --- a/ranger-client/src/test/java/com/flipkart/ranger/client/AbstractRangerHubClientTest.java +++ b/ranger-client/src/test/java/com/flipkart/ranger/client/AbstractRangerHubClientTest.java @@ -15,7 +15,6 @@ */ package com.flipkart.ranger.client; -import com.fasterxml.jackson.databind.ObjectMapper; import com.flipkart.ranger.client.stubs.RangerTestHub; import com.flipkart.ranger.client.stubs.TestShardInfo; import com.flipkart.ranger.client.utils.RangerHubTestUtils; @@ -31,7 +30,6 @@ @Slf4j public class AbstractRangerHubClientTest { - private static final ObjectMapper mapper = new ObjectMapper(); private static final Service service = new Service("test-ns", "test-s"); @Test diff --git a/ranger-server-bundle/pom.xml b/ranger-server-bundle/pom.xml index caebca1c..e4788a0f 100644 --- a/ranger-server-bundle/pom.xml +++ b/ranger-server-bundle/pom.xml @@ -18,7 +18,7 @@ com.flipkart.ranger - ranger-zk-client + ranger-client ${project.version} @@ -39,7 +39,7 @@ com.flipkart.ranger - ranger-zk-client + ranger-client ${project.version} test-jar diff --git a/ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/RangerServerBundle.java b/ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/RangerServerBundle.java new file mode 100644 index 00000000..e9ceefb6 --- /dev/null +++ b/ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/RangerServerBundle.java @@ -0,0 +1,78 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.server.bundle; + +import com.flipkart.ranger.client.RangerHubClient; +import com.flipkart.ranger.core.model.Criteria; +import com.flipkart.ranger.core.model.Deserializer; +import com.flipkart.ranger.server.bundle.resources.RangerResource; +import com.flipkart.ranger.server.bundle.rotation.BirTask; +import com.flipkart.ranger.server.bundle.rotation.OorTask; +import com.flipkart.ranger.server.bundle.rotation.RotationStatus; +import com.google.common.annotations.VisibleForTesting; +import io.dropwizard.Configuration; +import io.dropwizard.ConfiguredBundle; +import io.dropwizard.setup.Bootstrap; +import io.dropwizard.setup.Environment; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public abstract class RangerServerBundle< + T, + C extends Criteria, + D extends Deserializer, + U extends Configuration + > implements ConfiguredBundle{ + + @Getter + private RangerHubClient rangerHub; + @Getter + @VisibleForTesting + private RotationStatus rotationStatus; + + protected abstract RangerHubClient withRangerHub(U configuration); + + protected abstract boolean withInitialRotationStatus(U configuration); + + @Override + public void initialize(Bootstrap bootstrap) { + + } + + @Override + public void run(U configuration, Environment environment) { + rangerHub = withRangerHub(configuration); + rotationStatus = new RotationStatus(withInitialRotationStatus(configuration)); + + environment.admin() + .addTask(new OorTask(rotationStatus)); + environment.admin() + .addTask(new BirTask(rotationStatus)); + + environment.jersey().register(new RangerResource<>(rangerHub)); + } + + public void start(){ + log.info("Starting the ranger hub"); + rangerHub.start(); + } + + public void stop(){ + log.info("Stopping the ranger hub"); + rangerHub.stop(); + } +} diff --git a/ranger-server/src/main/java/com/flipkart/ranger/server/model/GenericResponse.java b/ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/model/GenericResponse.java similarity index 94% rename from ranger-server/src/main/java/com/flipkart/ranger/server/model/GenericResponse.java rename to ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/model/GenericResponse.java index 58ef067a..e680cf34 100644 --- a/ranger-server/src/main/java/com/flipkart/ranger/server/model/GenericResponse.java +++ b/ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/model/GenericResponse.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.flipkart.ranger.server.model; +package com.flipkart.ranger.server.bundle.model; import lombok.*; diff --git a/ranger-server/src/main/java/com/flipkart/ranger/server/resources/RangerResource.java b/ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/resources/RangerResource.java similarity index 70% rename from ranger-server/src/main/java/com/flipkart/ranger/server/resources/RangerResource.java rename to ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/resources/RangerResource.java index ec559fbb..d753056a 100644 --- a/ranger-server/src/main/java/com/flipkart/ranger/server/resources/RangerResource.java +++ b/ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/resources/RangerResource.java @@ -13,14 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.flipkart.ranger.server.resources; +package com.flipkart.ranger.server.bundle.resources; import com.codahale.metrics.annotation.Metered; +import com.flipkart.ranger.client.RangerHubClient; +import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.core.model.ServiceNode; -import com.flipkart.ranger.server.manager.RangerClientManager; -import com.flipkart.ranger.server.model.GenericResponse; -import com.flipkart.ranger.server.model.ShardInfo; +import com.flipkart.ranger.server.bundle.model.GenericResponse; import lombok.extern.slf4j.Slf4j; import lombok.val; @@ -38,36 +38,36 @@ @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) @Path("/ranger") -public class RangerResource { +public class RangerResource> { - private final RangerClientManager clientManager; + private final RangerHubClient rangerHub; @Inject - public RangerResource(RangerClientManager clientManager){ - this.clientManager = clientManager; + public RangerResource(RangerHubClient rangerHub){ + this.rangerHub = rangerHub; } @GET @Path("/services/v1") @Metered - public GenericResponse> getServices() { + public GenericResponse> getServices() throws Exception { return GenericResponse.>builder() .success(true) - .data(clientManager.getHubClient().getServices()) + .data(rangerHub.getServices()) .build(); } @GET @Path("/nodes/v1/{namespace}/{serviceName}") @Metered - public GenericResponse>> getNodes( + public GenericResponse>> getNodes( @NotNull @NotEmpty @PathParam("namespace") final String namespace, @NotNull @NotEmpty @PathParam("serviceName") final String serviceName ){ val service = new Service(namespace, serviceName); - Optional>> nodeList = clientManager.getHubClient().getAllNodes( + Optional>> nodeList = rangerHub.getAllNodes( service, null); - return GenericResponse.>>builder() + return GenericResponse.>>builder() .success(true) .data(nodeList.orElse(Collections.emptyList())) .build(); diff --git a/ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/rotation/BirTask.java b/ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/rotation/BirTask.java new file mode 100644 index 00000000..78ae414b --- /dev/null +++ b/ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/rotation/BirTask.java @@ -0,0 +1,40 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.server.bundle.rotation; + +import io.dropwizard.servlets.tasks.Task; +import lombok.extern.slf4j.Slf4j; + +import java.io.PrintWriter; +import java.util.List; +import java.util.Map; + +@Slf4j +public class BirTask extends Task { + + private RotationStatus rotationStatus; + + public BirTask(RotationStatus rotationStatus) { + super("ranger-bir"); + this.rotationStatus = rotationStatus; + } + + @Override + public void execute(Map> map, PrintWriter printWriter) { + rotationStatus.bir(); + log.info("Taking node back into rotation on ranger"); + } +} diff --git a/ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/rotation/OorTask.java b/ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/rotation/OorTask.java new file mode 100644 index 00000000..f1076d42 --- /dev/null +++ b/ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/rotation/OorTask.java @@ -0,0 +1,40 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.server.bundle.rotation; + +import io.dropwizard.servlets.tasks.Task; +import lombok.extern.slf4j.Slf4j; + +import java.io.PrintWriter; +import java.util.List; +import java.util.Map; + +@Slf4j +public class OorTask extends Task { + + private RotationStatus rotationStatus; + + public OorTask(RotationStatus rotationStatus) { + super("ranger-oor"); + this.rotationStatus = rotationStatus; + } + + @Override + public void execute(Map> map, PrintWriter printWriter) { + rotationStatus.oor(); + log.info("Taking node out of rotation on ranger"); + } +} diff --git a/ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/rotation/RotationStatus.java b/ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/rotation/RotationStatus.java new file mode 100644 index 00000000..ce37a56d --- /dev/null +++ b/ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/rotation/RotationStatus.java @@ -0,0 +1,38 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.server.bundle.rotation; + +import java.util.concurrent.atomic.AtomicBoolean; + +public class RotationStatus { + private AtomicBoolean rotationStatus; + + public RotationStatus(boolean initialStatus) { + rotationStatus = new AtomicBoolean(initialStatus); + } + + public void oor() { + rotationStatus.set(false); + } + + public void bir() { + rotationStatus.set(true); + } + + public boolean status() { + return rotationStatus.get(); + } +} diff --git a/ranger-server-bundle/src/test/java/com/flipkart/ranger/server/bundle/RangerServerBundleTest.java b/ranger-server-bundle/src/test/java/com/flipkart/ranger/server/bundle/RangerServerBundleTest.java new file mode 100644 index 00000000..568c8f03 --- /dev/null +++ b/ranger-server-bundle/src/test/java/com/flipkart/ranger/server/bundle/RangerServerBundleTest.java @@ -0,0 +1,99 @@ +package com.flipkart.ranger.server.bundle; + +import com.codahale.metrics.MetricRegistry; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.flipkart.ranger.client.RangerHubClient; +import com.flipkart.ranger.client.stubs.TestDeserializer; +import com.flipkart.ranger.client.stubs.TestShardInfo; +import com.flipkart.ranger.client.utils.RangerHubTestUtils; +import com.flipkart.ranger.core.TestUtils; +import com.flipkart.ranger.core.model.Criteria; +import com.flipkart.ranger.core.model.Service; +import com.flipkart.ranger.core.model.ServiceNode; +import io.dropwizard.Configuration; +import io.dropwizard.jersey.DropwizardResourceConfig; +import io.dropwizard.jersey.setup.JerseyEnvironment; +import io.dropwizard.lifecycle.setup.LifecycleEnvironment; +import io.dropwizard.setup.AdminEnvironment; +import io.dropwizard.setup.Bootstrap; +import io.dropwizard.setup.Environment; +import org.eclipse.jetty.util.component.LifeCycle; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.util.Optional; + +import static com.flipkart.ranger.client.utils.RangerHubTestUtils.service; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; + +public class RangerServerBundleTest { + + private final JerseyEnvironment jerseyEnvironment = mock(JerseyEnvironment.class); + private final MetricRegistry metricRegistry = mock(MetricRegistry.class); + private final LifecycleEnvironment lifecycleEnvironment = new LifecycleEnvironment(metricRegistry); + private final Environment environment = mock(Environment.class); + private final Bootstrap bootstrap = mock(Bootstrap.class); + private final Configuration configuration = mock(Configuration.class); + + private final RangerServerBundle, TestDeserializer, Configuration> + rangerServerBundle = new RangerServerBundle, TestDeserializer, Configuration>() { + @Override + protected RangerHubClient> withRangerHub(Configuration configuration) { + return RangerHubTestUtils.getTestHub(); + } + + @Override + protected boolean withInitialRotationStatus(Configuration configuration) { + return false; + } + }; + + @Before + public void setup() throws Exception { + when(jerseyEnvironment.getResourceConfig()).thenReturn(new DropwizardResourceConfig()); + when(environment.jersey()).thenReturn(jerseyEnvironment); + when(environment.lifecycle()).thenReturn(lifecycleEnvironment); + when(environment.getObjectMapper()).thenReturn(new ObjectMapper()); + AdminEnvironment adminEnvironment = mock(AdminEnvironment.class); + doNothing().when(adminEnvironment).addTask(any()); + when(environment.admin()).thenReturn(adminEnvironment); + + rangerServerBundle.initialize(bootstrap); + rangerServerBundle.run(configuration, environment); + rangerServerBundle.start(); + for (LifeCycle lifeCycle : lifecycleEnvironment.getManagedObjects()){ + lifeCycle.start(); + } + } + + + @Test + public void testRangerBundle(){ + TestUtils.sleepForSeconds(3); + Optional> node = rangerServerBundle.getRangerHub().getNode(service); + Assert.assertTrue(node.isPresent()); + Assert.assertTrue(node.get().getHost().equalsIgnoreCase("localhost")); + Assert.assertEquals(9200, node.get().getPort()); + Assert.assertEquals(1, node.get().getNodeData().getShardId()); + + node = rangerServerBundle.getRangerHub().getNode(new Service("test", "test")); + Assert.assertFalse(node.isPresent()); + + node = rangerServerBundle.getRangerHub().getNode(service, nodeData -> nodeData.getShardId() == 2); + Assert.assertFalse(node.isPresent()); + + node = rangerServerBundle.getRangerHub().getNode(new Service("test", "test"), nodeData -> nodeData.getShardId() == 1); + Assert.assertFalse(node.isPresent()); + } + + @After + public void tearDown() throws Exception { + for (LifeCycle lifeCycle: lifecycleEnvironment.getManagedObjects()){ + lifeCycle.stop(); + } + rangerServerBundle.stop(); + } +} diff --git a/ranger-server/config/local.yml b/ranger-server/config/local.yml index ff20fde8..837f06d7 100644 --- a/ranger-server/config/local.yml +++ b/ranger-server/config/local.yml @@ -1,9 +1,12 @@ name: ranger-server -zookeeper: localhost:2181 -namespace: test -disablePushUpdaters: true initialRotationStatus: false +rangerConfiguration: + namespace: test + zookeeper: localhost:2181 + disablePushUpdaters: true + + server: maxThreads: 128 minThreads: 128 diff --git a/ranger-server/pom.xml b/ranger-server/pom.xml index 7f09fd9a..daeef095 100644 --- a/ranger-server/pom.xml +++ b/ranger-server/pom.xml @@ -21,6 +21,11 @@ ranger-zk-client ${project.version} + + com.flipkart.ranger + ranger-server-bundle + ${project.version} + com.fasterxml.jackson.core jackson-databind diff --git a/ranger-server/src/main/java/com/flipkart/ranger/server/App.java b/ranger-server/src/main/java/com/flipkart/ranger/server/App.java index 1f594014..c9399a9b 100644 --- a/ranger-server/src/main/java/com/flipkart/ranger/server/App.java +++ b/ranger-server/src/main/java/com/flipkart/ranger/server/App.java @@ -1,11 +1,28 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.flipkart.ranger.server; +import com.flipkart.ranger.client.RangerHubClient; +import com.flipkart.ranger.core.model.Criteria; +import com.flipkart.ranger.server.bundle.RangerServerBundle; import com.flipkart.ranger.server.healthcheck.RangerHealthCheck; -import com.flipkart.ranger.server.manager.RangerClientManager; -import com.flipkart.ranger.server.resources.RangerResource; -import com.flipkart.ranger.server.rotation.BirTask; -import com.flipkart.ranger.server.rotation.OorTask; -import com.flipkart.ranger.server.rotation.RotationStatus; +import com.flipkart.ranger.server.manager.RangerBundleManager; +import com.flipkart.ranger.server.model.ShardInfo; +import com.flipkart.ranger.server.util.RangerServerUtils; +import com.flipkart.ranger.zookeeper.serde.ZkNodeDataDeserializer; import io.dropwizard.Application; import io.dropwizard.setup.Bootstrap; import io.dropwizard.setup.Environment; @@ -15,8 +32,6 @@ @Slf4j public class App extends Application { - private RotationStatus rotationStatus; - public static void main(String[] args) throws Exception { new App().run(args); } @@ -28,15 +43,30 @@ public void initialize(Bootstrap bootstrap) { @Override public void run(AppConfiguration appConfiguration, Environment environment) { - rotationStatus = new RotationStatus(appConfiguration.isInitialRotationStatus()); - val rangerClientManager = new RangerClientManager(appConfiguration, environment.getObjectMapper()); + val rangerConfiguration = appConfiguration.getRangerConfiguration(); + RangerServerUtils.verifyPreconditions(rangerConfiguration); + val curatorFramework = RangerServerUtils.buildCurator(rangerConfiguration); + + val rangerServerBundle = new RangerServerBundle, ZkNodeDataDeserializer, + AppConfiguration>() { + @Override + protected RangerHubClient> withRangerHub(AppConfiguration configuration) { + return RangerServerUtils.buildRangerHub(curatorFramework, rangerConfiguration, environment.getObjectMapper()); + } + + @Override + protected boolean withInitialRotationStatus(AppConfiguration configuration) { + return appConfiguration.isInitialRotationStatus(); + } + }; + rangerServerBundle.run(appConfiguration, environment); + + val rangerClientManager = new RangerBundleManager(curatorFramework, rangerServerBundle); environment.lifecycle().manage(rangerClientManager); - environment.jersey().register(new RangerResource(rangerClientManager)); - environment.admin() - .addTask(new OorTask(rotationStatus)); - environment.admin() - .addTask(new BirTask(rotationStatus)); + environment.healthChecks().register( - "ranger-healthcheck", new RangerHealthCheck(rangerClientManager.getCuratorFramework())); + "ranger-health-check", + new RangerHealthCheck(rangerClientManager.getCuratorFramework()) + ); } } diff --git a/ranger-server/src/main/java/com/flipkart/ranger/server/AppConfiguration.java b/ranger-server/src/main/java/com/flipkart/ranger/server/AppConfiguration.java index 200730c9..057bba05 100644 --- a/ranger-server/src/main/java/com/flipkart/ranger/server/AppConfiguration.java +++ b/ranger-server/src/main/java/com/flipkart/ranger/server/AppConfiguration.java @@ -1,18 +1,31 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.flipkart.ranger.server; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.flipkart.ranger.client.RangerClientConstants; -import com.flipkart.ranger.core.model.Service; +import com.flipkart.ranger.server.config.RangerConfiguration; import io.dropwizard.Configuration; import lombok.AllArgsConstructor; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; -import javax.validation.constraints.Min; +import javax.validation.Valid; import javax.validation.constraints.NotEmpty; import javax.validation.constraints.NotNull; -import java.util.List; @Getter @Setter @@ -25,21 +38,10 @@ public class AppConfiguration extends Configuration { @NotNull private String name; - @NotEmpty - @NotNull - private String zookeeper; - - @NotEmpty + @Valid @NotNull - private String namespace; - - @Min(1000) - private int refreshTimeMs = RangerClientConstants.MINIMUM_REFRESH_TIME; - - private boolean disablePushUpdaters = true; - - // Specify these if you don't want to use the data source! - private List services; + private RangerConfiguration rangerConfiguration; private boolean initialRotationStatus; + } diff --git a/ranger-server/src/main/java/com/flipkart/ranger/server/config/RangerConfiguration.java b/ranger-server/src/main/java/com/flipkart/ranger/server/config/RangerConfiguration.java new file mode 100644 index 00000000..ca4c92fa --- /dev/null +++ b/ranger-server/src/main/java/com/flipkart/ranger/server/config/RangerConfiguration.java @@ -0,0 +1,30 @@ +package com.flipkart.ranger.server.config; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.flipkart.ranger.client.RangerClientConstants; +import com.flipkart.ranger.core.model.Service; +import lombok.*; + +import javax.validation.constraints.Min; +import javax.validation.constraints.NotEmpty; +import javax.validation.constraints.NotNull; +import java.util.List; + +@Getter +@Setter +@AllArgsConstructor +@NoArgsConstructor +@Builder +@JsonIgnoreProperties(ignoreUnknown = true) +public class RangerConfiguration { + @NotEmpty + @NotNull + private String namespace; + @NotEmpty + @NotNull + private String zookeeper; + private boolean disablePushUpdaters; + @Min(1000) + private int nodeRefreshTimeMs = RangerClientConstants.MINIMUM_REFRESH_TIME; + private List services; +} diff --git a/ranger-server/src/main/java/com/flipkart/ranger/server/healthcheck/RangerHealthCheck.java b/ranger-server/src/main/java/com/flipkart/ranger/server/healthcheck/RangerHealthCheck.java index 000c6252..b27968ef 100644 --- a/ranger-server/src/main/java/com/flipkart/ranger/server/healthcheck/RangerHealthCheck.java +++ b/ranger-server/src/main/java/com/flipkart/ranger/server/healthcheck/RangerHealthCheck.java @@ -1,3 +1,18 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.flipkart.ranger.server.healthcheck; import com.codahale.metrics.health.HealthCheck; diff --git a/ranger-server/src/main/java/com/flipkart/ranger/server/manager/RangerBundleManager.java b/ranger-server/src/main/java/com/flipkart/ranger/server/manager/RangerBundleManager.java new file mode 100644 index 00000000..bf060353 --- /dev/null +++ b/ranger-server/src/main/java/com/flipkart/ranger/server/manager/RangerBundleManager.java @@ -0,0 +1,67 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.server.manager; + +import com.flipkart.ranger.server.bundle.RangerServerBundle; +import io.dropwizard.lifecycle.Managed; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.apache.curator.framework.CuratorFramework; + +import javax.inject.Inject; +import javax.inject.Singleton; + +@Singleton +@Getter +@Slf4j +public class RangerBundleManager implements Managed { + + private final CuratorFramework curatorFramework; + private final RangerServerBundle rangerServerBundle; + + @Inject + public RangerBundleManager( + CuratorFramework curatorFramework, + RangerServerBundle rangerServerBundle + ){ + this.curatorFramework = curatorFramework; + this.rangerServerBundle = rangerServerBundle; + } + + + @Override + public void start() { + log.info("Starting the ranger client manager"); + + curatorFramework.start(); + try { + curatorFramework.blockUntilConnected(); + } + catch (InterruptedException e) { + log.error("Curator block interrupted", e); + } + rangerServerBundle.start(); + log.info("Started the ranger client manager"); + } + + @Override + public void stop() { + log.info("Stopping the ranger client manager"); + rangerServerBundle.stop(); + curatorFramework.close(); + log.info("Stopped the ranger client manager"); + } +} diff --git a/ranger-server/src/main/java/com/flipkart/ranger/server/manager/RangerClientManager.java b/ranger-server/src/main/java/com/flipkart/ranger/server/manager/RangerClientManager.java deleted file mode 100644 index ceb2b48c..00000000 --- a/ranger-server/src/main/java/com/flipkart/ranger/server/manager/RangerClientManager.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright 2015 Flipkart Internet Pvt. Ltd. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.flipkart.ranger.server.manager; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.flipkart.ranger.client.RangerClientConstants; -import com.flipkart.ranger.client.zk.UnshardedRangerZKHubClient; -import com.flipkart.ranger.core.model.Criteria; -import com.flipkart.ranger.core.model.ServiceNode; -import com.flipkart.ranger.server.AppConfiguration; -import com.flipkart.ranger.server.model.ShardInfo; -import com.google.common.base.Preconditions; -import io.dropwizard.lifecycle.Managed; -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.CuratorFrameworkFactory; -import org.apache.curator.retry.RetryForever; - -import javax.inject.Inject; -import javax.inject.Singleton; -import java.io.IOException; - -@Singleton -@Getter -@Slf4j -public class RangerClientManager implements Managed { - - private final AppConfiguration appConfiguration; - private final CuratorFramework curatorFramework; - private final UnshardedRangerZKHubClient> hubClient; - - @Inject - public RangerClientManager(AppConfiguration appConfiguration, ObjectMapper mapper){ - Preconditions.checkNotNull(appConfiguration, "Ranger configuration can't be null"); - Preconditions.checkNotNull(appConfiguration.getZookeeper(), "Zookeeper can't be null"); - Preconditions.checkNotNull(appConfiguration.getNamespace(), "Namespace can't be null"); - Preconditions.checkNotNull(mapper, "Mapper can't be null"); - - this.appConfiguration = appConfiguration; - this.curatorFramework = CuratorFrameworkFactory.newClient( - appConfiguration.getZookeeper(), - new RetryForever(RangerClientConstants.CONNECTION_RETRY_TIME) - ); - this.hubClient = UnshardedRangerZKHubClient.>builder() - .namespace(appConfiguration.getNamespace()) - .connectionString(appConfiguration.getZookeeper()) - .curatorFramework(curatorFramework) - .disablePushUpdaters(appConfiguration.isDisablePushUpdaters()) - .mapper(mapper) - .services(appConfiguration.getServices()) - .refreshTimeMs(appConfiguration.getRefreshTimeMs()) - .deserializer(data -> { - try { - mapper.readValue(data, new TypeReference>() { - }); - } catch (IOException e) { - log.warn("Error parsing node data with value {}", new String(data)); - } - return null; - }) - .build(); - } - - @Override - public void start() { - log.info("Starting the ranger client manager"); - curatorFramework.start(); - try { - curatorFramework.blockUntilConnected(); - } - catch (InterruptedException e) { - log.error("Curator block interrupted", e); - } - hubClient.start(); - log.info("Started the ranger client manager"); - } - - @Override - public void stop() { - log.info("Stopping the ranger client manager"); - hubClient.stop(); - curatorFramework.close(); - log.info("Stopped the ranger client manager"); - } -} diff --git a/ranger-server/src/main/java/com/flipkart/ranger/server/rotation/BirTask.java b/ranger-server/src/main/java/com/flipkart/ranger/server/rotation/BirTask.java deleted file mode 100644 index 05ffb6cb..00000000 --- a/ranger-server/src/main/java/com/flipkart/ranger/server/rotation/BirTask.java +++ /dev/null @@ -1,25 +0,0 @@ -package com.flipkart.ranger.server.rotation; - -import io.dropwizard.servlets.tasks.Task; -import lombok.extern.slf4j.Slf4j; - -import java.io.PrintWriter; -import java.util.List; -import java.util.Map; - -@Slf4j -public class BirTask extends Task { - - private RotationStatus rotationStatus; - - public BirTask(RotationStatus rotationStatus) { - super("ranger-bir"); - this.rotationStatus = rotationStatus; - } - - @Override - public void execute(Map> map, PrintWriter printWriter) { - rotationStatus.bir(); - log.info("Taking node back into rotation on ranger"); - } -} diff --git a/ranger-server/src/main/java/com/flipkart/ranger/server/rotation/OorTask.java b/ranger-server/src/main/java/com/flipkart/ranger/server/rotation/OorTask.java deleted file mode 100644 index c7496e33..00000000 --- a/ranger-server/src/main/java/com/flipkart/ranger/server/rotation/OorTask.java +++ /dev/null @@ -1,26 +0,0 @@ -package com.flipkart.ranger.server.rotation; - - -import io.dropwizard.servlets.tasks.Task; -import lombok.extern.slf4j.Slf4j; - -import java.io.PrintWriter; -import java.util.List; -import java.util.Map; - -@Slf4j -public class OorTask extends Task { - - private RotationStatus rotationStatus; - - public OorTask(RotationStatus rotationStatus) { - super("ranger-oor"); - this.rotationStatus = rotationStatus; - } - - @Override - public void execute(Map> map, PrintWriter printWriter) { - rotationStatus.oor(); - log.info("Taking node out of rotation on ranger"); - } -} diff --git a/ranger-server/src/main/java/com/flipkart/ranger/server/rotation/RotationStatus.java b/ranger-server/src/main/java/com/flipkart/ranger/server/rotation/RotationStatus.java deleted file mode 100644 index 1a32e4cf..00000000 --- a/ranger-server/src/main/java/com/flipkart/ranger/server/rotation/RotationStatus.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.flipkart.ranger.server.rotation; - -import java.util.concurrent.atomic.AtomicBoolean; - -public class RotationStatus { - private AtomicBoolean rotationStatus; - - public RotationStatus(boolean initialStatus) { - rotationStatus = new AtomicBoolean(initialStatus); - } - - public void oor() { - rotationStatus.set(false); - } - - public void bir() { - rotationStatus.set(true); - } - - public boolean status() { - return rotationStatus.get(); - } -} diff --git a/ranger-server/src/main/java/com/flipkart/ranger/server/util/RangerServerUtils.java b/ranger-server/src/main/java/com/flipkart/ranger/server/util/RangerServerUtils.java new file mode 100644 index 00000000..8520195c --- /dev/null +++ b/ranger-server/src/main/java/com/flipkart/ranger/server/util/RangerServerUtils.java @@ -0,0 +1,64 @@ +package com.flipkart.ranger.server.util; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.flipkart.ranger.client.RangerClientConstants; +import com.flipkart.ranger.client.zk.UnshardedRangerZKHubClient; +import com.flipkart.ranger.core.model.Criteria; +import com.flipkart.ranger.core.model.ServiceNode; +import com.flipkart.ranger.server.config.RangerConfiguration; +import com.flipkart.ranger.server.model.ShardInfo; +import com.google.common.base.Preconditions; +import lombok.extern.slf4j.Slf4j; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.retry.RetryForever; + +import java.io.IOException; + +@Slf4j +public class RangerServerUtils { + + private RangerServerUtils(){} + + public static void verifyPreconditions(RangerConfiguration rangerConfiguration){ + Preconditions.checkNotNull(rangerConfiguration, + "ranger configuration can't be null"); + Preconditions.checkNotNull(rangerConfiguration.getNamespace(), + "Namespace can't be null"); + Preconditions.checkNotNull(rangerConfiguration.getZookeeper(), + "Zookeeper can't be null"); + } + + public static CuratorFramework buildCurator(RangerConfiguration rangerConfiguration){ + return CuratorFrameworkFactory.newClient( + rangerConfiguration.getZookeeper(), + new RetryForever(RangerClientConstants.CONNECTION_RETRY_TIME) + ); + } + + public static UnshardedRangerZKHubClient> buildRangerHub( + CuratorFramework curatorFramework, + RangerConfiguration rangerConfiguration, + ObjectMapper mapper + ){ + return UnshardedRangerZKHubClient.>builder() + .namespace(rangerConfiguration.getNamespace()) + .connectionString(rangerConfiguration.getZookeeper()) + .curatorFramework(curatorFramework) + .disablePushUpdaters(rangerConfiguration.isDisablePushUpdaters()) + .mapper(mapper) + .services(rangerConfiguration.getServices()) + .refreshTimeMs(rangerConfiguration.getNodeRefreshTimeMs()) + .deserializer(data -> { + try { + mapper.readValue(data, new TypeReference>() { + }); + } catch (IOException e) { + log.warn("Error parsing node data with value {}", new String(data)); + } + return null; + }) + .build(); + } +} diff --git a/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/AbstractRangerZKHubClient.java b/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/AbstractRangerZKHubClient.java index e10ab1a5..5dac14e2 100644 --- a/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/AbstractRangerZKHubClient.java +++ b/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/AbstractRangerZKHubClient.java @@ -29,8 +29,6 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.CuratorFrameworkFactory; -import org.apache.curator.retry.RetryForever; import java.util.List; @@ -63,13 +61,7 @@ public AbstractRangerZKHubClient( this.disablePushUpdaters = disablePushUpdaters; this.connectionString = connectionString; this.services = services; - - this.curatorFramework = null != curatorFramework ? curatorFramework : - CuratorFrameworkFactory.builder() - .namespace(namespace) - .connectString(this.connectionString) - .retryPolicy(new RetryForever(5000)) - .build(); + this.curatorFramework = curatorFramework; } From 9b7254650f0dfdd0cec037e05deda2db6ed30ffc Mon Sep 17 00:00:00 2001 From: Koushik R Date: Thu, 21 Oct 2021 12:34:50 +0530 Subject: [PATCH 09/69] a) Added test-jar to ranger-client. b) Changed the http node data sources to provide default empty list in response --- pom.xml | 1 + ranger-client/pom.xml | 17 +++++++++++++++++ .../http/model/ServiceDataSourceResponse.java | 1 + .../http/servicefinder/HttpNodeDataSource.java | 10 +++++++--- .../servicefinderhub/HttpServiceDataSource.java | 8 ++++++-- .../http/serviceprovider/HttpNodeDataSink.java | 6 +++--- 6 files changed, 35 insertions(+), 8 deletions(-) diff --git a/pom.xml b/pom.xml index 6165389f..cba14863 100644 --- a/pom.xml +++ b/pom.xml @@ -123,6 +123,7 @@ guava-retrying ${guava-retrying.version} + org.awaitility awaitility diff --git a/ranger-client/pom.xml b/ranger-client/pom.xml index bc563c1a..28df3cfb 100644 --- a/ranger-client/pom.xml +++ b/ranger-client/pom.xml @@ -29,4 +29,21 @@ ${jackson.version} + + + + + org.apache.maven.plugins + maven-jar-plugin + 3.2.0 + + + + test-jar + + + + + + \ No newline at end of file diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceDataSourceResponse.java b/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceDataSourceResponse.java index 33b378a8..bbe33e5b 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceDataSourceResponse.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceDataSourceResponse.java @@ -19,6 +19,7 @@ import com.flipkart.ranger.core.model.Service; import lombok.*; +import java.util.Collections; import java.util.List; @AllArgsConstructor diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpNodeDataSource.java b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpNodeDataSource.java index bef6aaa9..47666d50 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpNodeDataSource.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpNodeDataSource.java @@ -32,6 +32,7 @@ import okhttp3.ResponseBody; import java.io.IOException; +import java.util.Collections; import java.util.List; import java.util.Optional; @@ -70,6 +71,7 @@ public Optional>> refresh(D deserializer) { .get() .build(); + List> serviceNodes = Collections.emptyList(); try (val response = httpClient.newCall(request).execute()) { if (response.isSuccessful()) { try (final ResponseBody body = response.body()) { @@ -79,12 +81,14 @@ public Optional>> refresh(D deserializer) { final byte[] bytes = body.bytes(); val serviceNodesResponse = deserializer.deserialize(bytes); if(serviceNodesResponse.isSuccess()){ - return Optional.of(FinderUtils.filterValidNodes( + serviceNodes = FinderUtils.filterValidNodes( service, serviceNodesResponse.getData(), - healthcheckZombieCheckThresholdTime(service))); + healthcheckZombieCheckThresholdTime(service)); + }else{ + log.warn("Http call to {} returned a failure response with error {}", httpUrl, serviceNodesResponse.getError()); } - log.warn("Http call to {} returned a failure response with error {}", httpUrl, serviceNodesResponse.getError()); + return Optional.of(serviceNodes); } } } else { diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSource.java b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSource.java index 9268c84e..e3aa61cd 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSource.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSource.java @@ -32,6 +32,7 @@ import java.io.IOException; import java.util.Collection; import java.util.Collections; +import java.util.List; @Slf4j public class HttpServiceDataSource extends HttpNodeDataStoreConnector implements ServiceDataSource { @@ -60,6 +61,7 @@ public Collection services() { .get() .build(); + List services = Collections.emptyList(); try (val response = httpClient.newCall(request).execute()) { if (response.isSuccessful()) { try (final ResponseBody body = response.body()) { @@ -70,9 +72,11 @@ public Collection services() { final byte[] bytes = body.bytes(); val serviceDataSourceResponse = mapper.readValue(bytes, ServiceDataSourceResponse.class); if(serviceDataSourceResponse.isSuccess()){ - return serviceDataSourceResponse.getData(); + services = serviceDataSourceResponse.getData(); + }else{ + log.warn("Http call to {} returned a failure response with error {}", httpUrl, serviceDataSourceResponse.getError()); } - log.warn("Http call to {} returned a failure response with error {}", httpUrl, serviceDataSourceResponse.getError()); + return services; } } } diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/serviceprovider/HttpNodeDataSink.java b/ranger-http/src/main/java/com/flipkart/ranger/http/serviceprovider/HttpNodeDataSink.java index c5df26a6..3988b91e 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/serviceprovider/HttpNodeDataSink.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/serviceprovider/HttpNodeDataSink.java @@ -81,10 +81,10 @@ private Optional registerService(HttpUrl httpUrl, R else { final byte[] bytes = body.bytes(); val serviceRegistrationResponse = mapper.readValue(bytes, ServiceRegistrationResponse.class); - if(serviceRegistrationResponse.isSuccess()) { - return Optional.of(serviceRegistrationResponse); + if(!serviceRegistrationResponse.isSuccess()){ + log.warn("Http call to {} returned a failure response with code {} and error {}", httpUrl, response.code(), serviceRegistrationResponse.getError()); } - log.warn("Http call to {} returned a failure response with code {} and error {}", httpUrl, response.code(), serviceRegistrationResponse.getError()); + return Optional.of(serviceRegistrationResponse); } } } From c6351bc0904e81e6630aa3882e98a6d5fef53d94 Mon Sep 17 00:00:00 2001 From: Koushik R Date: Thu, 21 Oct 2021 12:42:33 +0530 Subject: [PATCH 10/69] Small refactoring in http sinks/source, sending out an empty list in case of an error --- .../flipkart/ranger/http/servicefinder/HttpNodeDataSource.java | 3 +-- .../ranger/http/servicefinderhub/HttpServiceDataSource.java | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpNodeDataSource.java b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpNodeDataSource.java index 47666d50..09e25700 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpNodeDataSource.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpNodeDataSource.java @@ -88,7 +88,6 @@ public Optional>> refresh(D deserializer) { }else{ log.warn("Http call to {} returned a failure response with error {}", httpUrl, serviceNodesResponse.getError()); } - return Optional.of(serviceNodes); } } } else { @@ -97,7 +96,7 @@ public Optional>> refresh(D deserializer) { } catch (IOException e) { Exceptions.illegalState("Error fetching data from server: " + httpUrl, e); } - return Optional.empty(); + return Optional.of(serviceNodes); } @Override diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSource.java b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSource.java index e3aa61cd..b5bcd7a7 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSource.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSource.java @@ -76,7 +76,6 @@ public Collection services() { }else{ log.warn("Http call to {} returned a failure response with error {}", httpUrl, serviceDataSourceResponse.getError()); } - return services; } } } @@ -89,6 +88,6 @@ public Collection services() { } log.error("No data returned from server: " + httpUrl); - return Collections.emptyList(); + return services; } } From d4098532fe9263434c3ca16621ba1986e3aa40d9 Mon Sep 17 00:00:00 2001 From: Koushik R Date: Thu, 21 Oct 2021 17:05:26 +0530 Subject: [PATCH 11/69] a) Added an http-server as well, which can act as a discovery hub atop a bunch of ranger http providers. WIP version #0 --- pom.xml | 2 + ranger-http-server/pom.xml | 52 +++++++++++++++++++ .../com/flipkart/ranger/http/server/App.java | 50 ++++++++++++++++++ .../ranger/http/server/AppConfiguration.java | 31 +++++++++++ .../config/RangerHttpConfiguration.java | 26 ++++++++++ .../manager/RangerHttpBundleManager.java | 36 +++++++++++++ .../server/util/RangerHttpServerUtils.java | 52 +++++++++++++++++++ .../server/bundle/RangerServerBundle.java | 16 +++--- .../bundle/resources/RangerResource.java | 35 ++++++++----- .../server/bundle/RangerServerBundleTest.java | 17 +++--- ranger-server-common/pom.xml | 22 ++++++++ .../ShardInfo.java | 2 +- ranger-server/pom.xml | 5 ++ .../java/com/flipkart/ranger/server/App.java | 12 +++-- .../ranger/server/util/RangerServerUtils.java | 2 +- 15 files changed, 330 insertions(+), 30 deletions(-) create mode 100644 ranger-http-server/pom.xml create mode 100644 ranger-http-server/src/main/java/com/flipkart/ranger/http/server/App.java create mode 100644 ranger-http-server/src/main/java/com/flipkart/ranger/http/server/AppConfiguration.java create mode 100644 ranger-http-server/src/main/java/com/flipkart/ranger/http/server/config/RangerHttpConfiguration.java create mode 100644 ranger-http-server/src/main/java/com/flipkart/ranger/http/server/manager/RangerHttpBundleManager.java create mode 100644 ranger-http-server/src/main/java/com/flipkart/ranger/http/server/util/RangerHttpServerUtils.java create mode 100644 ranger-server-common/pom.xml rename {ranger-server/src/main/java/com/flipkart/ranger/server/model => ranger-server-common/src/main/java/com.flipkart.ranger.server.common}/ShardInfo.java (97%) diff --git a/pom.xml b/pom.xml index cba14863..08855dac 100644 --- a/pom.xml +++ b/pom.xml @@ -17,6 +17,8 @@ ranger-http-client ranger-server ranger-server-bundle + ranger-http-server + ranger-server-common diff --git a/ranger-http-server/pom.xml b/ranger-http-server/pom.xml new file mode 100644 index 00000000..b5a482ee --- /dev/null +++ b/ranger-http-server/pom.xml @@ -0,0 +1,52 @@ + + + + ranger + com.flipkart.ranger + 1.0-SNAPSHOT + + 4.0.0 + + ranger-http-server + + + 2.0.23 + + + + + com.flipkart.ranger + ranger-http-client + ${project.version} + + + com.flipkart.ranger + ranger-server-bundle + ${project.version} + + + com.flipkart.ranger + ranger-server-common + ${project.version} + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + + + io.dropwizard + dropwizard-core + ${dropwizard.version} + + + org.mockito + mockito-core + 3.6.0 + test + + + + \ No newline at end of file diff --git a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/App.java b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/App.java new file mode 100644 index 00000000..e853c81f --- /dev/null +++ b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/App.java @@ -0,0 +1,50 @@ +package com.flipkart.ranger.http.server; + +import com.flipkart.ranger.client.RangerHubClient; +import com.flipkart.ranger.core.model.Criteria; +import com.flipkart.ranger.http.serde.HTTPResponseDataDeserializer; +import com.flipkart.ranger.http.server.manager.RangerHttpBundleManager; +import com.flipkart.ranger.http.server.util.RangerHttpServerUtils; +import com.flipkart.ranger.server.bundle.RangerServerBundle; +import com.flipkart.ranger.server.common.ShardInfo; +import com.google.common.collect.Lists; +import io.dropwizard.Application; +import io.dropwizard.setup.Environment; +import lombok.val; + +import java.util.List; +import java.util.stream.Collectors; + +public class App extends Application { + + @Override + public void run(AppConfiguration appConfiguration, Environment environment) { + val rangerConfiguration = appConfiguration.getRangerConfiguration(); + RangerHttpServerUtils.verifyPreconditions(rangerConfiguration); + + val rangerServerBundle = new RangerServerBundle, HTTPResponseDataDeserializer, + AppConfiguration>() { + + @Override + protected List>> withHubs(AppConfiguration configuration) { + val clientConfigs = configuration.getRangerConfiguration().getHttpClientConfigs(); + return clientConfigs.stream().map(clientConfig -> + RangerHttpServerUtils.buildRangerHub( + rangerConfiguration.getNamespace(), + rangerConfiguration.getNodeRefreshTimeMs(), + clientConfig, + environment.getObjectMapper() + )).collect(Collectors.toList()); + } + + @Override + protected boolean withInitialRotationStatus(AppConfiguration configuration) { + return appConfiguration.isInitialRotationStatus(); + } + }; + rangerServerBundle.run(appConfiguration, environment); + + val rangerClientManager = new RangerHttpBundleManager(rangerServerBundle); + environment.lifecycle().manage(rangerClientManager); + } +} diff --git a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/AppConfiguration.java b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/AppConfiguration.java new file mode 100644 index 00000000..372a2ba9 --- /dev/null +++ b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/AppConfiguration.java @@ -0,0 +1,31 @@ +package com.flipkart.ranger.http.server; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.flipkart.ranger.http.server.config.RangerHttpConfiguration; +import io.dropwizard.Configuration; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +import javax.validation.Valid; +import javax.validation.constraints.NotEmpty; +import javax.validation.constraints.NotNull; + + +@Getter +@Setter +@AllArgsConstructor +@NoArgsConstructor +@JsonIgnoreProperties(ignoreUnknown = true) +public class AppConfiguration extends Configuration { + @NotEmpty + @NotNull + private String name; + + @Valid + @NotNull + private RangerHttpConfiguration rangerConfiguration; + + private boolean initialRotationStatus; +} diff --git a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/config/RangerHttpConfiguration.java b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/config/RangerHttpConfiguration.java new file mode 100644 index 00000000..b2f1314d --- /dev/null +++ b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/config/RangerHttpConfiguration.java @@ -0,0 +1,26 @@ +package com.flipkart.ranger.http.server.config; + +import com.flipkart.ranger.core.model.Service; +import com.flipkart.ranger.http.config.HttpClientConfig; +import lombok.*; + +import javax.validation.constraints.NotEmpty; +import javax.validation.constraints.NotNull; +import java.util.List; + +@AllArgsConstructor +@NoArgsConstructor +@Getter +@Setter +@Builder +public class RangerHttpConfiguration { + + @NotEmpty + @NotNull + private String namespace; + @NotEmpty + @NotNull + private List httpClientConfigs; + private int nodeRefreshTimeMs; + +} diff --git a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/manager/RangerHttpBundleManager.java b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/manager/RangerHttpBundleManager.java new file mode 100644 index 00000000..c1e0d92b --- /dev/null +++ b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/manager/RangerHttpBundleManager.java @@ -0,0 +1,36 @@ +package com.flipkart.ranger.http.server.manager; + +import com.flipkart.ranger.server.bundle.RangerServerBundle; +import io.dropwizard.lifecycle.Managed; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + +import javax.inject.Inject; +import javax.inject.Singleton; + +@Slf4j +@Singleton +@Getter +public class RangerHttpBundleManager implements Managed { + + private final RangerServerBundle rangerServerBundle; + + @Inject + public RangerHttpBundleManager(RangerServerBundle rangerServerBundle){ + this.rangerServerBundle = rangerServerBundle; + } + + @Override + public void start() { + log.info("Starting the ranger http bundle manager"); + rangerServerBundle.start(); + log.info("Started the ranger http bundle manager"); + } + + @Override + public void stop() { + log.info("Stopping the ranger http bundle manager"); + rangerServerBundle.stop(); + log.info("Stopped the ranger http bundle manager"); + } +} diff --git a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/util/RangerHttpServerUtils.java b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/util/RangerHttpServerUtils.java new file mode 100644 index 00000000..c3a6e533 --- /dev/null +++ b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/util/RangerHttpServerUtils.java @@ -0,0 +1,52 @@ +package com.flipkart.ranger.http.server.util; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.flipkart.ranger.client.http.UnshardedRangerHttpHubClient; +import com.flipkart.ranger.core.model.Criteria; +import com.flipkart.ranger.core.model.ServiceNode; +import com.flipkart.ranger.http.config.HttpClientConfig; +import com.flipkart.ranger.http.server.config.RangerHttpConfiguration; +import com.flipkart.ranger.server.common.ShardInfo; +import com.google.common.base.Preconditions; +import lombok.extern.slf4j.Slf4j; + +import java.io.IOException; + +@Slf4j +public class RangerHttpServerUtils { + + private RangerHttpServerUtils(){} + + public static void verifyPreconditions(RangerHttpConfiguration rangerConfiguration){ + Preconditions.checkNotNull(rangerConfiguration, + "ranger configuration can't be null"); + Preconditions.checkNotNull(rangerConfiguration.getNamespace(), + "Namespace can't be null"); + Preconditions.checkArgument(null != rangerConfiguration.getHttpClientConfigs() && !rangerConfiguration.getHttpClientConfigs().isEmpty(), + "Http client config can't be null"); + } + + public static UnshardedRangerHttpHubClient> buildRangerHub( + String namespace, + int nodeRefreshTimeMs, + HttpClientConfig clientConfig, + ObjectMapper mapper + ){ + return UnshardedRangerHttpHubClient.>builder() + .namespace(namespace) + .mapper(mapper) + .clientConfig(clientConfig) + .nodeRefreshIntervalMs(nodeRefreshTimeMs) + .deserializer(data -> { + try { + mapper.readValue(data, new TypeReference>() { + }); + } catch (IOException e) { + log.warn("Error parsing node data with value {}", new String(data)); + } + return null; + }) + .build(); + } +} diff --git a/ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/RangerServerBundle.java b/ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/RangerServerBundle.java index e9ceefb6..93aa32e4 100644 --- a/ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/RangerServerBundle.java +++ b/ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/RangerServerBundle.java @@ -30,6 +30,8 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import java.util.List; + @Slf4j public abstract class RangerServerBundle< T, @@ -39,12 +41,12 @@ public abstract class RangerServerBundle< > implements ConfiguredBundle{ @Getter - private RangerHubClient rangerHub; + private List> hubs; @Getter @VisibleForTesting private RotationStatus rotationStatus; - protected abstract RangerHubClient withRangerHub(U configuration); + protected abstract List> withHubs(U configuration); protected abstract boolean withInitialRotationStatus(U configuration); @@ -55,7 +57,7 @@ public void initialize(Bootstrap bootstrap) { @Override public void run(U configuration, Environment environment) { - rangerHub = withRangerHub(configuration); + hubs = withHubs(configuration); rotationStatus = new RotationStatus(withInitialRotationStatus(configuration)); environment.admin() @@ -63,16 +65,16 @@ public void run(U configuration, Environment environment) { environment.admin() .addTask(new BirTask(rotationStatus)); - environment.jersey().register(new RangerResource<>(rangerHub)); + environment.jersey().register(new RangerResource<>(hubs)); } public void start(){ - log.info("Starting the ranger hub"); - rangerHub.start(); + log.info("Starting the ranger hubs"); + hubs.forEach(RangerHubClient::start); } public void stop(){ log.info("Stopping the ranger hub"); - rangerHub.stop(); + hubs.forEach(RangerHubClient::stop); } } diff --git a/ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/resources/RangerResource.java b/ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/resources/RangerResource.java index d753056a..bc589fa3 100644 --- a/ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/resources/RangerResource.java +++ b/ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/resources/RangerResource.java @@ -29,10 +29,7 @@ import javax.validation.constraints.NotNull; import javax.ws.rs.*; import javax.ws.rs.core.MediaType; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Optional; +import java.util.*; @Slf4j @Consumes(MediaType.APPLICATION_JSON) @@ -40,20 +37,28 @@ @Path("/ranger") public class RangerResource> { - private final RangerHubClient rangerHub; + private final List> rangerHubs; @Inject - public RangerResource(RangerHubClient rangerHub){ - this.rangerHub = rangerHub; + public RangerResource(List> rangerHubs){ + this.rangerHubs = rangerHubs; } @GET @Path("/services/v1") @Metered - public GenericResponse> getServices() throws Exception { + public GenericResponse> getServices() { + Collection services = new ArrayList<>(); + for (RangerHubClient hub : rangerHubs) { + try { + services.addAll(hub.getServices()); + } catch (Exception e) { + log.warn("Call to a hub failed with exception, {}", e.getMessage()); + } + } return GenericResponse.>builder() .success(true) - .data(rangerHub.getServices()) + .data(services) .build(); } @@ -65,11 +70,17 @@ public GenericResponse>> getNodes( @NotNull @NotEmpty @PathParam("serviceName") final String serviceName ){ val service = new Service(namespace, serviceName); - Optional>> nodeList = rangerHub.getAllNodes( - service, null); + List> serviceNodes = new ArrayList<>(); + for (RangerHubClient hub : rangerHubs) { + try { + serviceNodes.addAll(hub.getAllNodes(service, null).orElse(Collections.emptyList())); + } catch (Exception e) { + log.warn("Call to a hub failed with exception, {}", e.getMessage()); + } + } return GenericResponse.>>builder() .success(true) - .data(nodeList.orElse(Collections.emptyList())) + .data(serviceNodes) .build(); } } diff --git a/ranger-server-bundle/src/test/java/com/flipkart/ranger/server/bundle/RangerServerBundleTest.java b/ranger-server-bundle/src/test/java/com/flipkart/ranger/server/bundle/RangerServerBundleTest.java index 568c8f03..4ab66c26 100644 --- a/ranger-server-bundle/src/test/java/com/flipkart/ranger/server/bundle/RangerServerBundleTest.java +++ b/ranger-server-bundle/src/test/java/com/flipkart/ranger/server/bundle/RangerServerBundleTest.java @@ -10,6 +10,7 @@ import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.core.model.ServiceNode; +import com.google.common.collect.Lists; import io.dropwizard.Configuration; import io.dropwizard.jersey.DropwizardResourceConfig; import io.dropwizard.jersey.setup.JerseyEnvironment; @@ -17,12 +18,14 @@ import io.dropwizard.setup.AdminEnvironment; import io.dropwizard.setup.Bootstrap; import io.dropwizard.setup.Environment; +import lombok.val; import org.eclipse.jetty.util.component.LifeCycle; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import java.util.List; import java.util.Optional; import static com.flipkart.ranger.client.utils.RangerHubTestUtils.service; @@ -40,9 +43,10 @@ public class RangerServerBundleTest { private final RangerServerBundle, TestDeserializer, Configuration> rangerServerBundle = new RangerServerBundle, TestDeserializer, Configuration>() { + @Override - protected RangerHubClient> withRangerHub(Configuration configuration) { - return RangerHubTestUtils.getTestHub(); + protected List>> withHubs(Configuration configuration) { + return Lists.newArrayList(RangerHubTestUtils.getTestHub()); } @Override @@ -73,19 +77,20 @@ public void setup() throws Exception { @Test public void testRangerBundle(){ TestUtils.sleepForSeconds(3); - Optional> node = rangerServerBundle.getRangerHub().getNode(service); + val hub = rangerServerBundle.getHubs().get(0); + Optional> node = hub.getNode(service); Assert.assertTrue(node.isPresent()); Assert.assertTrue(node.get().getHost().equalsIgnoreCase("localhost")); Assert.assertEquals(9200, node.get().getPort()); Assert.assertEquals(1, node.get().getNodeData().getShardId()); - node = rangerServerBundle.getRangerHub().getNode(new Service("test", "test")); + node = hub.getNode(new Service("test", "test")); Assert.assertFalse(node.isPresent()); - node = rangerServerBundle.getRangerHub().getNode(service, nodeData -> nodeData.getShardId() == 2); + node = hub.getNode(service, nodeData -> nodeData.getShardId() == 2); Assert.assertFalse(node.isPresent()); - node = rangerServerBundle.getRangerHub().getNode(new Service("test", "test"), nodeData -> nodeData.getShardId() == 1); + node = hub.getNode(new Service("test", "test"), nodeData -> nodeData.getShardId() == 1); Assert.assertFalse(node.isPresent()); } diff --git a/ranger-server-common/pom.xml b/ranger-server-common/pom.xml new file mode 100644 index 00000000..7866fe4c --- /dev/null +++ b/ranger-server-common/pom.xml @@ -0,0 +1,22 @@ + + + + ranger + com.flipkart.ranger + 1.0-SNAPSHOT + + 4.0.0 + + ranger-server-common + + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + + + + \ No newline at end of file diff --git a/ranger-server/src/main/java/com/flipkart/ranger/server/model/ShardInfo.java b/ranger-server-common/src/main/java/com.flipkart.ranger.server.common/ShardInfo.java similarity index 97% rename from ranger-server/src/main/java/com/flipkart/ranger/server/model/ShardInfo.java rename to ranger-server-common/src/main/java/com.flipkart.ranger.server.common/ShardInfo.java index 74a1c0af..4da5c5a3 100644 --- a/ranger-server/src/main/java/com/flipkart/ranger/server/model/ShardInfo.java +++ b/ranger-server-common/src/main/java/com.flipkart.ranger.server.common/ShardInfo.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.flipkart.ranger.server.model; +package com.flipkart.ranger.server.common; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import lombok.*; diff --git a/ranger-server/pom.xml b/ranger-server/pom.xml index daeef095..0f8f36ab 100644 --- a/ranger-server/pom.xml +++ b/ranger-server/pom.xml @@ -26,6 +26,11 @@ ranger-server-bundle ${project.version} + + com.flipkart.ranger + ranger-server-common + ${project.version} + com.fasterxml.jackson.core jackson-databind diff --git a/ranger-server/src/main/java/com/flipkart/ranger/server/App.java b/ranger-server/src/main/java/com/flipkart/ranger/server/App.java index c9399a9b..d4318c43 100644 --- a/ranger-server/src/main/java/com/flipkart/ranger/server/App.java +++ b/ranger-server/src/main/java/com/flipkart/ranger/server/App.java @@ -20,15 +20,18 @@ import com.flipkart.ranger.server.bundle.RangerServerBundle; import com.flipkart.ranger.server.healthcheck.RangerHealthCheck; import com.flipkart.ranger.server.manager.RangerBundleManager; -import com.flipkart.ranger.server.model.ShardInfo; +import com.flipkart.ranger.server.common.ShardInfo; import com.flipkart.ranger.server.util.RangerServerUtils; import com.flipkart.ranger.zookeeper.serde.ZkNodeDataDeserializer; +import com.google.common.collect.Lists; import io.dropwizard.Application; import io.dropwizard.setup.Bootstrap; import io.dropwizard.setup.Environment; import lombok.extern.slf4j.Slf4j; import lombok.val; +import java.util.List; + @Slf4j public class App extends Application { @@ -49,9 +52,12 @@ public void run(AppConfiguration appConfiguration, Environment environment) { val rangerServerBundle = new RangerServerBundle, ZkNodeDataDeserializer, AppConfiguration>() { + @Override - protected RangerHubClient> withRangerHub(AppConfiguration configuration) { - return RangerServerUtils.buildRangerHub(curatorFramework, rangerConfiguration, environment.getObjectMapper()); + protected List>> withHubs(AppConfiguration configuration) { + return Lists.newArrayList( + RangerServerUtils.buildRangerHub(curatorFramework, rangerConfiguration, environment.getObjectMapper()) + ); } @Override diff --git a/ranger-server/src/main/java/com/flipkart/ranger/server/util/RangerServerUtils.java b/ranger-server/src/main/java/com/flipkart/ranger/server/util/RangerServerUtils.java index 8520195c..fc702f1f 100644 --- a/ranger-server/src/main/java/com/flipkart/ranger/server/util/RangerServerUtils.java +++ b/ranger-server/src/main/java/com/flipkart/ranger/server/util/RangerServerUtils.java @@ -7,7 +7,7 @@ import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.server.config.RangerConfiguration; -import com.flipkart.ranger.server.model.ShardInfo; +import com.flipkart.ranger.server.common.ShardInfo; import com.google.common.base.Preconditions; import lombok.extern.slf4j.Slf4j; import org.apache.curator.framework.CuratorFramework; From 6a2293019a672eb5a9f7e79f0f2650eefbc6613a Mon Sep 17 00:00:00 2001 From: Koushik R Date: Thu, 21 Oct 2021 18:01:13 +0530 Subject: [PATCH 12/69] Added a local.yml for ranger-http-server --- ranger-http-server/config/local.yml | 38 +++++++++++++++++++ .../com/flipkart/ranger/http/server/App.java | 20 ++++++++++ 2 files changed, 58 insertions(+) create mode 100644 ranger-http-server/config/local.yml diff --git a/ranger-http-server/config/local.yml b/ranger-http-server/config/local.yml new file mode 100644 index 00000000..e0f6ac70 --- /dev/null +++ b/ranger-http-server/config/local.yml @@ -0,0 +1,38 @@ + +name: ranger-http-server +initialRotationStatus: false + +rangerConfiguration: + namespace: test + httpClientConfigs: + - host: localhost + port: 8080 + +server: + maxThreads: 128 + minThreads: 128 + applicationConnectors: + - type: http + port: 7080 + adminConnectors: + - type: http + port: 7081 + applicationContextPath: / + requestLog: + appenders: + - type: console + timeZone: IST + +logging: + level: INFO + + loggers: + com.flipkart.ranger.http.server: DEBUG + + appenders: + - type: console + threshold: INFO + timeZone: IST + logFormat: "%(%-5level) [%date] [%thread] [%logger{0}]: %message%n" + + diff --git a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/App.java b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/App.java index e853c81f..e220cc3e 100644 --- a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/App.java +++ b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/App.java @@ -1,5 +1,6 @@ package com.flipkart.ranger.http.server; +import com.codahale.metrics.health.HealthCheck; import com.flipkart.ranger.client.RangerHubClient; import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.http.serde.HTTPResponseDataDeserializer; @@ -9,6 +10,7 @@ import com.flipkart.ranger.server.common.ShardInfo; import com.google.common.collect.Lists; import io.dropwizard.Application; +import io.dropwizard.setup.Bootstrap; import io.dropwizard.setup.Environment; import lombok.val; @@ -17,6 +19,15 @@ public class App extends Application { + public static void main(String[] args) throws Exception { + new App().run(args); + } + + @Override + public void initialize(Bootstrap bootstrap) { + + } + @Override public void run(AppConfiguration appConfiguration, Environment environment) { val rangerConfiguration = appConfiguration.getRangerConfiguration(); @@ -46,5 +57,14 @@ protected boolean withInitialRotationStatus(AppConfiguration configuration) { val rangerClientManager = new RangerHttpBundleManager(rangerServerBundle); environment.lifecycle().manage(rangerClientManager); + environment.healthChecks().register( + "ranger-http-health-check", + new HealthCheck() { + @Override + protected Result check() { + return Result.healthy(); + } + }) + ); } } From 4b48f73481cd31c0485ae96b91d1bca0cadcaf0b Mon Sep 17 00:00:00 2001 From: Koushik R Date: Thu, 21 Oct 2021 18:29:52 +0530 Subject: [PATCH 13/69] Added tests for ranger-common where shardInfo is being maintained --- .../com/flipkart/ranger/http/server/App.java | 6 +-- .../server/util/RangerHttpServerUtils.java | 2 +- ranger-server-common/pom.xml | 1 - .../ranger/common/server}/ShardInfo.java | 2 +- .../ranger/common/server/ShardInfoTest.java | 44 +++++++++++++++++++ .../src/test/resources/fixtures/env1.json | 4 ++ .../src/test/resources/fixtures/env2.json | 3 ++ .../java/com/flipkart/ranger/server/App.java | 2 +- .../ranger/server/util/RangerServerUtils.java | 2 +- 9 files changed, 57 insertions(+), 9 deletions(-) rename ranger-server-common/src/main/java/{com.flipkart.ranger.server.common => com/flipkart/ranger/common/server}/ShardInfo.java (97%) create mode 100644 ranger-server-common/src/test/java/com/flipkart/ranger/common/server/ShardInfoTest.java create mode 100644 ranger-server-common/src/test/resources/fixtures/env1.json create mode 100644 ranger-server-common/src/test/resources/fixtures/env2.json diff --git a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/App.java b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/App.java index e220cc3e..a548395d 100644 --- a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/App.java +++ b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/App.java @@ -7,8 +7,7 @@ import com.flipkart.ranger.http.server.manager.RangerHttpBundleManager; import com.flipkart.ranger.http.server.util.RangerHttpServerUtils; import com.flipkart.ranger.server.bundle.RangerServerBundle; -import com.flipkart.ranger.server.common.ShardInfo; -import com.google.common.collect.Lists; +import com.flipkart.ranger.common.server.ShardInfo; import io.dropwizard.Application; import io.dropwizard.setup.Bootstrap; import io.dropwizard.setup.Environment; @@ -64,7 +63,6 @@ protected boolean withInitialRotationStatus(AppConfiguration configuration) { protected Result check() { return Result.healthy(); } - }) - ); + }); } } diff --git a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/util/RangerHttpServerUtils.java b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/util/RangerHttpServerUtils.java index c3a6e533..7278a7fe 100644 --- a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/util/RangerHttpServerUtils.java +++ b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/util/RangerHttpServerUtils.java @@ -7,7 +7,7 @@ import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.http.config.HttpClientConfig; import com.flipkart.ranger.http.server.config.RangerHttpConfiguration; -import com.flipkart.ranger.server.common.ShardInfo; +import com.flipkart.ranger.common.server.ShardInfo; import com.google.common.base.Preconditions; import lombok.extern.slf4j.Slf4j; diff --git a/ranger-server-common/pom.xml b/ranger-server-common/pom.xml index 7866fe4c..44565467 100644 --- a/ranger-server-common/pom.xml +++ b/ranger-server-common/pom.xml @@ -18,5 +18,4 @@ ${jackson.version} - \ No newline at end of file diff --git a/ranger-server-common/src/main/java/com.flipkart.ranger.server.common/ShardInfo.java b/ranger-server-common/src/main/java/com/flipkart/ranger/common/server/ShardInfo.java similarity index 97% rename from ranger-server-common/src/main/java/com.flipkart.ranger.server.common/ShardInfo.java rename to ranger-server-common/src/main/java/com/flipkart/ranger/common/server/ShardInfo.java index 4da5c5a3..7d82620f 100644 --- a/ranger-server-common/src/main/java/com.flipkart.ranger.server.common/ShardInfo.java +++ b/ranger-server-common/src/main/java/com/flipkart/ranger/common/server/ShardInfo.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.flipkart.ranger.server.common; +package com.flipkart.ranger.common.server; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import lombok.*; diff --git a/ranger-server-common/src/test/java/com/flipkart/ranger/common/server/ShardInfoTest.java b/ranger-server-common/src/test/java/com/flipkart/ranger/common/server/ShardInfoTest.java new file mode 100644 index 00000000..b25e830e --- /dev/null +++ b/ranger-server-common/src/test/java/com/flipkart/ranger/common/server/ShardInfoTest.java @@ -0,0 +1,44 @@ +package com.flipkart.ranger.common.server; + +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.SneakyThrows; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.Arrays; +import java.util.stream.Collectors; + +public class ShardInfoTest { + private static final ObjectMapper mapper = new ObjectMapper(); + + public static String getResource(String path) { + final InputStream data = ShardInfoTest.class.getClassLoader().getResourceAsStream(path); + return new BufferedReader( + new InputStreamReader(data)) + .lines() + .collect(Collectors.joining("\n")); + } + + @SneakyThrows + public static T getResource(String path, Class klass) { + final String data = getResource(path); + return mapper.readValue(data, klass); + } + + @Test + public void testShardInfo(){ + ShardInfo shardInfo1 = getResource("fixtures/env1.json", ShardInfo.class); + ShardInfo shardInfo2 = getResource("fixtures/env2.json", ShardInfo.class); + + Assert.assertNotNull(shardInfo1); + Assert.assertNotNull(shardInfo2); + + Assert.assertNotEquals(shardInfo1, shardInfo2); + Arrays.asList(shardInfo1, shardInfo2).forEach(shardInfo -> Assert.assertEquals("e", shardInfo.getEnvironment())); + Assert.assertEquals("r", shardInfo1.getRegion()); + Assert.assertNull(shardInfo2.getRegion()); + } +} diff --git a/ranger-server-common/src/test/resources/fixtures/env1.json b/ranger-server-common/src/test/resources/fixtures/env1.json new file mode 100644 index 00000000..1a5445d9 --- /dev/null +++ b/ranger-server-common/src/test/resources/fixtures/env1.json @@ -0,0 +1,4 @@ +{ + "environment" : "e", + "region" : "r" +} \ No newline at end of file diff --git a/ranger-server-common/src/test/resources/fixtures/env2.json b/ranger-server-common/src/test/resources/fixtures/env2.json new file mode 100644 index 00000000..ddfd1a62 --- /dev/null +++ b/ranger-server-common/src/test/resources/fixtures/env2.json @@ -0,0 +1,3 @@ +{ + "environment" : "e" +} \ No newline at end of file diff --git a/ranger-server/src/main/java/com/flipkart/ranger/server/App.java b/ranger-server/src/main/java/com/flipkart/ranger/server/App.java index d4318c43..c8d786e7 100644 --- a/ranger-server/src/main/java/com/flipkart/ranger/server/App.java +++ b/ranger-server/src/main/java/com/flipkart/ranger/server/App.java @@ -20,7 +20,7 @@ import com.flipkart.ranger.server.bundle.RangerServerBundle; import com.flipkart.ranger.server.healthcheck.RangerHealthCheck; import com.flipkart.ranger.server.manager.RangerBundleManager; -import com.flipkart.ranger.server.common.ShardInfo; +import com.flipkart.ranger.common.server.ShardInfo; import com.flipkart.ranger.server.util.RangerServerUtils; import com.flipkart.ranger.zookeeper.serde.ZkNodeDataDeserializer; import com.google.common.collect.Lists; diff --git a/ranger-server/src/main/java/com/flipkart/ranger/server/util/RangerServerUtils.java b/ranger-server/src/main/java/com/flipkart/ranger/server/util/RangerServerUtils.java index fc702f1f..80f5be14 100644 --- a/ranger-server/src/main/java/com/flipkart/ranger/server/util/RangerServerUtils.java +++ b/ranger-server/src/main/java/com/flipkart/ranger/server/util/RangerServerUtils.java @@ -7,7 +7,7 @@ import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.server.config.RangerConfiguration; -import com.flipkart.ranger.server.common.ShardInfo; +import com.flipkart.ranger.common.server.ShardInfo; import com.google.common.base.Preconditions; import lombok.extern.slf4j.Slf4j; import org.apache.curator.framework.CuratorFramework; From 4de42dfa99d0f70bcce0988336d6bda4d43fc60d Mon Sep 17 00:00:00 2001 From: Koushik R Date: Thu, 21 Oct 2021 18:41:58 +0530 Subject: [PATCH 14/69] Removed services from the configuration of server, all services by default --- .../com/flipkart/ranger/server/config/RangerConfiguration.java | 1 - .../java/com/flipkart/ranger/server/util/RangerServerUtils.java | 1 - 2 files changed, 2 deletions(-) diff --git a/ranger-server/src/main/java/com/flipkart/ranger/server/config/RangerConfiguration.java b/ranger-server/src/main/java/com/flipkart/ranger/server/config/RangerConfiguration.java index ca4c92fa..252bf773 100644 --- a/ranger-server/src/main/java/com/flipkart/ranger/server/config/RangerConfiguration.java +++ b/ranger-server/src/main/java/com/flipkart/ranger/server/config/RangerConfiguration.java @@ -26,5 +26,4 @@ public class RangerConfiguration { private boolean disablePushUpdaters; @Min(1000) private int nodeRefreshTimeMs = RangerClientConstants.MINIMUM_REFRESH_TIME; - private List services; } diff --git a/ranger-server/src/main/java/com/flipkart/ranger/server/util/RangerServerUtils.java b/ranger-server/src/main/java/com/flipkart/ranger/server/util/RangerServerUtils.java index 80f5be14..f4033a27 100644 --- a/ranger-server/src/main/java/com/flipkart/ranger/server/util/RangerServerUtils.java +++ b/ranger-server/src/main/java/com/flipkart/ranger/server/util/RangerServerUtils.java @@ -48,7 +48,6 @@ public static UnshardedRangerZKHubClient> buildRa .curatorFramework(curatorFramework) .disablePushUpdaters(rangerConfiguration.isDisablePushUpdaters()) .mapper(mapper) - .services(rangerConfiguration.getServices()) .refreshTimeMs(rangerConfiguration.getNodeRefreshTimeMs()) .deserializer(data -> { try { From bb9852d37666498078671b3687283d86b5cc6ef7 Mon Sep 17 00:00:00 2001 From: Koushik R Date: Thu, 21 Oct 2021 22:53:25 +0530 Subject: [PATCH 15/69] Added README.md to server bundle. Renamed ranger-server to ranger-zk-server to denote that zk is the backend --- pom.xml | 4 +- ranger-server-bundle/README.md | 64 +++++++++++++++++++ .../server/bundle/RangerServerBundle.java | 8 +++ .../bundle/resources/RangerResource.java | 10 +-- .../config/local.yml | 0 {ranger-server => ranger-zk-server}/pom.xml | 2 +- .../java/com/flipkart/ranger/server/App.java | 0 .../ranger/server/AppConfiguration.java | 0 .../server/config/RangerConfiguration.java | 0 .../server/healthcheck/RangerHealthCheck.java | 0 .../server/manager/RangerBundleManager.java | 0 .../ranger/server/util/RangerServerUtils.java | 0 12 files changed, 80 insertions(+), 8 deletions(-) create mode 100644 ranger-server-bundle/README.md rename {ranger-server => ranger-zk-server}/config/local.yml (100%) rename {ranger-server => ranger-zk-server}/pom.xml (97%) rename {ranger-server => ranger-zk-server}/src/main/java/com/flipkart/ranger/server/App.java (100%) rename {ranger-server => ranger-zk-server}/src/main/java/com/flipkart/ranger/server/AppConfiguration.java (100%) rename {ranger-server => ranger-zk-server}/src/main/java/com/flipkart/ranger/server/config/RangerConfiguration.java (100%) rename {ranger-server => ranger-zk-server}/src/main/java/com/flipkart/ranger/server/healthcheck/RangerHealthCheck.java (100%) rename {ranger-server => ranger-zk-server}/src/main/java/com/flipkart/ranger/server/manager/RangerBundleManager.java (100%) rename {ranger-server => ranger-zk-server}/src/main/java/com/flipkart/ranger/server/util/RangerServerUtils.java (100%) diff --git a/pom.xml b/pom.xml index 08855dac..969e657e 100644 --- a/pom.xml +++ b/pom.xml @@ -15,10 +15,10 @@ ranger-client ranger-zk-client ranger-http-client - ranger-server + ranger-server-common ranger-server-bundle + ranger-zk-server ranger-http-server - ranger-server-common diff --git a/ranger-server-bundle/README.md b/ranger-server-bundle/README.md new file mode 100644 index 00000000..accd8594 --- /dev/null +++ b/ranger-server-bundle/README.md @@ -0,0 +1,64 @@ +# Ranger Server Bundle + +Ranger server bundle is a common dropwizard bundle atop which we could implement any http based ranger backend. +- Provides the core interface atop which any ranger http backend could be built. +- Provides type-safe generic interface for integration with the above. +- A really easy plug and play system to get started. +- Abstracts out ranger's finder hub's complexity. +- Written atop the wonderful. [Dropwizard](http://dropwizard.io/) + +#### Using a server bundle to initialize a ZK backend. + +``` + val rangerServerBundle = new RangerServerBundle, ZkNodeDataDeserializer, + AppConfiguration>() { + + @Override + protected List>> withHubs(AppConfiguration configuration) { + return Lists.newArrayList( + RangerServerUtils.buildRangerHub(curatorFramework, rangerConfiguration, environment.getObjectMapper()) + ); + } + + @Override + protected boolean withInitialRotationStatus(AppConfiguration configuration) { + return appConfiguration.isInitialRotationStatus(); + } + }; + rangerServerBundle.run(appConfiguration, environment); + + rangerServerBundle.start() +``` + +#### Using a server bundle to initialize a HTTP backend. + +``` + val rangerServerBundle = new RangerServerBundle, HTTPResponseDataDeserializer, + AppConfiguration>() { + + @Override + protected List>> withHubs(AppConfiguration configuration) { + val clientConfigs = configuration.getRangerConfiguration().getHttpClientConfigs(); + return clientConfigs.stream().map(clientConfig -> + RangerHttpServerUtils.buildRangerHub( + rangerConfiguration.getNamespace(), + rangerConfiguration.getNodeRefreshTimeMs(), + clientConfig, + environment.getObjectMapper() + )).collect(Collectors.toList()); + } + + @Override + protected boolean withInitialRotationStatus(AppConfiguration configuration) { + return appConfiguration.isInitialRotationStatus(); + } + }; + rangerServerBundle.run(appConfiguration, environment); + rangerServerBundle.start(); +``` + +Stop the bundle once you are done with it. + +``` +rangerServerBundle.stop(); +``` \ No newline at end of file diff --git a/ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/RangerServerBundle.java b/ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/RangerServerBundle.java index 93aa32e4..c017e52f 100644 --- a/ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/RangerServerBundle.java +++ b/ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/RangerServerBundle.java @@ -40,6 +40,14 @@ public abstract class RangerServerBundle< U extends Configuration > implements ConfiguredBundle{ + /** + Why are we taking a list of hubs, instead of one? To be able to aggregate from different dataSources if need be + instead of just fetching from a single dataSource. + + Please check the {@link RangerResource} for how a list is getting aggregated. + + You could also define your custom aggregation by using the {@link RangerHubClient} + */ @Getter private List> hubs; @Getter diff --git a/ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/resources/RangerResource.java b/ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/resources/RangerResource.java index bc589fa3..0b1214f7 100644 --- a/ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/resources/RangerResource.java +++ b/ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/resources/RangerResource.java @@ -49,13 +49,13 @@ public RangerResource(List> rangerHubs){ @Metered public GenericResponse> getServices() { Collection services = new ArrayList<>(); - for (RangerHubClient hub : rangerHubs) { + rangerHubs.forEach(hub -> { try { services.addAll(hub.getServices()); } catch (Exception e) { log.warn("Call to a hub failed with exception, {}", e.getMessage()); } - } + }); return GenericResponse.>builder() .success(true) .data(services) @@ -71,13 +71,13 @@ public GenericResponse>> getNodes( ){ val service = new Service(namespace, serviceName); List> serviceNodes = new ArrayList<>(); - for (RangerHubClient hub : rangerHubs) { + rangerHubs.forEach(hub -> { try { - serviceNodes.addAll(hub.getAllNodes(service, null).orElse(Collections.emptyList())); + serviceNodes.addAll(hub.getAllNodes(service).orElse(Collections.emptyList())); } catch (Exception e) { log.warn("Call to a hub failed with exception, {}", e.getMessage()); } - } + }); return GenericResponse.>>builder() .success(true) .data(serviceNodes) diff --git a/ranger-server/config/local.yml b/ranger-zk-server/config/local.yml similarity index 100% rename from ranger-server/config/local.yml rename to ranger-zk-server/config/local.yml diff --git a/ranger-server/pom.xml b/ranger-zk-server/pom.xml similarity index 97% rename from ranger-server/pom.xml rename to ranger-zk-server/pom.xml index 0f8f36ab..b5897e56 100644 --- a/ranger-server/pom.xml +++ b/ranger-zk-server/pom.xml @@ -9,7 +9,7 @@ 4.0.0 - ranger-server + ranger-zk-server 2.0.23 diff --git a/ranger-server/src/main/java/com/flipkart/ranger/server/App.java b/ranger-zk-server/src/main/java/com/flipkart/ranger/server/App.java similarity index 100% rename from ranger-server/src/main/java/com/flipkart/ranger/server/App.java rename to ranger-zk-server/src/main/java/com/flipkart/ranger/server/App.java diff --git a/ranger-server/src/main/java/com/flipkart/ranger/server/AppConfiguration.java b/ranger-zk-server/src/main/java/com/flipkart/ranger/server/AppConfiguration.java similarity index 100% rename from ranger-server/src/main/java/com/flipkart/ranger/server/AppConfiguration.java rename to ranger-zk-server/src/main/java/com/flipkart/ranger/server/AppConfiguration.java diff --git a/ranger-server/src/main/java/com/flipkart/ranger/server/config/RangerConfiguration.java b/ranger-zk-server/src/main/java/com/flipkart/ranger/server/config/RangerConfiguration.java similarity index 100% rename from ranger-server/src/main/java/com/flipkart/ranger/server/config/RangerConfiguration.java rename to ranger-zk-server/src/main/java/com/flipkart/ranger/server/config/RangerConfiguration.java diff --git a/ranger-server/src/main/java/com/flipkart/ranger/server/healthcheck/RangerHealthCheck.java b/ranger-zk-server/src/main/java/com/flipkart/ranger/server/healthcheck/RangerHealthCheck.java similarity index 100% rename from ranger-server/src/main/java/com/flipkart/ranger/server/healthcheck/RangerHealthCheck.java rename to ranger-zk-server/src/main/java/com/flipkart/ranger/server/healthcheck/RangerHealthCheck.java diff --git a/ranger-server/src/main/java/com/flipkart/ranger/server/manager/RangerBundleManager.java b/ranger-zk-server/src/main/java/com/flipkart/ranger/server/manager/RangerBundleManager.java similarity index 100% rename from ranger-server/src/main/java/com/flipkart/ranger/server/manager/RangerBundleManager.java rename to ranger-zk-server/src/main/java/com/flipkart/ranger/server/manager/RangerBundleManager.java diff --git a/ranger-server/src/main/java/com/flipkart/ranger/server/util/RangerServerUtils.java b/ranger-zk-server/src/main/java/com/flipkart/ranger/server/util/RangerServerUtils.java similarity index 100% rename from ranger-server/src/main/java/com/flipkart/ranger/server/util/RangerServerUtils.java rename to ranger-zk-server/src/main/java/com/flipkart/ranger/server/util/RangerServerUtils.java From 81fc343f197c668e1c1df8424f9108e0f9d63dc9 Mon Sep 17 00:00:00 2001 From: Koushik R Date: Tue, 26 Oct 2021 14:44:24 +0530 Subject: [PATCH 16/69] a) Fixed the deserializer in the ranger app, using the hub --- .../ranger/client/http/SimpleRangerHttpClient.java | 4 ++-- .../ranger/client/http/SimpleRangerHttpClientTest.java | 2 +- .../ranger/http/server/util/RangerHttpServerUtils.java | 3 ++- .../java/com/flipkart/ranger/common/server/ShardInfo.java | 3 +++ .../flipkart/ranger/client/zk/SimpleRangerZKClient.java | 8 ++++---- .../ranger/client/zk/SimpleRangerZKClientTest.java | 2 +- 6 files changed, 13 insertions(+), 9 deletions(-) diff --git a/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/SimpleRangerHttpClient.java b/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/SimpleRangerHttpClient.java index 8efb5b2b..edc6142a 100644 --- a/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/SimpleRangerHttpClient.java +++ b/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/SimpleRangerHttpClient.java @@ -32,7 +32,7 @@ import java.util.Optional; @Slf4j -public class SimpleRangerHttpClient, D extends HTTPResponseDataDeserializer> implements RangerClient { +public class SimpleRangerHttpClient> implements RangerClient { private final C criteria; private final SimpleUnshardedServiceFinder serviceFinder; @@ -45,7 +45,7 @@ public SimpleRangerHttpClient( int nodeRefreshIntervalMs, HttpClientConfig clientConfig, C criteria, - D deserializer + HTTPResponseDataDeserializer deserializer ) { Preconditions.checkNotNull(mapper, "Mapper can't be null"); Preconditions.checkNotNull(namespace, "namespace can't be null"); diff --git a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/SimpleRangerHttpClientTest.java b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/SimpleRangerHttpClientTest.java index c55b4971..71a02db0 100644 --- a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/SimpleRangerHttpClientTest.java +++ b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/SimpleRangerHttpClientTest.java @@ -30,7 +30,7 @@ public class SimpleRangerHttpClientTest extends BaseRangerHttpClientTest{ @Test public void testSimpleHttpRangerClient(){ - val client = SimpleRangerHttpClient., HTTPResponseDataDeserializer>builder() + val client = SimpleRangerHttpClient.>builder() .clientConfig(getHttpClientConfig()) .mapper(getObjectMapper()) .deserializer(this::read) diff --git a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/util/RangerHttpServerUtils.java b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/util/RangerHttpServerUtils.java index 7278a7fe..ecca614f 100644 --- a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/util/RangerHttpServerUtils.java +++ b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/util/RangerHttpServerUtils.java @@ -6,6 +6,7 @@ import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.http.config.HttpClientConfig; +import com.flipkart.ranger.http.model.ServiceNodesResponse; import com.flipkart.ranger.http.server.config.RangerHttpConfiguration; import com.flipkart.ranger.common.server.ShardInfo; import com.google.common.base.Preconditions; @@ -40,7 +41,7 @@ public static UnshardedRangerHttpHubClient> build .nodeRefreshIntervalMs(nodeRefreshTimeMs) .deserializer(data -> { try { - mapper.readValue(data, new TypeReference>() { + mapper.readValue(data, new TypeReference>() { }); } catch (IOException e) { log.warn("Error parsing node data with value {}", new String(data)); diff --git a/ranger-server-common/src/main/java/com/flipkart/ranger/common/server/ShardInfo.java b/ranger-server-common/src/main/java/com/flipkart/ranger/common/server/ShardInfo.java index 7d82620f..ca1ec17a 100644 --- a/ranger-server-common/src/main/java/com/flipkart/ranger/common/server/ShardInfo.java +++ b/ranger-server-common/src/main/java/com/flipkart/ranger/common/server/ShardInfo.java @@ -20,6 +20,9 @@ import java.util.Objects; +/* + An example nodeData with which we have written our ranger-servers. When you write your own servers you could define your own node data! + */ @AllArgsConstructor @NoArgsConstructor @Getter diff --git a/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/SimpleRangerZKClient.java b/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/SimpleRangerZKClient.java index bf38c933..2e16cefa 100644 --- a/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/SimpleRangerZKClient.java +++ b/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/SimpleRangerZKClient.java @@ -37,11 +37,11 @@ @Slf4j @Getter -public class SimpleRangerZKClient, D extends ZkNodeDataDeserializer> implements RangerClient { +public class SimpleRangerZKClient> implements RangerClient { private final SimpleShardedServiceFinder serviceFinder; private final C criteria; - private final D deserializer; + private final ZkNodeDataDeserializer deserializer; @Builder(builderMethodName = "fromConnectionString", builderClassName = "FromConnectionStringBuilder") public SimpleRangerZKClient( @@ -52,7 +52,7 @@ public SimpleRangerZKClient( boolean disableWatchers, String connectionString, C criteria, - D deserializer + ZkNodeDataDeserializer deserializer ){ this( namespace, @@ -75,7 +75,7 @@ public SimpleRangerZKClient( boolean disableWatchers, CuratorFramework curatorFramework, C criteria, - D deserializer + ZkNodeDataDeserializer deserializer ){ Preconditions.checkNotNull(mapper, "Mapper can't be null"); Preconditions.checkNotNull(namespace, "namespace can't be null"); diff --git a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/SimpleRangerZKClientTest.java b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/SimpleRangerZKClientTest.java index f2fa998e..df041ae1 100644 --- a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/SimpleRangerZKClientTest.java +++ b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/SimpleRangerZKClientTest.java @@ -14,7 +14,7 @@ public class SimpleRangerZKClientTest extends BaseRangerZKClientTest { @Test public void testBaseClient(){ - val client = SimpleRangerZKClient., ZkNodeDataDeserializer>fromCurator() + val client = SimpleRangerZKClient.>fromCurator() .curatorFramework(getCuratorFramework()) .deserializer(this::read) .namespace("test-n") From fed0be6de664c34af20c1a35c2f591a4c9d7111a Mon Sep 17 00:00:00 2001 From: Koushik R Date: Tue, 26 Oct 2021 14:46:36 +0530 Subject: [PATCH 17/69] Optimized unnecessary imports --- .../ranger/client/http/SimpleRangerHttpClientTest.java | 1 - .../src/main/java/com/flipkart/ranger/http/server/App.java | 2 +- .../ranger/http/server/config/RangerHttpConfiguration.java | 1 - .../ranger/http/server/util/RangerHttpServerUtils.java | 3 +-- .../ranger/http/model/ServiceDataSourceResponse.java | 1 - .../ranger/server/bundle/resources/RangerResource.java | 5 ++++- .../flipkart/ranger/client/zk/SimpleRangerZKClientTest.java | 1 - .../src/main/java/com/flipkart/ranger/server/App.java | 2 +- .../flipkart/ranger/server/config/RangerConfiguration.java | 2 -- .../com/flipkart/ranger/server/util/RangerServerUtils.java | 2 +- 10 files changed, 8 insertions(+), 12 deletions(-) diff --git a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/SimpleRangerHttpClientTest.java b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/SimpleRangerHttpClientTest.java index 71a02db0..0ca506ac 100644 --- a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/SimpleRangerHttpClientTest.java +++ b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/SimpleRangerHttpClientTest.java @@ -19,7 +19,6 @@ import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.units.TestNodeData; -import com.flipkart.ranger.http.serde.HTTPResponseDataDeserializer; import lombok.val; import org.junit.Assert; import org.junit.Test; diff --git a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/App.java b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/App.java index a548395d..6a2ad71c 100644 --- a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/App.java +++ b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/App.java @@ -2,12 +2,12 @@ import com.codahale.metrics.health.HealthCheck; import com.flipkart.ranger.client.RangerHubClient; +import com.flipkart.ranger.common.server.ShardInfo; import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.http.serde.HTTPResponseDataDeserializer; import com.flipkart.ranger.http.server.manager.RangerHttpBundleManager; import com.flipkart.ranger.http.server.util.RangerHttpServerUtils; import com.flipkart.ranger.server.bundle.RangerServerBundle; -import com.flipkart.ranger.common.server.ShardInfo; import io.dropwizard.Application; import io.dropwizard.setup.Bootstrap; import io.dropwizard.setup.Environment; diff --git a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/config/RangerHttpConfiguration.java b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/config/RangerHttpConfiguration.java index b2f1314d..c08c49ca 100644 --- a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/config/RangerHttpConfiguration.java +++ b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/config/RangerHttpConfiguration.java @@ -1,6 +1,5 @@ package com.flipkart.ranger.http.server.config; -import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.http.config.HttpClientConfig; import lombok.*; diff --git a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/util/RangerHttpServerUtils.java b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/util/RangerHttpServerUtils.java index ecca614f..5187094e 100644 --- a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/util/RangerHttpServerUtils.java +++ b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/util/RangerHttpServerUtils.java @@ -3,12 +3,11 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.flipkart.ranger.client.http.UnshardedRangerHttpHubClient; +import com.flipkart.ranger.common.server.ShardInfo; import com.flipkart.ranger.core.model.Criteria; -import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.http.config.HttpClientConfig; import com.flipkart.ranger.http.model.ServiceNodesResponse; import com.flipkart.ranger.http.server.config.RangerHttpConfiguration; -import com.flipkart.ranger.common.server.ShardInfo; import com.google.common.base.Preconditions; import lombok.extern.slf4j.Slf4j; diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceDataSourceResponse.java b/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceDataSourceResponse.java index bbe33e5b..33b378a8 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceDataSourceResponse.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceDataSourceResponse.java @@ -19,7 +19,6 @@ import com.flipkart.ranger.core.model.Service; import lombok.*; -import java.util.Collections; import java.util.List; @AllArgsConstructor diff --git a/ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/resources/RangerResource.java b/ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/resources/RangerResource.java index 0b1214f7..46c6c2f8 100644 --- a/ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/resources/RangerResource.java +++ b/ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/resources/RangerResource.java @@ -29,7 +29,10 @@ import javax.validation.constraints.NotNull; import javax.ws.rs.*; import javax.ws.rs.core.MediaType; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; @Slf4j @Consumes(MediaType.APPLICATION_JSON) diff --git a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/SimpleRangerZKClientTest.java b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/SimpleRangerZKClientTest.java index df041ae1..30f2c866 100644 --- a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/SimpleRangerZKClientTest.java +++ b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/SimpleRangerZKClientTest.java @@ -3,7 +3,6 @@ import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.units.TestNodeData; -import com.flipkart.ranger.zookeeper.serde.ZkNodeDataDeserializer; import lombok.val; import org.junit.Assert; import org.junit.Test; diff --git a/ranger-zk-server/src/main/java/com/flipkart/ranger/server/App.java b/ranger-zk-server/src/main/java/com/flipkart/ranger/server/App.java index c8d786e7..50562b0c 100644 --- a/ranger-zk-server/src/main/java/com/flipkart/ranger/server/App.java +++ b/ranger-zk-server/src/main/java/com/flipkart/ranger/server/App.java @@ -16,11 +16,11 @@ package com.flipkart.ranger.server; import com.flipkart.ranger.client.RangerHubClient; +import com.flipkart.ranger.common.server.ShardInfo; import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.server.bundle.RangerServerBundle; import com.flipkart.ranger.server.healthcheck.RangerHealthCheck; import com.flipkart.ranger.server.manager.RangerBundleManager; -import com.flipkart.ranger.common.server.ShardInfo; import com.flipkart.ranger.server.util.RangerServerUtils; import com.flipkart.ranger.zookeeper.serde.ZkNodeDataDeserializer; import com.google.common.collect.Lists; diff --git a/ranger-zk-server/src/main/java/com/flipkart/ranger/server/config/RangerConfiguration.java b/ranger-zk-server/src/main/java/com/flipkart/ranger/server/config/RangerConfiguration.java index 252bf773..decba526 100644 --- a/ranger-zk-server/src/main/java/com/flipkart/ranger/server/config/RangerConfiguration.java +++ b/ranger-zk-server/src/main/java/com/flipkart/ranger/server/config/RangerConfiguration.java @@ -2,13 +2,11 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.flipkart.ranger.client.RangerClientConstants; -import com.flipkart.ranger.core.model.Service; import lombok.*; import javax.validation.constraints.Min; import javax.validation.constraints.NotEmpty; import javax.validation.constraints.NotNull; -import java.util.List; @Getter @Setter diff --git a/ranger-zk-server/src/main/java/com/flipkart/ranger/server/util/RangerServerUtils.java b/ranger-zk-server/src/main/java/com/flipkart/ranger/server/util/RangerServerUtils.java index f4033a27..57c0ef78 100644 --- a/ranger-zk-server/src/main/java/com/flipkart/ranger/server/util/RangerServerUtils.java +++ b/ranger-zk-server/src/main/java/com/flipkart/ranger/server/util/RangerServerUtils.java @@ -4,10 +4,10 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.flipkart.ranger.client.RangerClientConstants; import com.flipkart.ranger.client.zk.UnshardedRangerZKHubClient; +import com.flipkart.ranger.common.server.ShardInfo; import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.server.config.RangerConfiguration; -import com.flipkart.ranger.common.server.ShardInfo; import com.google.common.base.Preconditions; import lombok.extern.slf4j.Slf4j; import org.apache.curator.framework.CuratorFramework; From fb694f58ba80ad643d55d4593571b271eb9c461f Mon Sep 17 00:00:00 2001 From: Koushik R Date: Tue, 9 Nov 2021 21:31:21 +0530 Subject: [PATCH 18/69] a. Added missing licenses b) Fixed minor formatting issues c) Fixed the error logging in ZkNodeDataSource and removed the unnecessary throw illegalState --- pom.xml | 2 +- .../com/flipkart/ranger/client/RangerClient.java | 7 +++---- .../flipkart/ranger/client/RangerHubClient.java | 16 +++++++--------- .../ranger/client/stubs/RangerTestHub.java | 15 +++++++++++++++ .../ranger/client/stubs/TestCriteria.java | 15 +++++++++++++++ .../ranger/client/stubs/TestDeserializer.java | 15 +++++++++++++++ .../client/stubs/TestServiceFinderFactory.java | 15 +++++++++++++++ .../ranger/client/stubs/TestShardInfo.java | 15 +++++++++++++++ .../stubs/TestSimpleUnshardedServiceFinder.java | 15 +++++++++++++++ .../ranger/client/utils/RangerHubTestUtils.java | 15 +++++++++++++++ .../com/flipkart/ranger/http/server/App.java | 15 +++++++++++++++ .../ranger/http/server/AppConfiguration.java | 15 +++++++++++++++ .../server/config/RangerHttpConfiguration.java | 15 +++++++++++++++ .../server/manager/RangerHttpBundleManager.java | 15 +++++++++++++++ .../http/server/util/RangerHttpServerUtils.java | 15 +++++++++++++++ .../ranger/client/zk/BaseRangerZKClientTest.java | 15 +++++++++++++++ .../client/zk/ShardedZKRangerClientTest.java | 15 +++++++++++++++ .../client/zk/SimpleRangerZKClientTest.java | 15 +++++++++++++++ .../client/zk/UnshardedZKRangerClientTest.java | 15 +++++++++++++++ .../server/config/RangerConfiguration.java | 15 +++++++++++++++ .../ranger/server/util/RangerServerUtils.java | 15 +++++++++++++++ .../servicefinder/ZkNodeDataSource.java | 1 - 22 files changed, 281 insertions(+), 15 deletions(-) diff --git a/pom.xml b/pom.xml index 969e657e..de45d82f 100644 --- a/pom.xml +++ b/pom.xml @@ -72,7 +72,7 @@ 3.8.0 1.8 - 1.18.16 + 1.18.20 4.13.2 4.1.0 diff --git a/ranger-client/src/main/java/com/flipkart/ranger/client/RangerClient.java b/ranger-client/src/main/java/com/flipkart/ranger/client/RangerClient.java index ac2d99e3..86fe20a1 100644 --- a/ranger-client/src/main/java/com/flipkart/ranger/client/RangerClient.java +++ b/ranger-client/src/main/java/com/flipkart/ranger/client/RangerClient.java @@ -1,19 +1,18 @@ /* * Copyright 2015 Flipkart Internet Pvt. Ltd. - *

+ * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

+ * * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ - package com.flipkart.ranger.client; import com.flipkart.ranger.core.model.Criteria; diff --git a/ranger-client/src/main/java/com/flipkart/ranger/client/RangerHubClient.java b/ranger-client/src/main/java/com/flipkart/ranger/client/RangerHubClient.java index f95b0f9c..9c6badcc 100644 --- a/ranger-client/src/main/java/com/flipkart/ranger/client/RangerHubClient.java +++ b/ranger-client/src/main/java/com/flipkart/ranger/client/RangerHubClient.java @@ -24,19 +24,17 @@ import java.util.Optional; public interface RangerHubClient> { + void start(); - void start(); + void stop(); - void stop(); + Collection getServices() throws Exception; - Collection getServices() throws Exception; + Optional> getNode(final Service service); - Optional> getNode(final Service service); + Optional> getNode(final Service service, final C criteria); - Optional> getNode(final Service service, final C criteria); - - Optional>> getAllNodes(final Service service); - - Optional>> getAllNodes(final Service service, final C criteria); + Optional>> getAllNodes(final Service service); + Optional>> getAllNodes(final Service service, final C criteria); } diff --git a/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/RangerTestHub.java b/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/RangerTestHub.java index e7e62e88..367aa766 100644 --- a/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/RangerTestHub.java +++ b/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/RangerTestHub.java @@ -1,3 +1,18 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.flipkart.ranger.client.stubs; import com.fasterxml.jackson.databind.ObjectMapper; diff --git a/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestCriteria.java b/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestCriteria.java index 4071b655..3f7f7497 100644 --- a/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestCriteria.java +++ b/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestCriteria.java @@ -1,3 +1,18 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.flipkart.ranger.client.stubs; import com.flipkart.ranger.core.model.Criteria; diff --git a/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestDeserializer.java b/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestDeserializer.java index 72d6bc98..9310f24e 100644 --- a/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestDeserializer.java +++ b/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestDeserializer.java @@ -1,3 +1,18 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.flipkart.ranger.client.stubs; import com.flipkart.ranger.core.model.Deserializer; diff --git a/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestServiceFinderFactory.java b/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestServiceFinderFactory.java index bbbe8b9a..c3242d95 100644 --- a/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestServiceFinderFactory.java +++ b/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestServiceFinderFactory.java @@ -1,3 +1,18 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.flipkart.ranger.client.stubs; import com.flipkart.ranger.core.finder.ServiceFinder; diff --git a/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestShardInfo.java b/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestShardInfo.java index 9f26e566..10e74007 100644 --- a/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestShardInfo.java +++ b/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestShardInfo.java @@ -1,3 +1,18 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.flipkart.ranger.client.stubs; import lombok.AllArgsConstructor; diff --git a/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestSimpleUnshardedServiceFinder.java b/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestSimpleUnshardedServiceFinder.java index 570975ad..71764674 100644 --- a/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestSimpleUnshardedServiceFinder.java +++ b/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestSimpleUnshardedServiceFinder.java @@ -1,3 +1,18 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.flipkart.ranger.client.stubs; import com.flipkart.ranger.core.finder.SimpleUnshardedServiceFinder; diff --git a/ranger-client/src/test/java/com/flipkart/ranger/client/utils/RangerHubTestUtils.java b/ranger-client/src/test/java/com/flipkart/ranger/client/utils/RangerHubTestUtils.java index 76ada370..affd5dad 100644 --- a/ranger-client/src/test/java/com/flipkart/ranger/client/utils/RangerHubTestUtils.java +++ b/ranger-client/src/test/java/com/flipkart/ranger/client/utils/RangerHubTestUtils.java @@ -1,3 +1,18 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.flipkart.ranger.client.utils; import com.fasterxml.jackson.databind.ObjectMapper; diff --git a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/App.java b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/App.java index 6a2ad71c..0dadbc08 100644 --- a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/App.java +++ b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/App.java @@ -1,3 +1,18 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.flipkart.ranger.http.server; import com.codahale.metrics.health.HealthCheck; diff --git a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/AppConfiguration.java b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/AppConfiguration.java index 372a2ba9..2b93198e 100644 --- a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/AppConfiguration.java +++ b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/AppConfiguration.java @@ -1,3 +1,18 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.flipkart.ranger.http.server; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; diff --git a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/config/RangerHttpConfiguration.java b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/config/RangerHttpConfiguration.java index c08c49ca..0792b871 100644 --- a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/config/RangerHttpConfiguration.java +++ b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/config/RangerHttpConfiguration.java @@ -1,3 +1,18 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.flipkart.ranger.http.server.config; import com.flipkart.ranger.http.config.HttpClientConfig; diff --git a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/manager/RangerHttpBundleManager.java b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/manager/RangerHttpBundleManager.java index c1e0d92b..58fd127f 100644 --- a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/manager/RangerHttpBundleManager.java +++ b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/manager/RangerHttpBundleManager.java @@ -1,3 +1,18 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.flipkart.ranger.http.server.manager; import com.flipkart.ranger.server.bundle.RangerServerBundle; diff --git a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/util/RangerHttpServerUtils.java b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/util/RangerHttpServerUtils.java index 5187094e..64f14164 100644 --- a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/util/RangerHttpServerUtils.java +++ b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/util/RangerHttpServerUtils.java @@ -1,3 +1,18 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.flipkart.ranger.http.server.util; import com.fasterxml.jackson.core.type.TypeReference; diff --git a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/BaseRangerZKClientTest.java b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/BaseRangerZKClientTest.java index 40b74f08..23534529 100644 --- a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/BaseRangerZKClientTest.java +++ b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/BaseRangerZKClientTest.java @@ -1,3 +1,18 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.flipkart.ranger.client.zk; import com.fasterxml.jackson.core.type.TypeReference; diff --git a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/ShardedZKRangerClientTest.java b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/ShardedZKRangerClientTest.java index a67fda0c..135c9632 100644 --- a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/ShardedZKRangerClientTest.java +++ b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/ShardedZKRangerClientTest.java @@ -1,3 +1,18 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.flipkart.ranger.client.zk; import com.flipkart.ranger.core.TestUtils; diff --git a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/SimpleRangerZKClientTest.java b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/SimpleRangerZKClientTest.java index 30f2c866..7ef07936 100644 --- a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/SimpleRangerZKClientTest.java +++ b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/SimpleRangerZKClientTest.java @@ -1,3 +1,18 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.flipkart.ranger.client.zk; import com.flipkart.ranger.core.model.Criteria; diff --git a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/UnshardedZKRangerClientTest.java b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/UnshardedZKRangerClientTest.java index 5dfe5e56..e29150bb 100644 --- a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/UnshardedZKRangerClientTest.java +++ b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/UnshardedZKRangerClientTest.java @@ -1,3 +1,18 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.flipkart.ranger.client.zk; import com.flipkart.ranger.core.TestUtils; diff --git a/ranger-zk-server/src/main/java/com/flipkart/ranger/server/config/RangerConfiguration.java b/ranger-zk-server/src/main/java/com/flipkart/ranger/server/config/RangerConfiguration.java index decba526..d5aa6fa3 100644 --- a/ranger-zk-server/src/main/java/com/flipkart/ranger/server/config/RangerConfiguration.java +++ b/ranger-zk-server/src/main/java/com/flipkart/ranger/server/config/RangerConfiguration.java @@ -1,3 +1,18 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.flipkart.ranger.server.config; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; diff --git a/ranger-zk-server/src/main/java/com/flipkart/ranger/server/util/RangerServerUtils.java b/ranger-zk-server/src/main/java/com/flipkart/ranger/server/util/RangerServerUtils.java index 57c0ef78..95f1540f 100644 --- a/ranger-zk-server/src/main/java/com/flipkart/ranger/server/util/RangerServerUtils.java +++ b/ranger-zk-server/src/main/java/com/flipkart/ranger/server/util/RangerServerUtils.java @@ -1,3 +1,18 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package com.flipkart.ranger.server.util; import com.fasterxml.jackson.core.type.TypeReference; diff --git a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinder/ZkNodeDataSource.java b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinder/ZkNodeDataSource.java index b71780f7..2dd34fac 100644 --- a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinder/ZkNodeDataSource.java +++ b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinder/ZkNodeDataSource.java @@ -88,7 +88,6 @@ private Optional>> checkForUpdateOnZookeeper(D deserializer) } catch (Exception e) { log.error("Error getting service data from zookeeper: ", e); - Exceptions.illegalState(e); } return Optional.empty(); } From beedfddbf805770ae486cdb1dbb3f5c2fae353ed Mon Sep 17 00:00:00 2001 From: Koushik R Date: Tue, 16 Nov 2021 12:28:12 +0530 Subject: [PATCH 19/69] Minor formatting issues fixed --- .../finder/SimpleUnshardedServiceFinder.java | 4 ++-- .../finderhub/ServiceFinderHubBuilder.java | 18 +++++++++--------- .../flipkart/ranger/core/model/Service.java | 4 ++-- .../BaseServiceProviderBuilder.java | 2 +- .../HttpServiceFinderHubBuilder.java | 2 +- .../client/zk/BaseRangerZKClientTest.java | 2 +- .../ZkServiceFinderHubBuilder.java | 8 ++++---- .../zookeeper/servicehub/ServiceHubTest.java | 2 +- 8 files changed, 21 insertions(+), 21 deletions(-) diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/SimpleUnshardedServiceFinder.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/SimpleUnshardedServiceFinder.java index 783a7a1c..cdf3583f 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/SimpleUnshardedServiceFinder.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/SimpleUnshardedServiceFinder.java @@ -23,8 +23,8 @@ public class SimpleUnshardedServiceFinder> extends ServiceFinder> { public SimpleUnshardedServiceFinder(ListBasedServiceRegistry serviceRegistry, - ShardSelector> shardSelector, - ServiceNodeSelector nodeSelector) { + ShardSelector> shardSelector, + ServiceNodeSelector nodeSelector) { super(serviceRegistry, shardSelector, nodeSelector); } } diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/ServiceFinderHubBuilder.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/ServiceFinderHubBuilder.java index 1b883e3b..02761c16 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/ServiceFinderHubBuilder.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/ServiceFinderHubBuilder.java @@ -32,43 +32,43 @@ */ public abstract class ServiceFinderHubBuilder, R extends ServiceRegistry> { private ServiceDataSource serviceDataSource; - private ServiceFinderFactory serviceFinderFactory; + private ServiceFinderFactory serviceFinderFactory; private long refreshFrequencyMs = 10_000; private final List> extraStartSignalConsumers = new ArrayList<>(); private final List> extraStopSignalConsumers = new ArrayList<>(); private final List> extraRefreshSignals = new ArrayList<>(); - public ServiceFinderHubBuilder withServiceDataSource(ServiceDataSource serviceDataSource) { + public ServiceFinderHubBuilder withServiceDataSource(ServiceDataSource serviceDataSource) { this.serviceDataSource = serviceDataSource; return this; } - public ServiceFinderHubBuilder withServiceFinderFactory(ServiceFinderFactory serviceFinderFactory) { + public ServiceFinderHubBuilder withServiceFinderFactory(ServiceFinderFactory serviceFinderFactory) { this.serviceFinderFactory = serviceFinderFactory; return this; } - public ServiceFinderHubBuilder withRefreshFrequencyMs(long refreshFrequencyMs) { + public ServiceFinderHubBuilder withRefreshFrequencyMs(long refreshFrequencyMs) { this.refreshFrequencyMs = refreshFrequencyMs; return this; } - public ServiceFinderHubBuilder withExtraStartSignalConsumer(Consumer consumer) { + public ServiceFinderHubBuilder withExtraStartSignalConsumer(Consumer consumer) { this.extraStartSignalConsumers.add(consumer); return this; } - public ServiceFinderHubBuilder withExtraStopSignalConsumer(Consumer consumer) { + public ServiceFinderHubBuilder withExtraStopSignalConsumer(Consumer consumer) { this.extraStopSignalConsumers.add(consumer); return this; } - public ServiceFinderHubBuilder withExtraRefreshSignal(Signal extraRefreshSignal) { + public ServiceFinderHubBuilder withExtraRefreshSignal(Signal extraRefreshSignal) { this.extraRefreshSignals.add(extraRefreshSignal); return this; } - public ServiceFinderHub build() { + public ServiceFinderHub build() { preBuild(); Preconditions.checkNotNull(serviceDataSource, "Provide a non-null service data source"); Preconditions.checkNotNull(serviceFinderFactory, "Provide a non-null service finder factory"); @@ -95,5 +95,5 @@ public ServiceFinderHub build() { protected abstract void preBuild(); - protected abstract void postBuild(ServiceFinderHub serviceFinderHub); + protected abstract void postBuild(ServiceFinderHub serviceFinderHub); } diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/model/Service.java b/ranger-core/src/main/java/com/flipkart/ranger/core/model/Service.java index a9f77786..a13e9942 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/model/Service.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/model/Service.java @@ -18,15 +18,15 @@ import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Data; +import lombok.NoArgsConstructor; @Data @AllArgsConstructor(access = AccessLevel.PUBLIC) +@NoArgsConstructor public class Service { private String namespace; private String serviceName; - public Service(){} - public String name() { return String.format("%s/%s", namespace, serviceName); } diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/serviceprovider/BaseServiceProviderBuilder.java b/ranger-core/src/main/java/com/flipkart/ranger/core/serviceprovider/BaseServiceProviderBuilder.java index ff6c0597..08f6a207 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/serviceprovider/BaseServiceProviderBuilder.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/serviceprovider/BaseServiceProviderBuilder.java @@ -144,7 +144,7 @@ public B withStopSignalHandlers(List> stopSignalHandlers) { return (B)this; } - public B withadditionalRefreshSignal(Signal additionalRefreshSignal) { + public B withAdditionalRefreshSignal(Signal additionalRefreshSignal) { this.additionalRefreshSignals.add(additionalRefreshSignal); return (B)this; } diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceFinderHubBuilder.java b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceFinderHubBuilder.java index 2c100288..96ccf5c4 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceFinderHubBuilder.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceFinderHubBuilder.java @@ -30,7 +30,7 @@ protected void preBuild() { } @Override - protected void postBuild(ServiceFinderHub serviceFinderHub) { + protected void postBuild(ServiceFinderHub serviceFinderHub) { log.info("No post build actions necessary"); } } diff --git a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/BaseRangerZKClientTest.java b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/BaseRangerZKClientTest.java index 23534529..b670c1b1 100644 --- a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/BaseRangerZKClientTest.java +++ b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/BaseRangerZKClientTest.java @@ -110,7 +110,7 @@ protected void initilizeProvider(){ .withSerializer(this::write) .withNodeData(new TestNodeData(1)) .withHealthcheck(() -> HealthcheckStatus.healthy) - .withadditionalRefreshSignal(refreshProviderSignal) + .withAdditionalRefreshSignal(refreshProviderSignal) .withCuratorFramework(getCuratorFramework()) .build(); provider.start(); diff --git a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZkServiceFinderHubBuilder.java b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZkServiceFinderHubBuilder.java index aead107e..e0fcc11d 100644 --- a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZkServiceFinderHubBuilder.java +++ b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZkServiceFinderHubBuilder.java @@ -34,17 +34,17 @@ public class ZkServiceFinderHubBuilder, R extends Servi private CuratorFramework curatorFramework; private String connectionString; - public ZkServiceFinderHubBuilder withNamespace(final String namespace) { + public ZkServiceFinderHubBuilder withNamespace(final String namespace) { this.namespace = namespace; return this; } - public ZkServiceFinderHubBuilder withCuratorFramework(CuratorFramework curatorFramework) { + public ZkServiceFinderHubBuilder withCuratorFramework(CuratorFramework curatorFramework) { this.curatorFramework = curatorFramework; return this; } - public ZkServiceFinderHubBuilder withConnectionString(final String connectionString) { + public ZkServiceFinderHubBuilder withConnectionString(final String connectionString) { this.connectionString = connectionString; return this; } @@ -65,7 +65,7 @@ protected void preBuild() { } @Override - protected void postBuild(ServiceFinderHub serviceFinderHub) { + protected void postBuild(ServiceFinderHub serviceFinderHub) { log.debug("No post build steps necessary"); } } diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/servicehub/ServiceHubTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/servicehub/ServiceHubTest.java index e563d2e2..e1731382 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/servicehub/ServiceHubTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/servicehub/ServiceHubTest.java @@ -112,7 +112,7 @@ public void testHub() { .withSerializer(this::write) .withNodeData(new TestShardInfo("prod")) .withHealthcheck(() -> HealthcheckStatus.healthy) - .withadditionalRefreshSignal(refreshProviderSignal) + .withAdditionalRefreshSignal(refreshProviderSignal) .withCuratorFramework(curatorFramework) .build(); provider1.start(); From 0847a28997aa672b5fb9e838797fbc4e8ee251a2 Mon Sep 17 00:00:00 2001 From: Koushik R Date: Wed, 17 Nov 2021 23:10:04 +0530 Subject: [PATCH 20/69] Minor formatting issues fixed w.r.t dual licence on RotationStatusMonitor --- .../monitor/sample/RotationStatusMonitor.java | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/sample/RotationStatusMonitor.java b/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/sample/RotationStatusMonitor.java index 655eadc0..b0bc1c48 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/sample/RotationStatusMonitor.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/sample/RotationStatusMonitor.java @@ -16,21 +16,6 @@ package com.flipkart.ranger.core.healthservice.monitor.sample; -/** - * Copyright 2015 Flipkart Internet Pvt. Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ import com.flipkart.ranger.core.healthcheck.HealthcheckStatus; import com.flipkart.ranger.core.healthservice.TimeEntity; import com.flipkart.ranger.core.healthservice.monitor.IsolatedHealthMonitor; From 826c456e7e0b9864f39543cea31dde01f1f933b5 Mon Sep 17 00:00:00 2001 From: Koushik R Date: Fri, 10 Dec 2021 00:42:59 +0530 Subject: [PATCH 21/69] java doc for RangerBundleManager if there would be confusion for why .run is called --- .../main/java/com/flipkart/ranger/server/App.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/ranger-zk-server/src/main/java/com/flipkart/ranger/server/App.java b/ranger-zk-server/src/main/java/com/flipkart/ranger/server/App.java index 50562b0c..b24747db 100644 --- a/ranger-zk-server/src/main/java/com/flipkart/ranger/server/App.java +++ b/ranger-zk-server/src/main/java/com/flipkart/ranger/server/App.java @@ -44,6 +44,18 @@ public void initialize(Bootstrap bootstrap) { } + /** + * The reason why rangerServerBundle.run is used here instead of initializing in the initialize method and letting run automatically called. + * + * a) RangerServerBundle's closure contains a rangerHubClient within it, now this could client could either be a ZK client or the httpClient. + * b) A ZK client would need the curatorFramework and the http client doesn't need anything. + * c) So the rangerServerBundle couldn't contain a curatorFramework and that has to be defined outside, for that we need the AppConfiguration, which + * is available in the run as an arg to be used, + * d) Also don't want to create two bundle managers (one for http and one for ZK) + * + * @param appConfiguration {@link AppConfiguration} The appConfiguration provided + * @param environment {@link Environment} The environment in scope. + */ @Override public void run(AppConfiguration appConfiguration, Environment environment) { val rangerConfiguration = appConfiguration.getRangerConfiguration(); From 32ea6d211e8a140d75fbded69a40a60445b3ba1d Mon Sep 17 00:00:00 2001 From: Koushik R Date: Fri, 10 Dec 2021 00:43:53 +0530 Subject: [PATCH 22/69] Typo in the javadoc fixed --- .../src/main/java/com/flipkart/ranger/server/App.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ranger-zk-server/src/main/java/com/flipkart/ranger/server/App.java b/ranger-zk-server/src/main/java/com/flipkart/ranger/server/App.java index b24747db..8b3d483d 100644 --- a/ranger-zk-server/src/main/java/com/flipkart/ranger/server/App.java +++ b/ranger-zk-server/src/main/java/com/flipkart/ranger/server/App.java @@ -47,9 +47,9 @@ public void initialize(Bootstrap bootstrap) { /** * The reason why rangerServerBundle.run is used here instead of initializing in the initialize method and letting run automatically called. * - * a) RangerServerBundle's closure contains a rangerHubClient within it, now this could client could either be a ZK client or the httpClient. + * a) RangerServerBundle's closure contains a rangerHubClient within it, now this client could either be a ZK client or the httpClient. * b) A ZK client would need the curatorFramework and the http client doesn't need anything. - * c) So the rangerServerBundle couldn't contain a curatorFramework and that has to be defined outside, for that we need the AppConfiguration, which + * c) So the RangerServerBundle couldn't contain a curatorFramework and that has to be defined outside, for that we need the AppConfiguration, which * is available in the run as an arg to be used, * d) Also don't want to create two bundle managers (one for http and one for ZK) * From 62f306b325a38fe9eb7b92612d862f88cc7555ff Mon Sep 17 00:00:00 2001 From: Koushik R Date: Fri, 10 Dec 2021 00:46:15 +0530 Subject: [PATCH 23/69] javadoc added for http App as well for the .run method on init --- .../java/com/flipkart/ranger/http/server/App.java | 14 ++++++++++++++ .../main/java/com/flipkart/ranger/server/App.java | 2 ++ 2 files changed, 16 insertions(+) diff --git a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/App.java b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/App.java index 0dadbc08..19a482df 100644 --- a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/App.java +++ b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/App.java @@ -42,6 +42,20 @@ public void initialize(Bootstrap bootstrap) { } + /** + * The reason why rangerServerBundle.run is used here instead of initializing in the initialize method and letting run automatically called. + * + * a) RangerServerBundle's closure contains a rangerHubClient within it, now this client could either be a ZK client or the httpClient. + * b) A ZK client would need the curatorFramework and the http client doesn't need anything. + * c) So the RangerServerBundle couldn't contain a curatorFramework and that has to be defined outside, for that we need the AppConfiguration, which + * is available in the run as an arg to be used, + * d) Also don't want to create two bundle managers (one for http and one for ZK) + * + * Also, the verifyPreconditions is different for both zk and http. + * + * @param appConfiguration {@link AppConfiguration} The appConfiguration provided + * @param environment {@link Environment} The environment in scope. + */ @Override public void run(AppConfiguration appConfiguration, Environment environment) { val rangerConfiguration = appConfiguration.getRangerConfiguration(); diff --git a/ranger-zk-server/src/main/java/com/flipkart/ranger/server/App.java b/ranger-zk-server/src/main/java/com/flipkart/ranger/server/App.java index 8b3d483d..98e0c457 100644 --- a/ranger-zk-server/src/main/java/com/flipkart/ranger/server/App.java +++ b/ranger-zk-server/src/main/java/com/flipkart/ranger/server/App.java @@ -53,6 +53,8 @@ public void initialize(Bootstrap bootstrap) { * is available in the run as an arg to be used, * d) Also don't want to create two bundle managers (one for http and one for ZK) * + * Also, the verifyPreconditions is different for both zk and http. + * * @param appConfiguration {@link AppConfiguration} The appConfiguration provided * @param environment {@link Environment} The environment in scope. */ From 964932ac7b4af4c58c086fa33919d84614956ca6 Mon Sep 17 00:00:00 2001 From: Koushik R Date: Mon, 13 Dec 2021 17:41:00 +0530 Subject: [PATCH 24/69] An incorrect exception from httpNodeDataSource was being thrown in case of failures. Fixed the same --- .../flipkart/ranger/http/servicefinder/HttpNodeDataSource.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpNodeDataSource.java b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpNodeDataSource.java index 09e25700..507c7a0c 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpNodeDataSource.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpNodeDataSource.java @@ -94,7 +94,7 @@ public Optional>> refresh(D deserializer) { log.warn("HTTP call to {} returned: {}", httpUrl, response.code()); } } catch (IOException e) { - Exceptions.illegalState("Error fetching data from server: " + httpUrl, e); + log.error("Error getting service data from the http endPoint: ", e); } return Optional.of(serviceNodes); } From b17b6359c2551b6e01aec1a0b804cacddc50f3fc Mon Sep 17 00:00:00 2001 From: Koushik R Date: Mon, 13 Dec 2021 17:47:51 +0530 Subject: [PATCH 25/69] There was an unnecessary deserializer used in RangerServerBundle, removed the same --- .../src/main/java/com/flipkart/ranger/http/server/App.java | 5 ++--- .../flipkart/ranger/server/bundle/RangerServerBundle.java | 4 +--- .../ranger/server/bundle/RangerServerBundleTest.java | 4 ++-- .../src/main/java/com/flipkart/ranger/server/App.java | 2 +- 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/App.java b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/App.java index 19a482df..009ebe4d 100644 --- a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/App.java +++ b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/App.java @@ -61,13 +61,12 @@ public void run(AppConfiguration appConfiguration, Environment environment) { val rangerConfiguration = appConfiguration.getRangerConfiguration(); RangerHttpServerUtils.verifyPreconditions(rangerConfiguration); - val rangerServerBundle = new RangerServerBundle, HTTPResponseDataDeserializer, + val rangerServerBundle = new RangerServerBundle, AppConfiguration>() { @Override protected List>> withHubs(AppConfiguration configuration) { - val clientConfigs = configuration.getRangerConfiguration().getHttpClientConfigs(); - return clientConfigs.stream().map(clientConfig -> + return configuration.getRangerConfiguration().getHttpClientConfigs().stream().map(clientConfig -> RangerHttpServerUtils.buildRangerHub( rangerConfiguration.getNamespace(), rangerConfiguration.getNodeRefreshTimeMs(), diff --git a/ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/RangerServerBundle.java b/ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/RangerServerBundle.java index c017e52f..dcdf8b12 100644 --- a/ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/RangerServerBundle.java +++ b/ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/RangerServerBundle.java @@ -36,9 +36,7 @@ public abstract class RangerServerBundle< T, C extends Criteria, - D extends Deserializer, - U extends Configuration - > implements ConfiguredBundle{ + U extends Configuration> implements ConfiguredBundle{ /** Why are we taking a list of hubs, instead of one? To be able to aggregate from different dataSources if need be diff --git a/ranger-server-bundle/src/test/java/com/flipkart/ranger/server/bundle/RangerServerBundleTest.java b/ranger-server-bundle/src/test/java/com/flipkart/ranger/server/bundle/RangerServerBundleTest.java index 4ab66c26..fb9355fa 100644 --- a/ranger-server-bundle/src/test/java/com/flipkart/ranger/server/bundle/RangerServerBundleTest.java +++ b/ranger-server-bundle/src/test/java/com/flipkart/ranger/server/bundle/RangerServerBundleTest.java @@ -41,8 +41,8 @@ public class RangerServerBundleTest { private final Bootstrap bootstrap = mock(Bootstrap.class); private final Configuration configuration = mock(Configuration.class); - private final RangerServerBundle, TestDeserializer, Configuration> - rangerServerBundle = new RangerServerBundle, TestDeserializer, Configuration>() { + private final RangerServerBundle, Configuration> + rangerServerBundle = new RangerServerBundle, Configuration>() { @Override protected List>> withHubs(Configuration configuration) { diff --git a/ranger-zk-server/src/main/java/com/flipkart/ranger/server/App.java b/ranger-zk-server/src/main/java/com/flipkart/ranger/server/App.java index 98e0c457..c81a8e60 100644 --- a/ranger-zk-server/src/main/java/com/flipkart/ranger/server/App.java +++ b/ranger-zk-server/src/main/java/com/flipkart/ranger/server/App.java @@ -64,7 +64,7 @@ public void run(AppConfiguration appConfiguration, Environment environment) { RangerServerUtils.verifyPreconditions(rangerConfiguration); val curatorFramework = RangerServerUtils.buildCurator(rangerConfiguration); - val rangerServerBundle = new RangerServerBundle, ZkNodeDataDeserializer, + val rangerServerBundle = new RangerServerBundle, AppConfiguration>() { @Override From 7aaf5fe3249cd4cd0df1e33664fca8da442d8061 Mon Sep 17 00:00:00 2001 From: Koushik R Date: Fri, 17 Dec 2021 11:43:06 +0530 Subject: [PATCH 26/69] a) Moved criteria to predicates b) Replaced lombak-able refrences with lombok c) Minor refactoring in ranger client --- .../client/AbstractRangerHubClient.java | 29 +++-- .../flipkart/ranger/client/RangerClient.java | 10 +- .../ranger/client/RangerClientConstants.java | 6 +- .../ranger/client/RangerHubClient.java | 10 +- .../client/AbstractRangerHubClientTest.java | 20 +-- .../ranger/client/stubs/RangerTestHub.java | 20 +-- .../ranger/client/stubs/TestCriteria.java | 8 +- .../stubs/TestServiceFinderFactory.java | 12 +- .../ranger/client/stubs/TestShardInfo.java | 27 ---- .../TestSimpleUnshardedServiceFinder.java | 62 +++++----- .../client/utils/RangerHubTestUtils.java | 15 ++- .../core/finder/BaseServiceFinderBuilder.java | 15 ++- .../ranger/core/finder/ServiceFinder.java | 16 ++- .../finder/SimpleShardedServiceFinder.java | 5 +- .../SimpleShardedServiceFinderBuilder.java | 16 +-- .../finder/SimpleUnshardedServiceFinder.java | 5 +- .../SimpleUnshardedServiceFinderBuilder.java | 16 +-- .../ListBasedServiceRegistry.java | 5 +- .../MapBasedServiceRegistry.java | 5 +- .../ServiceRegistryUpdater.java | 6 +- .../shardselector/ListShardSelector.java | 15 +-- .../shardselector/MatchingShardSelector.java | 8 +- .../shardselector/NoopShardSelector.java | 7 +- .../core/finderhub/ServiceFinderFactory.java | 5 +- .../core/finderhub/ServiceFinderHub.java | 15 ++- .../finderhub/ServiceFinderHubBuilder.java | 21 ++-- .../core/finderhub/StaticDataSource.java | 3 +- .../ranger/core/model/NodeDataSource.java | 4 +- .../flipkart/ranger/core/model/Service.java | 16 ++- .../ranger/core/model/ShardSelector.java | 5 +- .../BaseServiceProviderBuilder.java | 23 ++-- .../flipkart/ranger/core/util/Exceptions.java | 4 +- .../ranger/core/util/FinderUtils.java | 4 +- .../core/finder/SimpleShardFinderTest.java | 16 +-- .../finder/UnshardedClusterFinderTest.java | 13 +- .../RoundRobinServiceNodeSelectorTest.java | 14 +-- .../MapBasedServiceRegistryTest.java | 9 +- .../shardselector/ListShardSelectorTest.java | 14 +-- .../MatchingShardSelectorTest.java | 22 +--- .../serviceprovider/ServiceProviderTest.java | 7 +- .../ranger/core/units/TestNodeData.java | 30 +---- .../ranger/core/utils/CriteriaUtils.java | 26 ---- .../ranger/core/utils/RangerTestUtils.java | 44 +++++++ .../ranger/core/utils/RegistryTestUtils.java | 24 ++-- .../ranger/core/{ => utils}/TestUtils.java | 6 +- .../http/ShardedRangerHttpHubClient.java | 22 ++-- .../client/http/SimpleRangerHttpClient.java | 20 +-- .../http/UnshardedRangerHttpHubClient.java | 16 +-- .../client/http/BaseRangerHttpClientTest.java | 8 +- .../http/ShardedRangerHttpClientTest.java | 13 +- .../http/SimpleRangerHttpClientTest.java | 9 +- .../http/UnshardedRangerHttpClientTest.java | 13 +- .../com/flipkart/ranger/http/server/App.java | 66 +--------- .../ranger/http/server/AppConfiguration.java | 22 ++-- .../http/server/bundle/HttpServerBundle.java | 97 +++++++++++++++ .../config/RangerHttpConfiguration.java | 16 +-- .../manager/RangerHttpBundleManager.java | 51 -------- .../server/util/RangerHttpServerUtils.java | 67 ---------- .../http/HttpServiceFinderBuilders.java | 7 +- .../http/HttpServiceProviderBuilders.java | 2 +- .../ranger/http/config/HttpClientConfig.java | 22 ++-- .../http/model/ServiceDataSourceResponse.java | 17 +-- .../http/model/ServiceNodesResponse.java | 17 ++- .../model/ServiceRegistrationResponse.java | 14 +-- .../servicefinder/HttpNodeDataSource.java | 6 +- .../HttpShardedServiceFinderBuilder.java | 9 +- .../HttpUnshardedServiceFinderBuilider.java | 11 +- .../HttpServiceFinderHubBuilder.java | 5 +- .../HttpShardedServiceFinderFactory.java | 11 +- .../HttpUnshardedServiceFinderFactory.java | 11 +- .../flipkart/ranger/http/ResourceHelper.java | 8 +- .../HttpNodeDataStoreConnectorTest.java | 7 +- .../http/config/HttpClientConfigTest.java | 3 +- .../http/model/ServiceNodeResponseTest.java | 16 +-- .../ServiceRegistrationResponseTest.java | 3 +- .../HttpShardedServiceFinderBuilderTest.java | 10 +- .../HttpServiceDataSourceTest.java | 13 +- .../server/bundle/RangerServerBundle.java | 86 ------------- .../zk/server/bundle/RangerServerBundle.java | 111 +++++++++++++++++ .../server/bundle/model/GenericResponse.java | 19 ++- .../server/bundle/model/LifecycleSignal.java | 14 +-- .../bundle/resources/RangerResource.java | 16 ++- .../server/bundle/rotation/BirTask.java | 2 +- .../server/bundle/rotation/OorTask.java | 2 +- .../bundle/rotation/RotationStatus.java | 2 +- .../server/bundle/RangerServerBundleTest.java | 66 +++++++--- .../ranger/common/server/ShardInfo.java | 37 ++---- .../client/zk/AbstractRangerZKHubClient.java | 12 +- .../client/zk/ShardedRangerZKHubClient.java | 12 +- .../client/zk/SimpleRangerZKClient.java | 22 ++-- .../client/zk/UnshardedRangerZKHubClient.java | 12 +- .../client/zk/BaseRangerZKClientTest.java | 5 +- .../client/zk/ShardedZKRangerClientTest.java | 13 +- .../client/zk/SimpleRangerZKClientTest.java | 7 +- .../zk/UnshardedZKRangerClientTest.java | 13 +- .../java/com/flipkart/ranger/server/App.java | 92 -------------- .../com/flipkart/ranger/zk/server/App.java | 42 +++++++ .../{ => zk}/server/AppConfiguration.java | 24 ++-- .../server/bundle/ZKServerBundle.java} | 66 +++++++--- .../server/config/RangerConfiguration.java | 20 +-- .../server/healthcheck/RangerHealthCheck.java | 3 +- .../server/lifecycle/CuratorLifecycle.java} | 39 ++---- .../zookeeper/ServiceFinderBuilders.java | 5 +- .../servicefinder/ZkNodeDataSource.java | 16 +-- .../ZkSimpleShardedServiceFinderBuilder.java | 9 +- ...ZkSimpleUnshardedServiceFinderBuilder.java | 11 +- .../ZKUnshardedServiceFinderFactory.java | 11 +- .../servicefinderhub/ZkServiceDataSource.java | 3 +- .../ZkServiceFinderHubBuilder.java | 11 +- .../ZkShardedServiceFinderFactory.java | 11 +- .../ServiceHealthAggregatorTest.java | 4 +- .../ServiceProviderIntegrationTest.java | 5 +- .../model/CustomShardSelectorTest.java | 38 +++--- .../model/ServiceNoProviderTest.java | 67 ++-------- .../model/ServiceProviderExtCuratorTest.java | 67 ++-------- .../model/ServiceProviderHealthcheckTest.java | 63 ++-------- .../zookeeper/model/ServiceProviderTest.java | 115 +++++------------- .../model/SimpleServiceProviderTest.java | 8 +- .../zookeeper/servicehub/ServiceHubTest.java | 37 ++---- 119 files changed, 1094 insertions(+), 1399 deletions(-) delete mode 100644 ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestShardInfo.java delete mode 100644 ranger-core/src/test/java/com/flipkart/ranger/core/utils/CriteriaUtils.java create mode 100644 ranger-core/src/test/java/com/flipkart/ranger/core/utils/RangerTestUtils.java rename ranger-core/src/test/java/com/flipkart/ranger/core/{ => utils}/TestUtils.java (90%) create mode 100644 ranger-http-server/src/main/java/com/flipkart/ranger/http/server/bundle/HttpServerBundle.java delete mode 100644 ranger-http-server/src/main/java/com/flipkart/ranger/http/server/manager/RangerHttpBundleManager.java delete mode 100644 ranger-http-server/src/main/java/com/flipkart/ranger/http/server/util/RangerHttpServerUtils.java delete mode 100644 ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/RangerServerBundle.java create mode 100644 ranger-server-bundle/src/main/java/com/flipkart/ranger/zk/server/bundle/RangerServerBundle.java rename ranger-server-bundle/src/main/java/com/flipkart/ranger/{ => zk}/server/bundle/model/GenericResponse.java (76%) rename ranger-core/src/main/java/com/flipkart/ranger/core/model/Criteria.java => ranger-server-bundle/src/main/java/com/flipkart/ranger/zk/server/bundle/model/LifecycleSignal.java (80%) rename ranger-server-bundle/src/main/java/com/flipkart/ranger/{ => zk}/server/bundle/resources/RangerResource.java (83%) rename ranger-server-bundle/src/main/java/com/flipkart/ranger/{ => zk}/server/bundle/rotation/BirTask.java (95%) rename ranger-server-bundle/src/main/java/com/flipkart/ranger/{ => zk}/server/bundle/rotation/OorTask.java (95%) rename ranger-server-bundle/src/main/java/com/flipkart/ranger/{ => zk}/server/bundle/rotation/RotationStatus.java (95%) rename ranger-server-bundle/src/test/java/com/flipkart/ranger/{ => zk}/server/bundle/RangerServerBundleTest.java (61%) delete mode 100644 ranger-zk-server/src/main/java/com/flipkart/ranger/server/App.java create mode 100644 ranger-zk-server/src/main/java/com/flipkart/ranger/zk/server/App.java rename ranger-zk-server/src/main/java/com/flipkart/ranger/{ => zk}/server/AppConfiguration.java (71%) rename ranger-zk-server/src/main/java/com/flipkart/ranger/{server/util/RangerServerUtils.java => zk/server/bundle/ZKServerBundle.java} (52%) rename ranger-zk-server/src/main/java/com/flipkart/ranger/{ => zk}/server/config/RangerConfiguration.java (77%) rename ranger-zk-server/src/main/java/com/flipkart/ranger/{ => zk}/server/healthcheck/RangerHealthCheck.java (96%) rename ranger-zk-server/src/main/java/com/flipkart/ranger/{server/manager/RangerBundleManager.java => zk/server/lifecycle/CuratorLifecycle.java} (56%) diff --git a/ranger-client/src/main/java/com/flipkart/ranger/client/AbstractRangerHubClient.java b/ranger-client/src/main/java/com/flipkart/ranger/client/AbstractRangerHubClient.java index 0f610873..7d548c85 100644 --- a/ranger-client/src/main/java/com/flipkart/ranger/client/AbstractRangerHubClient.java +++ b/ranger-client/src/main/java/com/flipkart/ranger/client/AbstractRangerHubClient.java @@ -19,33 +19,38 @@ import com.flipkart.ranger.core.finderhub.ServiceDataSource; import com.flipkart.ranger.core.finderhub.ServiceFinderFactory; import com.flipkart.ranger.core.finderhub.ServiceFinderHub; -import com.flipkart.ranger.core.model.*; +import com.flipkart.ranger.core.model.Deserializer; +import com.flipkart.ranger.core.model.Service; +import com.flipkart.ranger.core.model.ServiceNode; +import com.flipkart.ranger.core.model.ServiceRegistry; import com.google.common.base.Preconditions; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import lombok.val; import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.function.Predicate; @Slf4j @Getter -public abstract class AbstractRangerHubClient, R extends ServiceRegistry, D extends Deserializer> implements RangerHubClient { +public abstract class AbstractRangerHubClient, D extends Deserializer> implements RangerHubClient { private final String namespace; private final ObjectMapper mapper; - private final C criteria; private final D deserializer; private int nodeRefreshTimeMs; - private ServiceFinderHub hub; + private ServiceFinderHub hub; + private Predicate criteria; public AbstractRangerHubClient( String namespace, ObjectMapper mapper, int nodeRefreshTimeMs, - C criteria, + Predicate criteria, D deserializer ){ this.namespace = namespace; @@ -80,7 +85,7 @@ public Optional> getNode( return getNode(service, criteria); } - public Optional>> getAllNodes( + public List> getAllNodes( final Service service ){ return getAllNodes(service, criteria); @@ -88,29 +93,29 @@ public Optional>> getAllNodes( public Optional> getNode( final Service service, - final C criteria + final Predicate criteria ){ val optionalFinder = this.getHub().finder(service); return optionalFinder.map(trcServiceFinder -> trcServiceFinder.get(criteria)); } - public Optional>> getAllNodes( + public List> getAllNodes( final Service service, - final C criteria + final Predicate criteria ){ val optionalFinder = this.getHub().finder(service); - return optionalFinder.map(trcServiceFinder -> trcServiceFinder.getAll(criteria)); + return optionalFinder.map(trServiceFinder -> trServiceFinder.getAll(criteria)).orElse(Collections.emptyList()); } public Collection getServices() throws Exception { return this.getHub().getServiceDataSource().services(); } - protected abstract ServiceFinderHub buildHub(); + protected abstract ServiceFinderHub buildHub(); protected abstract ServiceDataSource buildServiceDataSource(); - protected abstract ServiceFinderFactory buildFinderFactory(); + protected abstract ServiceFinderFactory buildFinderFactory(); } diff --git a/ranger-client/src/main/java/com/flipkart/ranger/client/RangerClient.java b/ranger-client/src/main/java/com/flipkart/ranger/client/RangerClient.java index 86fe20a1..fe7e0af0 100644 --- a/ranger-client/src/main/java/com/flipkart/ranger/client/RangerClient.java +++ b/ranger-client/src/main/java/com/flipkart/ranger/client/RangerClient.java @@ -15,13 +15,13 @@ */ package com.flipkart.ranger.client; -import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.ServiceNode; import java.util.List; import java.util.Optional; +import java.util.function.Predicate; -public interface RangerClient> { +public interface RangerClient { void start(); @@ -29,9 +29,9 @@ public interface RangerClient> { Optional> getNode(); - Optional> getNode(final C criteria); + Optional> getNode(final Predicate criteria); - Optional>> getAllNodes(); + List> getAllNodes(); - Optional>> getAllNodes(final C criteria); + List> getAllNodes(final Predicate criteria); } diff --git a/ranger-client/src/main/java/com/flipkart/ranger/client/RangerClientConstants.java b/ranger-client/src/main/java/com/flipkart/ranger/client/RangerClientConstants.java index 5f9fa9c0..046f402b 100644 --- a/ranger-client/src/main/java/com/flipkart/ranger/client/RangerClientConstants.java +++ b/ranger-client/src/main/java/com/flipkart/ranger/client/RangerClientConstants.java @@ -15,10 +15,10 @@ */ package com.flipkart.ranger.client; -public class RangerClientConstants { - - private RangerClientConstants(){} +import lombok.experimental.UtilityClass; +@UtilityClass +public class RangerClientConstants { public static final int CONNECTION_RETRY_TIME = 5000; public static final int MINIMUM_REFRESH_TIME = 5000; } diff --git a/ranger-client/src/main/java/com/flipkart/ranger/client/RangerHubClient.java b/ranger-client/src/main/java/com/flipkart/ranger/client/RangerHubClient.java index 9c6badcc..2dde78cd 100644 --- a/ranger-client/src/main/java/com/flipkart/ranger/client/RangerHubClient.java +++ b/ranger-client/src/main/java/com/flipkart/ranger/client/RangerHubClient.java @@ -15,15 +15,15 @@ */ package com.flipkart.ranger.client; -import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.core.model.ServiceNode; import java.util.Collection; import java.util.List; import java.util.Optional; +import java.util.function.Predicate; -public interface RangerHubClient> { +public interface RangerHubClient { void start(); void stop(); @@ -32,9 +32,9 @@ public interface RangerHubClient> { Optional> getNode(final Service service); - Optional> getNode(final Service service, final C criteria); + Optional> getNode(final Service service, final Predicate criteria); - Optional>> getAllNodes(final Service service); + List> getAllNodes(final Service service); - Optional>> getAllNodes(final Service service, final C criteria); + List> getAllNodes(final Service service, final Predicate criteria); } diff --git a/ranger-client/src/test/java/com/flipkart/ranger/client/AbstractRangerHubClientTest.java b/ranger-client/src/test/java/com/flipkart/ranger/client/AbstractRangerHubClientTest.java index 94ea0ee4..30ec5bac 100644 --- a/ranger-client/src/test/java/com/flipkart/ranger/client/AbstractRangerHubClientTest.java +++ b/ranger-client/src/test/java/com/flipkart/ranger/client/AbstractRangerHubClientTest.java @@ -15,13 +15,14 @@ */ package com.flipkart.ranger.client; -import com.flipkart.ranger.client.stubs.RangerTestHub; -import com.flipkart.ranger.client.stubs.TestShardInfo; import com.flipkart.ranger.client.utils.RangerHubTestUtils; -import com.flipkart.ranger.core.TestUtils; import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.core.model.ServiceNode; +import com.flipkart.ranger.core.units.TestNodeData; +import com.flipkart.ranger.core.utils.RangerTestUtils; +import com.flipkart.ranger.core.utils.TestUtils; import lombok.extern.slf4j.Slf4j; +import lombok.val; import org.junit.Assert; import org.junit.Test; @@ -30,28 +31,29 @@ @Slf4j public class AbstractRangerHubClientTest { - private static final Service service = new Service("test-ns", "test-s"); + private static final Service service = RangerTestUtils.getService("test-ns", "test-s"); @Test public void testAbstractHubClient() { - RangerTestHub testAbstractHub = RangerHubTestUtils.getTestHub(); + val testAbstractHub = RangerHubTestUtils.getTestHub(); testAbstractHub.start(); + TestUtils.sleepForSeconds(3); - Optional> node = testAbstractHub.getNode(service); + Optional> node = testAbstractHub.getNode(service); Assert.assertTrue(node.isPresent()); Assert.assertTrue(node.get().getHost().equalsIgnoreCase("localhost")); Assert.assertEquals(9200, node.get().getPort()); Assert.assertEquals(1, node.get().getNodeData().getShardId()); - node = testAbstractHub.getNode(new Service("test", "test")); + node = testAbstractHub.getNode(RangerTestUtils.getService("test", "test")); Assert.assertFalse(node.isPresent()); node = testAbstractHub.getNode(service, nodeData -> nodeData.getShardId() == 2); Assert.assertFalse(node.isPresent()); - node = testAbstractHub.getNode(new Service("test", "test"), nodeData -> nodeData.getShardId() == 1); + node = testAbstractHub.getNode(RangerTestUtils.getService("test", "test"), nodeData -> nodeData.getShardId() == 1); Assert.assertFalse(node.isPresent()); - testAbstractHub.start(); + testAbstractHub.stop(); } } diff --git a/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/RangerTestHub.java b/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/RangerTestHub.java index 367aa766..059d4ef5 100644 --- a/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/RangerTestHub.java +++ b/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/RangerTestHub.java @@ -20,25 +20,31 @@ import com.flipkart.ranger.client.utils.RangerHubTestUtils; import com.flipkart.ranger.core.finder.serviceregistry.ListBasedServiceRegistry; import com.flipkart.ranger.core.finderhub.*; -import com.flipkart.ranger.core.model.Criteria; +import com.flipkart.ranger.core.units.TestNodeData; import com.google.common.collect.Lists; +import lombok.Builder; -public class RangerTestHub extends AbstractRangerHubClient, ListBasedServiceRegistry, TestDeserializer> { +import java.util.function.Predicate; - public RangerTestHub(String namespace, ObjectMapper mapper, int nodeRefreshTimeMs, Criteria criteria, TestDeserializer deserilizer) { +public class RangerTestHub extends AbstractRangerHubClient, TestDeserializer> { + + @Builder + public RangerTestHub(String namespace, ObjectMapper mapper, int nodeRefreshTimeMs, + Predicate criteria, TestDeserializer deserilizer) { super(namespace, mapper, nodeRefreshTimeMs, criteria, deserilizer); } @Override - protected ServiceFinderHub, ListBasedServiceRegistry> buildHub() { - return new ServiceFinderHubBuilder, ListBasedServiceRegistry>() { + protected ServiceFinderHub> buildHub() { + return new ServiceFinderHubBuilder>() { @Override protected void preBuild() { } @Override - protected void postBuild(ServiceFinderHub, ListBasedServiceRegistry> serviceFinderHub) { + protected void postBuild(ServiceFinderHub> serviceFinderHub) { } }.withServiceDataSource(buildServiceDataSource()) @@ -52,7 +58,7 @@ protected ServiceDataSource buildServiceDataSource() { } @Override - protected ServiceFinderFactory, ListBasedServiceRegistry> buildFinderFactory() { + protected ServiceFinderFactory> buildFinderFactory() { return new TestServiceFinderFactory(); } } diff --git a/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestCriteria.java b/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestCriteria.java index 3f7f7497..36511b46 100644 --- a/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestCriteria.java +++ b/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestCriteria.java @@ -15,10 +15,12 @@ */ package com.flipkart.ranger.client.stubs; -import com.flipkart.ranger.core.model.Criteria; +import com.flipkart.ranger.core.units.TestNodeData; -public class TestCriteria implements Criteria { - public boolean apply(TestShardInfo nodeData) { +import java.util.function.Predicate; + +public class TestCriteria implements Predicate { + public boolean test(TestNodeData nodeData) { return nodeData.getShardId() == 1; } } diff --git a/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestServiceFinderFactory.java b/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestServiceFinderFactory.java index c3242d95..504ab91d 100644 --- a/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestServiceFinderFactory.java +++ b/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestServiceFinderFactory.java @@ -16,21 +16,21 @@ package com.flipkart.ranger.client.stubs; import com.flipkart.ranger.core.finder.ServiceFinder; -import com.flipkart.ranger.core.finder.SimpleUnshardedServiceFinder; import com.flipkart.ranger.core.finder.serviceregistry.ListBasedServiceRegistry; import com.flipkart.ranger.core.finderhub.ServiceFinderFactory; -import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.Deserializer; import com.flipkart.ranger.core.model.Service; +import com.flipkart.ranger.core.units.TestNodeData; +import lombok.val; -public class TestServiceFinderFactory implements ServiceFinderFactory, ListBasedServiceRegistry> { +public class TestServiceFinderFactory implements ServiceFinderFactory> { @Override - public ServiceFinder, ListBasedServiceRegistry> buildFinder(Service service) { - SimpleUnshardedServiceFinder> finder = new TestSimpleUnshardedServiceFinder>() + public ServiceFinder> buildFinder(Service service) { + val finder = new TestSimpleUnshardedServiceFinder() .withNamespace(service.getNamespace()) .withServiceName(service.getServiceName()) - .withDeserializer(new Deserializer() { + .withDeserializer(new Deserializer() { @Override public int hashCode() { return super.hashCode(); diff --git a/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestShardInfo.java b/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestShardInfo.java deleted file mode 100644 index 10e74007..00000000 --- a/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestShardInfo.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright 2015 Flipkart Internet Pvt. Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.flipkart.ranger.client.stubs; - -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; - -@Data -@Builder -@AllArgsConstructor -public class TestShardInfo { - private int shardId; -} diff --git a/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestSimpleUnshardedServiceFinder.java b/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestSimpleUnshardedServiceFinder.java index 71764674..bed6c38e 100644 --- a/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestSimpleUnshardedServiceFinder.java +++ b/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestSimpleUnshardedServiceFinder.java @@ -17,54 +17,58 @@ import com.flipkart.ranger.core.finder.SimpleUnshardedServiceFinder; import com.flipkart.ranger.core.finder.SimpleUnshardedServiceFinderBuilder; -import com.flipkart.ranger.core.model.*; +import com.flipkart.ranger.core.model.Deserializer; +import com.flipkart.ranger.core.model.NodeDataSource; +import com.flipkart.ranger.core.model.Service; +import com.flipkart.ranger.core.model.ServiceNode; +import com.flipkart.ranger.core.units.TestNodeData; import com.google.common.collect.Lists; import lombok.Builder; import java.util.List; -import java.util.Optional; @Builder -public class TestSimpleUnshardedServiceFinder > - extends SimpleUnshardedServiceFinderBuilder, Deserializer, Criteria> { +public class TestSimpleUnshardedServiceFinder + extends SimpleUnshardedServiceFinderBuilder, Deserializer> { @Override - public SimpleUnshardedServiceFinder> build() { + public SimpleUnshardedServiceFinder build() { return buildFinder(); } @Override - protected NodeDataSource> dataSource(Service service) { - return new NodeDataSource>() { - @Override - public Optional>> refresh(Deserializer deserializer) { - return Optional.of( - Lists.newArrayList( - new ServiceNode<>("localhost", 9200, TestShardInfo.builder().shardId(1).build()) - ) - ); - } + protected NodeDataSource> dataSource(Service service) { + return new TestDataSource(); + } + + static class TestDataSource implements NodeDataSource>{ + + @Override + public List> refresh(Deserializer deserializer) { + return Lists.newArrayList( + new ServiceNode<>("localhost", 9200, TestNodeData.builder().shardId(1).build()) + ); + } - @Override - public void start() { + @Override + public void start() { - } + } - @Override - public void ensureConnected() { + @Override + public void ensureConnected() { - } + } - @Override - public void stop() { + @Override + public void stop() { - } + } - @Override - public boolean isActive() { - return true; - } - }; + @Override + public boolean isActive() { + return true; + } } } diff --git a/ranger-client/src/test/java/com/flipkart/ranger/client/utils/RangerHubTestUtils.java b/ranger-client/src/test/java/com/flipkart/ranger/client/utils/RangerHubTestUtils.java index affd5dad..a48c1999 100644 --- a/ranger-client/src/test/java/com/flipkart/ranger/client/utils/RangerHubTestUtils.java +++ b/ranger-client/src/test/java/com/flipkart/ranger/client/utils/RangerHubTestUtils.java @@ -20,15 +20,22 @@ import com.flipkart.ranger.client.stubs.TestCriteria; import com.flipkart.ranger.client.stubs.TestDeserializer; import com.flipkart.ranger.core.model.Service; +import com.flipkart.ranger.core.utils.RangerTestUtils; +import lombok.experimental.UtilityClass; +@UtilityClass public class RangerHubTestUtils { - public static final Service service = new Service("test-ns", "test-s"); + public static final Service service = RangerTestUtils.getService("test-ns", "test-s"); private static final ObjectMapper mapper = new ObjectMapper(); - private RangerHubTestUtils(){} - public static RangerTestHub getTestHub(){ - return new RangerTestHub(service.getNamespace(), mapper, 1000, new TestCriteria(), new TestDeserializer<>()); + return RangerTestHub.builder() + .namespace(service.getNamespace()) + .mapper(mapper) + .nodeRefreshTimeMs(1000) + .criteria(new TestCriteria()) + .deserilizer(new TestDeserializer<>()) + .build(); } } diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/BaseServiceFinderBuilder.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/BaseServiceFinderBuilder.java index a83df33c..b48436a8 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/BaseServiceFinderBuilder.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/BaseServiceFinderBuilder.java @@ -38,17 +38,16 @@ public abstract class BaseServiceFinderBuilder < T, R extends ServiceRegistry, - F extends ServiceFinder, - B extends BaseServiceFinderBuilder, - D extends Deserializer, - C extends Criteria> { + F extends ServiceFinder, + B extends BaseServiceFinderBuilder, + D extends Deserializer> { protected String namespace; protected String serviceName; protected int nodeRefreshIntervalMs; protected boolean disablePushUpdaters; protected D deserializer; - protected ShardSelector shardSelector; + protected ShardSelector shardSelector; protected ServiceNodeSelector nodeSelector = new RandomServiceNodeSelector<>(); protected final List> additionalRefreshSignals = new ArrayList<>(); protected final List> startSignalHandlers = Lists.newArrayList(); @@ -69,7 +68,7 @@ public B withDeserializer(D deserializer) { return (B)this; } - public B withShardSelector(ShardSelector shardSelector) { + public B withShardSelector(ShardSelector shardSelector) { this.shardSelector = shardSelector; return (B)this; } @@ -141,7 +140,7 @@ protected F buildFinder() { serviceName, nodeRefreshIntervalMs); nodeRefreshIntervalMs = 1000; } - val service = new Service(namespace, serviceName); + val service = Service.builder().namespace(namespace).serviceName(serviceName).build(); val finder = buildFinder(service, shardSelector, nodeSelector); val registry = finder.getServiceRegistry(); val signalGenerators = new ArrayList>(); @@ -177,7 +176,7 @@ protected List> implementationSpecificRefreshSignals(Service service, protected abstract F buildFinder( Service service, - ShardSelector shardSelector, + ShardSelector shardSelector, ServiceNodeSelector nodeSelector); } diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/ServiceFinder.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/ServiceFinder.java index 0ab7ab8a..559bfbc3 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/ServiceFinder.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/ServiceFinder.java @@ -16,19 +16,23 @@ package com.flipkart.ranger.core.finder; -import com.flipkart.ranger.core.model.*; +import com.flipkart.ranger.core.model.ServiceNode; +import com.flipkart.ranger.core.model.ServiceNodeSelector; +import com.flipkart.ranger.core.model.ServiceRegistry; +import com.flipkart.ranger.core.model.ShardSelector; import com.flipkart.ranger.core.signals.ExternalTriggeredSignal; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import java.util.Collections; import java.util.List; +import java.util.function.Predicate; @Slf4j -public abstract class ServiceFinder, ServiceRegistryType extends ServiceRegistry> { +public abstract class ServiceFinder> { @Getter private final ServiceRegistryType serviceRegistry; - private final ShardSelector shardSelector; + private final ShardSelector shardSelector; private final ServiceNodeSelector nodeSelector; @Getter private final ExternalTriggeredSignal startSignal = new ExternalTriggeredSignal<>(() -> null, Collections.emptyList()); @@ -37,14 +41,14 @@ public abstract class ServiceFinder, ServiceRegistryTyp protected ServiceFinder( ServiceRegistryType serviceRegistry, - ShardSelector shardSelector, + ShardSelector shardSelector, ServiceNodeSelector nodeSelector) { this.serviceRegistry = serviceRegistry; this.shardSelector = shardSelector; this.nodeSelector = nodeSelector; } - public ServiceNode get(C criteria) { + public ServiceNode get(Predicate criteria) { List> nodes = shardSelector.nodes(criteria, serviceRegistry); if (null == nodes || nodes.isEmpty()) { return null; @@ -52,7 +56,7 @@ public ServiceNode get(C criteria) { return nodeSelector.select(nodes); } - public List> getAll(C criteria) { + public List> getAll(Predicate criteria) { return shardSelector.nodes(criteria, serviceRegistry); } diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/SimpleShardedServiceFinder.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/SimpleShardedServiceFinder.java index 44e66eb5..130c92ac 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/SimpleShardedServiceFinder.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/SimpleShardedServiceFinder.java @@ -17,13 +17,12 @@ package com.flipkart.ranger.core.finder; import com.flipkart.ranger.core.finder.serviceregistry.MapBasedServiceRegistry; -import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.ServiceNodeSelector; import com.flipkart.ranger.core.model.ShardSelector; -public class SimpleShardedServiceFinder> extends ServiceFinder> { +public class SimpleShardedServiceFinder extends ServiceFinder> { public SimpleShardedServiceFinder(MapBasedServiceRegistry serviceRegistry, - ShardSelector> shardSelector, + ShardSelector> shardSelector, ServiceNodeSelector nodeSelector) { super(serviceRegistry, shardSelector, nodeSelector); } diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/SimpleShardedServiceFinderBuilder.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/SimpleShardedServiceFinderBuilder.java index a6aafb65..138333f4 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/SimpleShardedServiceFinderBuilder.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/SimpleShardedServiceFinderBuilder.java @@ -18,21 +18,23 @@ import com.flipkart.ranger.core.finder.serviceregistry.MapBasedServiceRegistry; import com.flipkart.ranger.core.finder.shardselector.MatchingShardSelector; -import com.flipkart.ranger.core.model.*; +import com.flipkart.ranger.core.model.Deserializer; +import com.flipkart.ranger.core.model.Service; +import com.flipkart.ranger.core.model.ServiceNodeSelector; +import com.flipkart.ranger.core.model.ShardSelector; -public abstract class SimpleShardedServiceFinderBuilder, D extends Deserializer, C extends Criteria> - extends BaseServiceFinderBuilder, SimpleShardedServiceFinder, B, D, C> { +public abstract class SimpleShardedServiceFinderBuilder, D extends Deserializer> + extends BaseServiceFinderBuilder, SimpleShardedServiceFinder, B, D> { @Override - protected SimpleShardedServiceFinder buildFinder( + protected SimpleShardedServiceFinder buildFinder( Service service, - ShardSelector> shardSelector, + ShardSelector> shardSelector, ServiceNodeSelector nodeSelector ) { if (null == shardSelector) { shardSelector = new MatchingShardSelector<>(); } - final MapBasedServiceRegistry serviceRegistry = new MapBasedServiceRegistry<>(service); - return new SimpleShardedServiceFinder<>(serviceRegistry, shardSelector, nodeSelector); + return new SimpleShardedServiceFinder<>(new MapBasedServiceRegistry<>(service), shardSelector, nodeSelector); } } diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/SimpleUnshardedServiceFinder.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/SimpleUnshardedServiceFinder.java index cdf3583f..10aab5bf 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/SimpleUnshardedServiceFinder.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/SimpleUnshardedServiceFinder.java @@ -17,13 +17,12 @@ package com.flipkart.ranger.core.finder; import com.flipkart.ranger.core.finder.serviceregistry.ListBasedServiceRegistry; -import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.ServiceNodeSelector; import com.flipkart.ranger.core.model.ShardSelector; -public class SimpleUnshardedServiceFinder> extends ServiceFinder> { +public class SimpleUnshardedServiceFinder extends ServiceFinder> { public SimpleUnshardedServiceFinder(ListBasedServiceRegistry serviceRegistry, - ShardSelector> shardSelector, + ShardSelector> shardSelector, ServiceNodeSelector nodeSelector) { super(serviceRegistry, shardSelector, nodeSelector); } diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/SimpleUnshardedServiceFinderBuilder.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/SimpleUnshardedServiceFinderBuilder.java index 7917b705..e59b9e0c 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/SimpleUnshardedServiceFinderBuilder.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/SimpleUnshardedServiceFinderBuilder.java @@ -17,21 +17,23 @@ import com.flipkart.ranger.core.finder.serviceregistry.ListBasedServiceRegistry; import com.flipkart.ranger.core.finder.shardselector.ListShardSelector; -import com.flipkart.ranger.core.model.*; +import com.flipkart.ranger.core.model.Deserializer; +import com.flipkart.ranger.core.model.Service; +import com.flipkart.ranger.core.model.ServiceNodeSelector; +import com.flipkart.ranger.core.model.ShardSelector; -public abstract class SimpleUnshardedServiceFinderBuilder, D extends Deserializer, C extends Criteria> - extends BaseServiceFinderBuilder, SimpleUnshardedServiceFinder, B, D, C> { +public abstract class SimpleUnshardedServiceFinderBuilder, D extends Deserializer> + extends BaseServiceFinderBuilder, SimpleUnshardedServiceFinder, B, D> { @Override - protected SimpleUnshardedServiceFinder buildFinder( + protected SimpleUnshardedServiceFinder buildFinder( Service service, - ShardSelector> shardSelector, + ShardSelector> shardSelector, ServiceNodeSelector nodeSelector ) { if (null == shardSelector) { shardSelector = new ListShardSelector<>(); } - final ListBasedServiceRegistry serviceRegistry = new ListBasedServiceRegistry<>(service); - return new SimpleUnshardedServiceFinder<>(serviceRegistry, shardSelector, nodeSelector); + return new SimpleUnshardedServiceFinder<>(new ListBasedServiceRegistry<>(service), shardSelector, nodeSelector); } } \ No newline at end of file diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/serviceregistry/ListBasedServiceRegistry.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/serviceregistry/ListBasedServiceRegistry.java index c0e7f0ab..66e2123f 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/serviceregistry/ListBasedServiceRegistry.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/serviceregistry/ListBasedServiceRegistry.java @@ -19,6 +19,7 @@ import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.model.ServiceRegistry; import com.google.common.collect.ImmutableList; +import lombok.val; import java.util.Collections; import java.util.List; @@ -33,8 +34,8 @@ public ListBasedServiceRegistry(Service service) { } public List> nodeList() { - List> nodes = this.nodes.get(); - return null == nodes ? Collections.emptyList() : nodes; + val nodeList = this.nodes.get(); + return null == nodeList ? Collections.emptyList() : nodeList; } @Override diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/serviceregistry/MapBasedServiceRegistry.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/serviceregistry/MapBasedServiceRegistry.java index afaf6fb3..8074d8ba 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/serviceregistry/MapBasedServiceRegistry.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/serviceregistry/MapBasedServiceRegistry.java @@ -21,6 +21,7 @@ import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.ImmutableListMultimap; import com.google.common.collect.ListMultimap; +import lombok.val; import java.util.ArrayList; import java.util.Collections; @@ -35,13 +36,13 @@ public MapBasedServiceRegistry(Service service) { } public ListMultimap> nodes() { - final ListMultimap> nodeList = nodes.get(); + val nodeList = nodes.get(); return null == nodeList ? ImmutableListMultimap.of() : nodeList; } @Override public List> nodeList() { - final ListMultimap> nodeList = nodes.get(); + val nodeList = nodes.get(); return null == nodeList ? Collections.emptyList() : new ArrayList<>(nodeList.values()); } diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/serviceregistry/ServiceRegistryUpdater.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/serviceregistry/ServiceRegistryUpdater.java index 5a5a2634..ec437e5c 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/serviceregistry/ServiceRegistryUpdater.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/serviceregistry/ServiceRegistryUpdater.java @@ -134,10 +134,10 @@ private void updateRegistry() throws InterruptedException { return; } val nodeList = nodeDataSource.refresh(deserializer); - if (nodeList.isPresent()) { - log.debug("Updating nodelist of size: {} for [{}]", nodeList.get().size(), + if (!nodeList.isEmpty()) { + log.debug("Updating nodelist of size: {} for [{}]", nodeList.size(), serviceRegistry.getService().getServiceName()); - serviceRegistry.updateNodes(nodeList.get()); + serviceRegistry.updateNodes(nodeList); initialRefreshCompleted.compareAndSet(false, true); } else { diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/shardselector/ListShardSelector.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/shardselector/ListShardSelector.java index d72fa6ae..1063d406 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/shardselector/ListShardSelector.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/shardselector/ListShardSelector.java @@ -16,22 +16,23 @@ package com.flipkart.ranger.core.finder.shardselector; import com.flipkart.ranger.core.finder.serviceregistry.ListBasedServiceRegistry; -import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.model.ShardSelector; import java.util.List; +import java.util.function.Predicate; import java.util.stream.Collectors; -public class ListShardSelector> implements ShardSelector> { +public class ListShardSelector implements ShardSelector> { @Override - public List> nodes(C criteria, ListBasedServiceRegistry serviceRegistry) { + public List> nodes(Predicate criteria, ListBasedServiceRegistry serviceRegistry) { if(null == criteria){ return serviceRegistry.nodeList(); - } - return serviceRegistry.nodeList().stream().filter(node -> criteria.apply(node.getNodeData())).collect(Collectors.toList()); + } + return serviceRegistry.nodeList() + .stream() + .filter(node -> criteria.test(node.getNodeData())) + .collect(Collectors.toList()); } - - } diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/shardselector/MatchingShardSelector.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/shardselector/MatchingShardSelector.java index f94b2363..360cdb03 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/shardselector/MatchingShardSelector.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/shardselector/MatchingShardSelector.java @@ -17,23 +17,23 @@ package com.flipkart.ranger.core.finder.shardselector; import com.flipkart.ranger.core.finder.serviceregistry.MapBasedServiceRegistry; -import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.model.ShardSelector; import java.util.List; import java.util.Map; +import java.util.function.Predicate; import java.util.stream.Collectors; -public class MatchingShardSelector> implements ShardSelector> { +public class MatchingShardSelector implements ShardSelector> { @Override - public List> nodes(C criteria, MapBasedServiceRegistry serviceRegistry) { + public List> nodes(Predicate criteria, MapBasedServiceRegistry serviceRegistry) { if(null == criteria) return serviceRegistry.nodeList(); return serviceRegistry.nodes() .entries() .stream() - .filter(e -> criteria.apply(e.getKey())) + .filter(e -> criteria.test(e.getKey())) .map(Map.Entry::getValue) .collect(Collectors.toList()); } diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/shardselector/NoopShardSelector.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/shardselector/NoopShardSelector.java index 41d3c8fa..7f6d308a 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/shardselector/NoopShardSelector.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/shardselector/NoopShardSelector.java @@ -16,15 +16,16 @@ package com.flipkart.ranger.core.finder.shardselector; import com.flipkart.ranger.core.finder.serviceregistry.ListBasedServiceRegistry; -import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.model.ShardSelector; import java.util.List; +import java.util.function.Predicate; + +public class NoopShardSelector implements ShardSelector> { -public class NoopShardSelector> implements ShardSelector> { @Override - public List> nodes(C criteria, ListBasedServiceRegistry serviceRegistry) { + public List> nodes(Predicate criteria, ListBasedServiceRegistry serviceRegistry) { return serviceRegistry.nodeList(); } } diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/ServiceFinderFactory.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/ServiceFinderFactory.java index d8b9c52e..8ec2eb71 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/ServiceFinderFactory.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/ServiceFinderFactory.java @@ -16,15 +16,14 @@ package com.flipkart.ranger.core.finderhub; import com.flipkart.ranger.core.finder.ServiceFinder; -import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.core.model.ServiceRegistry; /** * */ -public interface ServiceFinderFactory, R extends ServiceRegistry> { +public interface ServiceFinderFactory> { - ServiceFinder buildFinder(final Service service); + ServiceFinder buildFinder(final Service service); } diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/ServiceFinderHub.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/ServiceFinderHub.java index d47298dc..16ca280d 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/ServiceFinderHub.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/ServiceFinderHub.java @@ -16,7 +16,6 @@ package com.flipkart.ranger.core.finderhub; import com.flipkart.ranger.core.finder.ServiceFinder; -import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.core.model.ServiceRegistry; import com.flipkart.ranger.core.signals.ExternalTriggeredSignal; @@ -42,8 +41,8 @@ * */ @Slf4j -public class ServiceFinderHub, R extends ServiceRegistry> { - private final AtomicReference>> finders = new AtomicReference<>(new HashMap<>()); +public class ServiceFinderHub> { + private final AtomicReference>> finders = new AtomicReference<>(new HashMap<>()); private final Lock updateLock = new ReentrantLock(); private final Condition updateCond = updateLock.newCondition(); private boolean updateAvailable = false; @@ -60,14 +59,14 @@ public class ServiceFinderHub, R extends ServiceRegistr @Getter private final ServiceDataSource serviceDataSource; - private final ServiceFinderFactory finderFactory; + private final ServiceFinderFactory finderFactory; private final AtomicBoolean alreadyUpdating = new AtomicBoolean(false); private Future monitorFuture = null; public ServiceFinderHub( ServiceDataSource serviceDataSource, - ServiceFinderFactory finderFactory) { + ServiceFinderFactory finderFactory) { this.serviceDataSource = serviceDataSource; this.finderFactory = finderFactory; this.refreshSignals.add(new ScheduledSignal<>("service-hub-updater", @@ -76,7 +75,7 @@ public ServiceFinderHub( 10_000)); } - public Optional> finder(final Service service) { + public Optional> finder(final Service service) { return Optional.ofNullable(finders.get().get(service)); } @@ -143,10 +142,10 @@ private void updateRegistry() { return; } alreadyUpdating.set(true); - final Map> updatedFinders = new HashMap<>(); + final Map> updatedFinders = new HashMap<>(); try { final Collection services = serviceDataSource.services(); - final Map> knownServiceFinders = finders.get(); + final Map> knownServiceFinders = finders.get(); val newFinders = services.stream() .filter(service -> !knownServiceFinders.containsKey(service)) .collect(Collectors.toMap(Function.identity(), finderFactory::buildFinder)); diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/ServiceFinderHubBuilder.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/ServiceFinderHubBuilder.java index 02761c16..bf25ef55 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/ServiceFinderHubBuilder.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/ServiceFinderHubBuilder.java @@ -15,7 +15,6 @@ */ package com.flipkart.ranger.core.finderhub; -import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.ServiceRegistry; import com.flipkart.ranger.core.signals.ScheduledSignal; import com.flipkart.ranger.core.signals.Signal; @@ -30,45 +29,45 @@ /** * */ -public abstract class ServiceFinderHubBuilder, R extends ServiceRegistry> { +public abstract class ServiceFinderHubBuilder> { private ServiceDataSource serviceDataSource; - private ServiceFinderFactory serviceFinderFactory; + private ServiceFinderFactory serviceFinderFactory; private long refreshFrequencyMs = 10_000; private final List> extraStartSignalConsumers = new ArrayList<>(); private final List> extraStopSignalConsumers = new ArrayList<>(); private final List> extraRefreshSignals = new ArrayList<>(); - public ServiceFinderHubBuilder withServiceDataSource(ServiceDataSource serviceDataSource) { + public ServiceFinderHubBuilder withServiceDataSource(ServiceDataSource serviceDataSource) { this.serviceDataSource = serviceDataSource; return this; } - public ServiceFinderHubBuilder withServiceFinderFactory(ServiceFinderFactory serviceFinderFactory) { + public ServiceFinderHubBuilder withServiceFinderFactory(ServiceFinderFactory serviceFinderFactory) { this.serviceFinderFactory = serviceFinderFactory; return this; } - public ServiceFinderHubBuilder withRefreshFrequencyMs(long refreshFrequencyMs) { + public ServiceFinderHubBuilder withRefreshFrequencyMs(long refreshFrequencyMs) { this.refreshFrequencyMs = refreshFrequencyMs; return this; } - public ServiceFinderHubBuilder withExtraStartSignalConsumer(Consumer consumer) { + public ServiceFinderHubBuilder withExtraStartSignalConsumer(Consumer consumer) { this.extraStartSignalConsumers.add(consumer); return this; } - public ServiceFinderHubBuilder withExtraStopSignalConsumer(Consumer consumer) { + public ServiceFinderHubBuilder withExtraStopSignalConsumer(Consumer consumer) { this.extraStopSignalConsumers.add(consumer); return this; } - public ServiceFinderHubBuilder withExtraRefreshSignal(Signal extraRefreshSignal) { + public ServiceFinderHubBuilder withExtraRefreshSignal(Signal extraRefreshSignal) { this.extraRefreshSignals.add(extraRefreshSignal); return this; } - public ServiceFinderHub build() { + public ServiceFinderHub build() { preBuild(); Preconditions.checkNotNull(serviceDataSource, "Provide a non-null service data source"); Preconditions.checkNotNull(serviceFinderFactory, "Provide a non-null service finder factory"); @@ -95,5 +94,5 @@ public ServiceFinderHub build() { protected abstract void preBuild(); - protected abstract void postBuild(ServiceFinderHub serviceFinderHub); + protected abstract void postBuild(ServiceFinderHub serviceFinderHub); } diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/StaticDataSource.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/StaticDataSource.java index 11ebe0e6..0d66faea 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/StaticDataSource.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/StaticDataSource.java @@ -20,7 +20,6 @@ import lombok.extern.slf4j.Slf4j; import java.util.Collection; -import java.util.List; /* A static data source to be used when we know the services beforehand and don't have to fetch from a source. @@ -30,7 +29,7 @@ @AllArgsConstructor public class StaticDataSource implements ServiceDataSource{ - private final List services; + private final Collection services; @Override public Collection services() throws Exception { diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/model/NodeDataSource.java b/ranger-core/src/main/java/com/flipkart/ranger/core/model/NodeDataSource.java index 79f75a8d..8568b478 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/model/NodeDataSource.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/model/NodeDataSource.java @@ -16,13 +16,13 @@ package com.flipkart.ranger.core.model; import java.util.List; -import java.util.Optional; /** * */ public interface NodeDataSource> extends NodeDataStoreConnector { - Optional>> refresh(D deserializer); + + List> refresh(D deserializer); default long healthcheckZombieCheckThresholdTime(Service service) { return System.currentTimeMillis() - 60000; //1 Minute diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/model/Service.java b/ranger-core/src/main/java/com/flipkart/ranger/core/model/Service.java index a13e9942..164740c7 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/model/Service.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/model/Service.java @@ -15,17 +15,15 @@ */ package com.flipkart.ranger.core.model; -import lombok.AccessLevel; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; +import lombok.Builder; +import lombok.Value; -@Data -@AllArgsConstructor(access = AccessLevel.PUBLIC) -@NoArgsConstructor + +@Value +@Builder public class Service { - private String namespace; - private String serviceName; + String namespace; + String serviceName; public String name() { return String.format("%s/%s", namespace, serviceName); diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/model/ShardSelector.java b/ranger-core/src/main/java/com/flipkart/ranger/core/model/ShardSelector.java index 4053fe07..e7e55fcc 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/model/ShardSelector.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/model/ShardSelector.java @@ -16,9 +16,10 @@ package com.flipkart.ranger.core.model; import java.util.List; +import java.util.function.Predicate; -public interface ShardSelector, ServiceRegistryType extends ServiceRegistry> { +public interface ShardSelector> { - List> nodes(C criteria, ServiceRegistryType serviceRegistry); + List> nodes(Predicate criteria, ServiceRegistryType serviceRegistry); } diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/serviceprovider/BaseServiceProviderBuilder.java b/ranger-core/src/main/java/com/flipkart/ranger/core/serviceprovider/BaseServiceProviderBuilder.java index 08f6a207..0025063e 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/serviceprovider/BaseServiceProviderBuilder.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/serviceprovider/BaseServiceProviderBuilder.java @@ -34,6 +34,7 @@ import lombok.AccessLevel; import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; +import lombok.val; import java.util.Collections; import java.util.List; @@ -174,16 +175,14 @@ protected final ServiceProvider buildProvider() { staleUpdateThresholdMs = 5000; } - final ServiceHealthAggregator serviceHealthAggregator = new ServiceHealthAggregator(); - for (IsolatedHealthMonitor isolatedMonitor : isolatedMonitors) { - serviceHealthAggregator.addIsolatedMonitor(isolatedMonitor); - } + val serviceHealthAggregator = new ServiceHealthAggregator(); + isolatedMonitors.forEach(serviceHealthAggregator::addIsolatedMonitor); healthchecks.add(serviceHealthAggregator); - final Service service = new Service(namespace, serviceName); - final NodeDataSink usableNodeDataSource = dataSink(service); + val service = Service.builder().namespace(namespace).serviceName(serviceName).build(); + val usableNodeDataSource = dataSink(service); - final ScheduledSignal healthcheckUpdateSignalGenerator + val healthcheckUpdateSignalGenerator = new ScheduledSignal<>( service, new HealthChecker(healthchecks, staleUpdateThresholdMs), @@ -191,19 +190,19 @@ protected final ServiceProvider buildProvider() { healthUpdateIntervalMs ); - final List healthServices = Collections.singletonList(serviceHealthAggregator); + val healthServices = Collections.singletonList(serviceHealthAggregator); - final List> signalGenerators + val signalGenerators = ImmutableList.>builder() .add(healthcheckUpdateSignalGenerator) .addAll(additionalRefreshSignals) .build(); - final ServiceProvider serviceProvider = new ServiceProvider<>(service, + val serviceProvider = new ServiceProvider<>(service, new ServiceNode<>(hostname, port, nodeData), serializer, usableNodeDataSource, signalGenerators); - final Signal startSignal = serviceProvider.getStartSignal(); + val startSignal = serviceProvider.getStartSignal(); startSignal .registerConsumers(startSignalHandlers) @@ -211,7 +210,7 @@ protected final ServiceProvider buildProvider() { .registerConsumer(x -> healthServices.forEach(HealthService::start)) .registerConsumer(x -> signalGenerators.forEach(Signal::start)); - final Signal stopSignal = serviceProvider.getStopSignal(); + val stopSignal = serviceProvider.getStopSignal(); stopSignal .registerConsumer(x -> healthServices.forEach(HealthService::stop)) .registerConsumer(x -> signalGenerators.forEach(Signal::stop)) diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/util/Exceptions.java b/ranger-core/src/main/java/com/flipkart/ranger/core/util/Exceptions.java index d93d167e..e431028f 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/util/Exceptions.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/util/Exceptions.java @@ -15,11 +15,13 @@ */ package com.flipkart.ranger.core.util; +import lombok.experimental.UtilityClass; + /** * */ +@UtilityClass public class Exceptions { - private Exceptions() {} public static void illegalState(String message, Throwable t) { throw new IllegalStateException(message, t); diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/util/FinderUtils.java b/ranger-core/src/main/java/com/flipkart/ranger/core/util/FinderUtils.java index d0d7be50..b5bb7285 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/util/FinderUtils.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/util/FinderUtils.java @@ -18,6 +18,7 @@ import com.flipkart.ranger.core.healthcheck.HealthcheckStatus; import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.core.model.ServiceNode; +import lombok.experimental.UtilityClass; import lombok.extern.slf4j.Slf4j; import java.util.Collection; @@ -28,10 +29,9 @@ * */ @Slf4j +@UtilityClass public class FinderUtils { - private FinderUtils() {} - public static List> filterValidNodes( final Service service, final Collection> serviceNodes, diff --git a/ranger-core/src/test/java/com/flipkart/ranger/core/finder/SimpleShardFinderTest.java b/ranger-core/src/test/java/com/flipkart/ranger/core/finder/SimpleShardFinderTest.java index 404d92fa..ddd1a71e 100644 --- a/ranger-core/src/test/java/com/flipkart/ranger/core/finder/SimpleShardFinderTest.java +++ b/ranger-core/src/test/java/com/flipkart/ranger/core/finder/SimpleShardFinderTest.java @@ -17,11 +17,10 @@ import com.flipkart.ranger.core.finder.nodeselector.RoundRobinServiceNodeSelector; import com.flipkart.ranger.core.finder.serviceregistry.MapBasedServiceRegistry; -import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.model.ShardSelector; import com.flipkart.ranger.core.units.TestNodeData; -import com.flipkart.ranger.core.utils.CriteriaUtils; +import com.flipkart.ranger.core.utils.RangerTestUtils; import com.flipkart.ranger.core.utils.RegistryTestUtils; import com.google.common.collect.Lists; import lombok.val; @@ -29,26 +28,27 @@ import org.junit.Test; import java.util.List; +import java.util.function.Predicate; public class SimpleShardFinderTest { - static class TestSimpleShardSelector> implements ShardSelector>{ + static class TestSimpleShardSelector implements ShardSelector>{ @Override - public List> nodes(C criteria, MapBasedServiceRegistry serviceRegistry) { + public List> nodes(Predicate criteria, MapBasedServiceRegistry serviceRegistry) { return Lists.newArrayList(); } } @Test public void testSimpleShardedFinder(){ - final MapBasedServiceRegistry serviceRegistry = RegistryTestUtils.getServiceRegistry(); - final TestSimpleShardSelector> shardSelector = new TestSimpleShardSelector<>(); - final RoundRobinServiceNodeSelector roundRobinServiceNodeSelector = new RoundRobinServiceNodeSelector<>(); + val serviceRegistry = RegistryTestUtils.getServiceRegistry(); + val shardSelector = new TestSimpleShardSelector(); + val roundRobinServiceNodeSelector = new RoundRobinServiceNodeSelector(); val simpleShardedFinder = new SimpleShardedServiceFinder<>( serviceRegistry, shardSelector, roundRobinServiceNodeSelector); val testNodeDataServiceNode = simpleShardedFinder.get( - CriteriaUtils.getCriteria(2)); + RangerTestUtils.getCriteria(2)); Assert.assertNull(testNodeDataServiceNode); } } diff --git a/ranger-core/src/test/java/com/flipkart/ranger/core/finder/UnshardedClusterFinderTest.java b/ranger-core/src/test/java/com/flipkart/ranger/core/finder/UnshardedClusterFinderTest.java index ad35be35..ae4e81d8 100644 --- a/ranger-core/src/test/java/com/flipkart/ranger/core/finder/UnshardedClusterFinderTest.java +++ b/ranger-core/src/test/java/com/flipkart/ranger/core/finder/UnshardedClusterFinderTest.java @@ -15,14 +15,13 @@ */ package com.flipkart.ranger.core.finder; -import com.flipkart.ranger.core.finder.serviceregistry.ListBasedServiceRegistry; import com.flipkart.ranger.core.finder.shardselector.ListShardSelector; -import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.model.ServiceNodeSelector; import com.flipkart.ranger.core.units.TestNodeData; -import com.flipkart.ranger.core.utils.CriteriaUtils; +import com.flipkart.ranger.core.utils.RangerTestUtils; import com.flipkart.ranger.core.utils.RegistryTestUtils; +import lombok.val; import org.junit.Assert; import org.junit.Test; @@ -39,14 +38,14 @@ public ServiceNode select(List> serviceN } @Test public void unshardedClusterFinder(){ - final ListBasedServiceRegistry unshardedRegistry = RegistryTestUtils.getUnshardedRegistry(); - final ListShardSelector> shardSelector = new ListShardSelector<>(); - SimpleUnshardedServiceFinder> simpleUnshardedServiceFinder = new SimpleUnshardedServiceFinder<>( + val unshardedRegistry = RegistryTestUtils.getUnshardedRegistry(); + val shardSelector = new ListShardSelector(); + val simpleUnshardedServiceFinder = new SimpleUnshardedServiceFinder<>( unshardedRegistry, shardSelector, new TestUnshardedNodeSelector() ); - final ServiceNode serviceNode = simpleUnshardedServiceFinder.get(CriteriaUtils.getCriteria(1)); + val serviceNode = simpleUnshardedServiceFinder.get(RangerTestUtils.getCriteria(1)); Assert.assertNotNull(serviceNode); Assert.assertEquals("localhost-1", serviceNode.getHost()); } diff --git a/ranger-core/src/test/java/com/flipkart/ranger/core/finder/nodeselector/RoundRobinServiceNodeSelectorTest.java b/ranger-core/src/test/java/com/flipkart/ranger/core/finder/nodeselector/RoundRobinServiceNodeSelectorTest.java index cb00c712..177048ea 100644 --- a/ranger-core/src/test/java/com/flipkart/ranger/core/finder/nodeselector/RoundRobinServiceNodeSelectorTest.java +++ b/ranger-core/src/test/java/com/flipkart/ranger/core/finder/nodeselector/RoundRobinServiceNodeSelectorTest.java @@ -17,20 +17,20 @@ import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.units.TestNodeData; -import com.google.common.collect.Lists; +import lombok.val; import org.junit.Assert; import org.junit.Test; -import java.util.List; +import java.util.ArrayList; public class RoundRobinServiceNodeSelectorTest { @Test public void testRandomNodeSelector(){ - final RoundRobinServiceNodeSelector roundRobinSelector = new RoundRobinServiceNodeSelector(); - final List> serviceNodes = Lists.newArrayList(); - serviceNodes.add(new ServiceNode<>("localhost-1", 9000, TestNodeData.builder().nodeId(1).build())); - serviceNodes.add(new ServiceNode<>("localhost-2", 9001, TestNodeData.builder().nodeId(2).build())); - serviceNodes.add(new ServiceNode<>("localhost-3", 9002, TestNodeData.builder().nodeId(3).build())); + val roundRobinSelector = new RoundRobinServiceNodeSelector(); + val serviceNodes = new ArrayList>(); + serviceNodes.add(new ServiceNode<>("localhost-1", 9000, TestNodeData.builder().shardId(1).build())); + serviceNodes.add(new ServiceNode<>("localhost-2", 9001, TestNodeData.builder().shardId(2).build())); + serviceNodes.add(new ServiceNode<>("localhost-3", 9002, TestNodeData.builder().shardId(3).build())); ServiceNode select = roundRobinSelector.select(serviceNodes); Assert.assertEquals("localhost-2", select.getHost()); select = roundRobinSelector.select(serviceNodes); diff --git a/ranger-core/src/test/java/com/flipkart/ranger/core/finder/serviceregistry/MapBasedServiceRegistryTest.java b/ranger-core/src/test/java/com/flipkart/ranger/core/finder/serviceregistry/MapBasedServiceRegistryTest.java index effc6f41..1a34ec4c 100644 --- a/ranger-core/src/test/java/com/flipkart/ranger/core/finder/serviceregistry/MapBasedServiceRegistryTest.java +++ b/ranger-core/src/test/java/com/flipkart/ranger/core/finder/serviceregistry/MapBasedServiceRegistryTest.java @@ -16,9 +16,8 @@ package com.flipkart.ranger.core.finder.serviceregistry; import com.flipkart.ranger.core.finder.shardselector.MatchingShardSelector; -import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.units.TestNodeData; -import com.flipkart.ranger.core.utils.CriteriaUtils; +import com.flipkart.ranger.core.utils.RangerTestUtils; import com.flipkart.ranger.core.utils.RegistryTestUtils; import lombok.val; import org.junit.Assert; @@ -28,11 +27,11 @@ public class MapBasedServiceRegistryTest { @Test public void testMapBasedServiceRegistryWithMatchingShardSelector(){ - final MapBasedServiceRegistry serviceRegistry = RegistryTestUtils.getServiceRegistry(); + val serviceRegistry = RegistryTestUtils.getServiceRegistry(); Assert.assertTrue(null != serviceRegistry.nodes() && !serviceRegistry.nodes().isEmpty()); - final MatchingShardSelector> matchingShardSelector = new MatchingShardSelector<>(); + val matchingShardSelector = new MatchingShardSelector(); val nodes = matchingShardSelector.nodes( - CriteriaUtils.getCriteria(1), serviceRegistry); + RangerTestUtils.getCriteria(1), serviceRegistry); Assert.assertFalse(nodes.isEmpty()); Assert.assertEquals("localhost-1", nodes.get(0).getHost()); } diff --git a/ranger-core/src/test/java/com/flipkart/ranger/core/finder/shardselector/ListShardSelectorTest.java b/ranger-core/src/test/java/com/flipkart/ranger/core/finder/shardselector/ListShardSelectorTest.java index 614c493b..187b8901 100644 --- a/ranger-core/src/test/java/com/flipkart/ranger/core/finder/shardselector/ListShardSelectorTest.java +++ b/ranger-core/src/test/java/com/flipkart/ranger/core/finder/shardselector/ListShardSelectorTest.java @@ -15,24 +15,20 @@ */ package com.flipkart.ranger.core.finder.shardselector; -import com.flipkart.ranger.core.finder.serviceregistry.ListBasedServiceRegistry; -import com.flipkart.ranger.core.model.Criteria; -import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.units.TestNodeData; -import com.flipkart.ranger.core.utils.CriteriaUtils; +import com.flipkart.ranger.core.utils.RangerTestUtils; import com.flipkart.ranger.core.utils.RegistryTestUtils; +import lombok.val; import org.junit.Assert; import org.junit.Test; -import java.util.List; - public class ListShardSelectorTest { @Test public void testNoOpShardSelector(){ - final ListBasedServiceRegistry serviceRegistry = RegistryTestUtils.getUnshardedRegistry(); - final ListShardSelector> shardSelector = new ListShardSelector<>(); - final List> nodes = shardSelector.nodes(CriteriaUtils.getCriteria(1), serviceRegistry); + val serviceRegistry = RegistryTestUtils.getUnshardedRegistry(); + val shardSelector = new ListShardSelector(); + val nodes = shardSelector.nodes(RangerTestUtils.getCriteria(1), serviceRegistry); Assert.assertFalse(nodes.isEmpty()); Assert.assertEquals("localhost-1", nodes.get(0).getHost()); } diff --git a/ranger-core/src/test/java/com/flipkart/ranger/core/finder/shardselector/MatchingShardSelectorTest.java b/ranger-core/src/test/java/com/flipkart/ranger/core/finder/shardselector/MatchingShardSelectorTest.java index ce53b3f9..6eba94e3 100644 --- a/ranger-core/src/test/java/com/flipkart/ranger/core/finder/shardselector/MatchingShardSelectorTest.java +++ b/ranger-core/src/test/java/com/flipkart/ranger/core/finder/shardselector/MatchingShardSelectorTest.java @@ -15,31 +15,21 @@ */ package com.flipkart.ranger.core.finder.shardselector; -import com.flipkart.ranger.core.finder.serviceregistry.MapBasedServiceRegistry; -import com.flipkart.ranger.core.model.Criteria; -import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.units.TestNodeData; +import com.flipkart.ranger.core.utils.RangerTestUtils; import com.flipkart.ranger.core.utils.RegistryTestUtils; +import lombok.val; import org.junit.Assert; import org.junit.Test; -import java.util.List; - public class MatchingShardSelectorTest { - private static final class TestCriteria implements Criteria{ - @Override - public boolean apply(TestNodeData nodeData) { - return nodeData.getNodeId() == 1; - } - } - @Test public void testMatchingShardSelector(){ - final MapBasedServiceRegistry serviceRegistry = RegistryTestUtils.getServiceRegistry(); - final MatchingShardSelector> shardSelector = new MatchingShardSelector<>(); - final List> nodes = shardSelector.nodes( - new TestCriteria(), serviceRegistry); + val serviceRegistry = RegistryTestUtils.getServiceRegistry(); + val shardSelector = new MatchingShardSelector(); + val nodes = shardSelector.nodes( + RangerTestUtils.getCriteria(1), serviceRegistry); Assert.assertFalse(nodes.isEmpty()); Assert.assertEquals("localhost-1", nodes.get(0).getHost()); } diff --git a/ranger-core/src/test/java/com/flipkart/ranger/core/serviceprovider/ServiceProviderTest.java b/ranger-core/src/test/java/com/flipkart/ranger/core/serviceprovider/ServiceProviderTest.java index 50e470a7..38503fd2 100644 --- a/ranger-core/src/test/java/com/flipkart/ranger/core/serviceprovider/ServiceProviderTest.java +++ b/ranger-core/src/test/java/com/flipkart/ranger/core/serviceprovider/ServiceProviderTest.java @@ -23,6 +23,7 @@ import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.units.TestNodeData; +import lombok.val; import org.junit.Assert; import org.junit.Test; @@ -118,19 +119,19 @@ public void testInvalidServiceProviderNoHealthCheck(){ @Test public void testBuildServiceProvider(){ - ServiceProvider> testProvider = new TestServiceProviderBuilder() + val testProvider = new TestServiceProviderBuilder() .withServiceName("test-service") .withNamespace("test") .withHostname("localhost-1") .withPort(9000) .withSerializer(new TestSerializerImpl()) - .withNodeData(TestNodeData.builder().nodeId(1).build()) + .withNodeData(TestNodeData.builder().shardId(1).build()) .withHealthcheck(Healthchecks.defaultHealthyCheck()) .withHealthUpdateIntervalMs(1000) .build(); testProvider.start(); Assert.assertNotNull(testNodeData); - Assert.assertEquals(1, testNodeData.getNodeId()); + Assert.assertEquals(1, testNodeData.getShardId()); } } diff --git a/ranger-core/src/test/java/com/flipkart/ranger/core/units/TestNodeData.java b/ranger-core/src/test/java/com/flipkart/ranger/core/units/TestNodeData.java index c38eaa19..324f90ea 100644 --- a/ranger-core/src/test/java/com/flipkart/ranger/core/units/TestNodeData.java +++ b/ranger-core/src/test/java/com/flipkart/ranger/core/units/TestNodeData.java @@ -15,33 +15,13 @@ */ package com.flipkart.ranger.core.units; -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.AllArgsConstructor; import lombok.Builder; -import lombok.Data; -import lombok.NoArgsConstructor; +import lombok.Value; +import lombok.extern.jackson.Jacksonized; -@Data -@AllArgsConstructor -@NoArgsConstructor +@Value +@Jacksonized @Builder public class TestNodeData { - @JsonProperty - private int nodeId; - - @Override - public boolean equals(Object obj){ - if(this == obj) - return true; - if(obj == null || obj.getClass()!= this.getClass()) - return false; - TestNodeData that = (TestNodeData) obj; - return (that.nodeId == this.nodeId); - } - - @Override - public int hashCode(){ - return this.nodeId; - } - + int shardId; } diff --git a/ranger-core/src/test/java/com/flipkart/ranger/core/utils/CriteriaUtils.java b/ranger-core/src/test/java/com/flipkart/ranger/core/utils/CriteriaUtils.java deleted file mode 100644 index 6231dce6..00000000 --- a/ranger-core/src/test/java/com/flipkart/ranger/core/utils/CriteriaUtils.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright 2015 Flipkart Internet Pvt. Ltd. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.flipkart.ranger.core.utils; - -import com.flipkart.ranger.core.model.Criteria; -import com.flipkart.ranger.core.units.TestNodeData; - -public class CriteriaUtils { - - public static Criteria getCriteria(int shardId){ - return nodeData -> nodeData.getNodeId() == shardId; - } -} diff --git a/ranger-core/src/test/java/com/flipkart/ranger/core/utils/RangerTestUtils.java b/ranger-core/src/test/java/com/flipkart/ranger/core/utils/RangerTestUtils.java new file mode 100644 index 00000000..bf6dcfd9 --- /dev/null +++ b/ranger-core/src/test/java/com/flipkart/ranger/core/utils/RangerTestUtils.java @@ -0,0 +1,44 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.core.utils; + +import com.flipkart.ranger.core.model.Service; +import com.flipkart.ranger.core.units.TestNodeData; +import lombok.experimental.UtilityClass; + +import java.util.function.Predicate; + +@UtilityClass +public class RangerTestUtils { + + private static String TEST_NAMESPACE = "test"; + private static String TEST_SERVICE = "test-service"; + + public static Predicate getCriteria(final int shardId){ + return nodeData -> nodeData.getShardId() == shardId; + } + + public static Service getService(){ + return getService(TEST_NAMESPACE, TEST_SERVICE); + } + + public static Service getService(String namespace, String serviceName){ + return Service.builder() + .serviceName(serviceName) + .namespace(namespace) + .build(); + } +} diff --git a/ranger-core/src/test/java/com/flipkart/ranger/core/utils/RegistryTestUtils.java b/ranger-core/src/test/java/com/flipkart/ranger/core/utils/RegistryTestUtils.java index 5ea19537..c235b4be 100644 --- a/ranger-core/src/test/java/com/flipkart/ranger/core/utils/RegistryTestUtils.java +++ b/ranger-core/src/test/java/com/flipkart/ranger/core/utils/RegistryTestUtils.java @@ -17,33 +17,35 @@ import com.flipkart.ranger.core.finder.serviceregistry.ListBasedServiceRegistry; import com.flipkart.ranger.core.finder.serviceregistry.MapBasedServiceRegistry; -import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.units.TestNodeData; import com.google.common.collect.Lists; +import lombok.experimental.UtilityClass; +import lombok.val; import java.util.List; +@UtilityClass public class RegistryTestUtils { public static MapBasedServiceRegistry getServiceRegistry(){ - final Service service = new Service("test", "test-service"); - final MapBasedServiceRegistry serviceRegistry = new MapBasedServiceRegistry<>(service); + val service = RangerTestUtils.getService(); + val serviceRegistry = new MapBasedServiceRegistry(service); List> serviceNodes = Lists.newArrayList(); - serviceNodes.add(new ServiceNode<>("localhost-1", 9000, TestNodeData.builder().nodeId(1).build())); - serviceNodes.add(new ServiceNode<>("localhost-2", 9001, TestNodeData.builder().nodeId(2).build())); - serviceNodes.add(new ServiceNode<>("localhost-3", 9002, TestNodeData.builder().nodeId(3).build())); + serviceNodes.add(new ServiceNode<>("localhost-1", 9000, TestNodeData.builder().shardId(1).build())); + serviceNodes.add(new ServiceNode<>("localhost-2", 9001, TestNodeData.builder().shardId(2).build())); + serviceNodes.add(new ServiceNode<>("localhost-3", 9002, TestNodeData.builder().shardId(3).build())); serviceRegistry.updateNodes(serviceNodes); return serviceRegistry; } public static ListBasedServiceRegistry getUnshardedRegistry(){ - final Service service = new Service("test", "test-service"); - final ListBasedServiceRegistry serviceRegistry = new ListBasedServiceRegistry<>(service); + val service = RangerTestUtils.getService(); + val serviceRegistry = new ListBasedServiceRegistry(service); List> serviceNodes = Lists.newArrayList(); - serviceNodes.add(new ServiceNode<>("localhost-1", 9000, TestNodeData.builder().nodeId(1).build())); - serviceNodes.add(new ServiceNode<>("localhost-2", 9001, TestNodeData.builder().nodeId(2).build())); - serviceNodes.add(new ServiceNode<>("localhost-3", 9002, TestNodeData.builder().nodeId(3).build())); + serviceNodes.add(new ServiceNode<>("localhost-1", 9000, TestNodeData.builder().shardId(1).build())); + serviceNodes.add(new ServiceNode<>("localhost-2", 9001, TestNodeData.builder().shardId(2).build())); + serviceNodes.add(new ServiceNode<>("localhost-3", 9002, TestNodeData.builder().shardId(3).build())); serviceRegistry.updateNodes(serviceNodes); return serviceRegistry; } diff --git a/ranger-core/src/test/java/com/flipkart/ranger/core/TestUtils.java b/ranger-core/src/test/java/com/flipkart/ranger/core/utils/TestUtils.java similarity index 90% rename from ranger-core/src/test/java/com/flipkart/ranger/core/TestUtils.java rename to ranger-core/src/test/java/com/flipkart/ranger/core/utils/TestUtils.java index cde529a6..cd6072ec 100644 --- a/ranger-core/src/test/java/com/flipkart/ranger/core/TestUtils.java +++ b/ranger-core/src/test/java/com/flipkart/ranger/core/utils/TestUtils.java @@ -13,7 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.flipkart.ranger.core; +package com.flipkart.ranger.core.utils; + +import lombok.experimental.UtilityClass; import java.util.concurrent.TimeUnit; @@ -22,7 +24,9 @@ /** * */ +@UtilityClass public class TestUtils { + public static void sleepForSeconds(int numSeconds) { await().pollDelay(numSeconds, TimeUnit.SECONDS).until(() -> true); } diff --git a/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/ShardedRangerHttpHubClient.java b/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/ShardedRangerHttpHubClient.java index 6f999046..7620b6e2 100644 --- a/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/ShardedRangerHttpHubClient.java +++ b/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/ShardedRangerHttpHubClient.java @@ -24,7 +24,6 @@ import com.flipkart.ranger.core.finderhub.ServiceFinderFactory; import com.flipkart.ranger.core.finderhub.ServiceFinderHub; import com.flipkart.ranger.core.finderhub.StaticDataSource; -import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.http.config.HttpClientConfig; import com.flipkart.ranger.http.serde.HTTPResponseDataDeserializer; @@ -34,13 +33,14 @@ import lombok.Builder; import lombok.extern.slf4j.Slf4j; -import java.util.List; +import java.util.Set; +import java.util.function.Predicate; @Slf4j -public class ShardedRangerHttpHubClient> - extends AbstractRangerHubClient, HTTPResponseDataDeserializer> { +public class ShardedRangerHttpHubClient + extends AbstractRangerHubClient, HTTPResponseDataDeserializer> { - private final List services; + private final Set services; private final HttpClientConfig clientConfig; @Builder @@ -48,10 +48,10 @@ public ShardedRangerHttpHubClient( String namespace, ObjectMapper mapper, int nodeRefreshIntervalMs, - C criteria, + Predicate criteria, HTTPResponseDataDeserializer deserializer, HttpClientConfig clientConfig, - List services + Set services ) { super(namespace, mapper, nodeRefreshIntervalMs, criteria, deserializer); this.clientConfig = clientConfig; @@ -59,8 +59,8 @@ public ShardedRangerHttpHubClient( } @Override - protected ServiceFinderHub> buildHub() { - return new HttpServiceFinderHubBuilder>() + protected ServiceFinderHub> buildHub() { + return new HttpServiceFinderHubBuilder>() .withServiceDataSource(buildServiceDataSource()) .withServiceFinderFactory(buildFinderFactory()) .withRefreshFrequencyMs(getNodeRefreshTimeMs()) @@ -79,8 +79,8 @@ protected ServiceDataSource buildServiceDataSource() { } @Override - protected ServiceFinderFactory> buildFinderFactory() { - return HttpShardedServiceFinderFactory.builder() + protected ServiceFinderFactory> buildFinderFactory() { + return HttpShardedServiceFinderFactory.builder() .httpClientConfig(clientConfig) .nodeRefreshIntervalMs(getNodeRefreshTimeMs()) .deserializer(getDeserializer()) diff --git a/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/SimpleRangerHttpClient.java b/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/SimpleRangerHttpClient.java index edc6142a..a0c9cb36 100644 --- a/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/SimpleRangerHttpClient.java +++ b/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/SimpleRangerHttpClient.java @@ -19,7 +19,6 @@ import com.flipkart.ranger.client.RangerClient; import com.flipkart.ranger.core.finder.SimpleUnshardedServiceFinder; import com.flipkart.ranger.core.finder.shardselector.ListShardSelector; -import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.http.HttpServiceFinderBuilders; import com.flipkart.ranger.http.config.HttpClientConfig; @@ -30,12 +29,13 @@ import java.util.List; import java.util.Optional; +import java.util.function.Predicate; @Slf4j -public class SimpleRangerHttpClient> implements RangerClient { +public class SimpleRangerHttpClient implements RangerClient { - private final C criteria; - private final SimpleUnshardedServiceFinder serviceFinder; + private final Predicate criteria; + private final SimpleUnshardedServiceFinder serviceFinder; @Builder public SimpleRangerHttpClient( @@ -44,7 +44,7 @@ public SimpleRangerHttpClient( ObjectMapper mapper, int nodeRefreshIntervalMs, HttpClientConfig clientConfig, - C criteria, + Predicate criteria, HTTPResponseDataDeserializer deserializer ) { Preconditions.checkNotNull(mapper, "Mapper can't be null"); @@ -53,7 +53,7 @@ public SimpleRangerHttpClient( this.criteria = criteria; - this.serviceFinder = HttpServiceFinderBuilders.httpUnshardedServiceFinderBuilider() + this.serviceFinder = HttpServiceFinderBuilders.httpUnshardedServiceFinderBuilider() .withClientConfig(clientConfig) .withServiceName(serviceName) .withNamespace(namespace) @@ -82,18 +82,18 @@ public Optional> getNode() { } @Override - public Optional>> getAllNodes() { + public List> getAllNodes() { return getAllNodes(criteria); } @Override - public Optional> getNode(C criteria) { + public Optional> getNode(Predicate criteria) { return Optional.ofNullable(this.serviceFinder.get(criteria)); } @Override - public Optional>> getAllNodes(C criteria) { - return Optional.ofNullable(this.serviceFinder.getAll(criteria)); + public List> getAllNodes(Predicate criteria) { + return this.serviceFinder.getAll(criteria); } } diff --git a/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/UnshardedRangerHttpHubClient.java b/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/UnshardedRangerHttpHubClient.java index 8897830f..43331a0c 100644 --- a/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/UnshardedRangerHttpHubClient.java +++ b/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/UnshardedRangerHttpHubClient.java @@ -24,7 +24,6 @@ import com.flipkart.ranger.core.finderhub.ServiceFinderFactory; import com.flipkart.ranger.core.finderhub.ServiceFinderHub; import com.flipkart.ranger.core.finderhub.StaticDataSource; -import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.http.config.HttpClientConfig; import com.flipkart.ranger.http.serde.HTTPResponseDataDeserializer; @@ -35,10 +34,11 @@ import lombok.extern.slf4j.Slf4j; import java.util.List; +import java.util.function.Predicate; @Slf4j -public class UnshardedRangerHttpHubClient> - extends AbstractRangerHubClient, HTTPResponseDataDeserializer> { +public class UnshardedRangerHttpHubClient + extends AbstractRangerHubClient, HTTPResponseDataDeserializer> { private final List services; private final HttpClientConfig clientConfig; @@ -48,7 +48,7 @@ public UnshardedRangerHttpHubClient( String namespace, ObjectMapper mapper, int nodeRefreshIntervalMs, - C criteria, + Predicate criteria, HTTPResponseDataDeserializer deserializer, HttpClientConfig clientConfig, List services @@ -59,8 +59,8 @@ public UnshardedRangerHttpHubClient( } @Override - protected ServiceFinderHub> buildHub() { - return new HttpServiceFinderHubBuilder>() + protected ServiceFinderHub> buildHub() { + return new HttpServiceFinderHubBuilder>() .withServiceDataSource(buildServiceDataSource()) .withServiceFinderFactory(buildFinderFactory()) .withRefreshFrequencyMs(getNodeRefreshTimeMs()) @@ -75,8 +75,8 @@ protected ServiceDataSource buildServiceDataSource() { } @Override - protected ServiceFinderFactory> buildFinderFactory() { - return HttpUnshardedServiceFinderFactory.builder() + protected ServiceFinderFactory> buildFinderFactory() { + return HttpUnshardedServiceFinderFactory.builder() .httpClientConfig(clientConfig) .nodeRefreshIntervalMs(getNodeRefreshTimeMs()) .deserializer(getDeserializer()) diff --git a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/BaseRangerHttpClientTest.java b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/BaseRangerHttpClientTest.java index f1e92621..98a9a17b 100644 --- a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/BaseRangerHttpClientTest.java +++ b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/BaseRangerHttpClientTest.java @@ -18,10 +18,10 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.flipkart.ranger.core.healthcheck.HealthcheckStatus; -import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.units.TestNodeData; import com.flipkart.ranger.core.util.Exceptions; +import com.flipkart.ranger.core.utils.RangerTestUtils; import com.flipkart.ranger.http.config.HttpClientConfig; import com.flipkart.ranger.http.model.ServiceDataSourceResponse; import com.flipkart.ranger.http.model.ServiceNodesResponse; @@ -49,8 +49,8 @@ public abstract class BaseRangerHttpClientTest { @Before public void startTestCluster() throws Exception { - final TestNodeData testNode = new TestNodeData(1); - final ServiceNode node = new ServiceNode<>("127.0.0.1", 80, testNode); + val testNode = TestNodeData.builder().shardId(1).build(); + val node = new ServiceNode<>("127.0.0.1", 80, testNode); node.setHealthcheckStatus(HealthcheckStatus.healthy); node.setLastUpdatedTimeStamp(System.currentTimeMillis()); val payload = objectMapper.writeValueAsBytes( @@ -66,7 +66,7 @@ public void startTestCluster() throws Exception { val responseObj = ServiceDataSourceResponse.builder() .success(true) .data(Lists.newArrayList( - new Service("test-n", "test-s") + RangerTestUtils.getService("test-n", "test-s") )) .build(); val response = objectMapper.writeValueAsBytes(responseObj); diff --git a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/ShardedRangerHttpClientTest.java b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/ShardedRangerHttpClientTest.java index 93596836..f09baa09 100644 --- a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/ShardedRangerHttpClientTest.java +++ b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/ShardedRangerHttpClientTest.java @@ -15,11 +15,10 @@ */ package com.flipkart.ranger.client.http; -import com.flipkart.ranger.core.TestUtils; -import com.flipkart.ranger.core.model.Criteria; -import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.units.TestNodeData; +import com.flipkart.ranger.core.utils.RangerTestUtils; +import com.flipkart.ranger.core.utils.TestUtils; import lombok.val; import org.junit.Assert; import org.junit.Test; @@ -30,7 +29,7 @@ public class ShardedRangerHttpClientTest extends BaseRangerHttpClientTest { @Test public void testShardedHttpHubClient(){ - val client = ShardedRangerHttpHubClient.>builder() + val client = ShardedRangerHttpHubClient.builder() .clientConfig(getHttpClientConfig()) .namespace("test-n") .deserializer(this::read) @@ -41,14 +40,14 @@ public void testShardedHttpHubClient(){ TestUtils.sleepForSeconds(6); - val service = new Service("test-n", "test-s"); + val service = RangerTestUtils.getService("test-n", "test-s"); Optional> node = client.getNode(service); Assert.assertTrue(node.isPresent()); - node = client.getNode(service, nodeData -> nodeData.getNodeId() == 1); + node = client.getNode(service, nodeData -> nodeData.getShardId() == 1); Assert.assertTrue(node.isPresent()); - node = client.getNode(service, nodeData -> nodeData.getNodeId() == 2); + node = client.getNode(service, nodeData -> nodeData.getShardId() == 2); Assert.assertFalse(node.isPresent()); client.stop(); diff --git a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/SimpleRangerHttpClientTest.java b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/SimpleRangerHttpClientTest.java index 0ca506ac..92be677f 100644 --- a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/SimpleRangerHttpClientTest.java +++ b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/SimpleRangerHttpClientTest.java @@ -15,10 +15,9 @@ */ package com.flipkart.ranger.client.http; -import com.flipkart.ranger.core.TestUtils; -import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.units.TestNodeData; +import com.flipkart.ranger.core.utils.TestUtils; import lombok.val; import org.junit.Assert; import org.junit.Test; @@ -29,7 +28,7 @@ public class SimpleRangerHttpClientTest extends BaseRangerHttpClientTest{ @Test public void testSimpleHttpRangerClient(){ - val client = SimpleRangerHttpClient.>builder() + val client = SimpleRangerHttpClient.builder() .clientConfig(getHttpClientConfig()) .mapper(getObjectMapper()) .deserializer(this::read) @@ -44,10 +43,10 @@ public void testSimpleHttpRangerClient(){ Optional> node = client.getNode(); Assert.assertTrue(node.isPresent()); - node = client.getNode(nodeData -> nodeData.getNodeId() == 1); + node = client.getNode(nodeData -> nodeData.getShardId() == 1); Assert.assertTrue(node.isPresent()); - node = client.getNode(nodeData -> nodeData.getNodeId() == 2); + node = client.getNode(nodeData -> nodeData.getShardId() == 2); Assert.assertFalse(node.isPresent()); client.stop(); diff --git a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/UnshardedRangerHttpClientTest.java b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/UnshardedRangerHttpClientTest.java index 10fd6017..c07d4224 100644 --- a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/UnshardedRangerHttpClientTest.java +++ b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/UnshardedRangerHttpClientTest.java @@ -15,11 +15,10 @@ */ package com.flipkart.ranger.client.http; -import com.flipkart.ranger.core.TestUtils; -import com.flipkart.ranger.core.model.Criteria; -import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.units.TestNodeData; +import com.flipkart.ranger.core.utils.RangerTestUtils; +import com.flipkart.ranger.core.utils.TestUtils; import lombok.val; import org.junit.Assert; import org.junit.Test; @@ -30,7 +29,7 @@ public class UnshardedRangerHttpClientTest extends BaseRangerHttpClientTest { @Test public void testUnshardedRangerHubClient(){ - val client = UnshardedRangerHttpHubClient.>builder() + val client = UnshardedRangerHttpHubClient.builder() .clientConfig(getHttpClientConfig()) .namespace("test-n") .deserializer(this::read) @@ -41,14 +40,14 @@ public void testUnshardedRangerHubClient(){ TestUtils.sleepForSeconds(6); - val service = new Service("test-n", "test-s"); + val service = RangerTestUtils.getService("test-n", "test-s"); Optional> node = client.getNode(service); Assert.assertTrue(node.isPresent()); - node = client.getNode(service, nodeData -> nodeData.getNodeId() == 1); + node = client.getNode(service, nodeData -> nodeData.getShardId() == 1); Assert.assertTrue(node.isPresent()); - node = client.getNode(service, nodeData -> nodeData.getNodeId() == 2); + node = client.getNode(service, nodeData -> nodeData.getShardId() == 2); Assert.assertFalse(node.isPresent()); client.stop(); diff --git a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/App.java b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/App.java index 009ebe4d..8fd4f451 100644 --- a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/App.java +++ b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/App.java @@ -15,21 +15,10 @@ */ package com.flipkart.ranger.http.server; -import com.codahale.metrics.health.HealthCheck; -import com.flipkart.ranger.client.RangerHubClient; -import com.flipkart.ranger.common.server.ShardInfo; -import com.flipkart.ranger.core.model.Criteria; -import com.flipkart.ranger.http.serde.HTTPResponseDataDeserializer; -import com.flipkart.ranger.http.server.manager.RangerHttpBundleManager; -import com.flipkart.ranger.http.server.util.RangerHttpServerUtils; -import com.flipkart.ranger.server.bundle.RangerServerBundle; +import com.flipkart.ranger.http.server.bundle.HttpServerBundle; import io.dropwizard.Application; import io.dropwizard.setup.Bootstrap; import io.dropwizard.setup.Environment; -import lombok.val; - -import java.util.List; -import java.util.stream.Collectors; public class App extends Application { @@ -39,58 +28,13 @@ public static void main(String[] args) throws Exception { @Override public void initialize(Bootstrap bootstrap) { - + bootstrap.addBundle(new HttpServerBundle()); } - /** - * The reason why rangerServerBundle.run is used here instead of initializing in the initialize method and letting run automatically called. - * - * a) RangerServerBundle's closure contains a rangerHubClient within it, now this client could either be a ZK client or the httpClient. - * b) A ZK client would need the curatorFramework and the http client doesn't need anything. - * c) So the RangerServerBundle couldn't contain a curatorFramework and that has to be defined outside, for that we need the AppConfiguration, which - * is available in the run as an arg to be used, - * d) Also don't want to create two bundle managers (one for http and one for ZK) - * - * Also, the verifyPreconditions is different for both zk and http. - * - * @param appConfiguration {@link AppConfiguration} The appConfiguration provided - * @param environment {@link Environment} The environment in scope. - */ @Override public void run(AppConfiguration appConfiguration, Environment environment) { - val rangerConfiguration = appConfiguration.getRangerConfiguration(); - RangerHttpServerUtils.verifyPreconditions(rangerConfiguration); - - val rangerServerBundle = new RangerServerBundle, - AppConfiguration>() { - - @Override - protected List>> withHubs(AppConfiguration configuration) { - return configuration.getRangerConfiguration().getHttpClientConfigs().stream().map(clientConfig -> - RangerHttpServerUtils.buildRangerHub( - rangerConfiguration.getNamespace(), - rangerConfiguration.getNodeRefreshTimeMs(), - clientConfig, - environment.getObjectMapper() - )).collect(Collectors.toList()); - } - - @Override - protected boolean withInitialRotationStatus(AppConfiguration configuration) { - return appConfiguration.isInitialRotationStatus(); - } - }; - rangerServerBundle.run(appConfiguration, environment); - - val rangerClientManager = new RangerHttpBundleManager(rangerServerBundle); - environment.lifecycle().manage(rangerClientManager); - environment.healthChecks().register( - "ranger-http-health-check", - new HealthCheck() { - @Override - protected Result check() { - return Result.healthy(); - } - }); + /* + Nothing to do run! + */ } } diff --git a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/AppConfiguration.java b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/AppConfiguration.java index 2b93198e..91998166 100644 --- a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/AppConfiguration.java +++ b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/AppConfiguration.java @@ -18,29 +18,25 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.flipkart.ranger.http.server.config.RangerHttpConfiguration; import io.dropwizard.Configuration; -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; +import lombok.Builder; +import lombok.Value; +import lombok.extern.jackson.Jacksonized; import javax.validation.Valid; import javax.validation.constraints.NotEmpty; import javax.validation.constraints.NotNull; -@Getter -@Setter -@AllArgsConstructor -@NoArgsConstructor +@Value +@Jacksonized +@Builder @JsonIgnoreProperties(ignoreUnknown = true) public class AppConfiguration extends Configuration { @NotEmpty @NotNull - private String name; - + String name; @Valid @NotNull - private RangerHttpConfiguration rangerConfiguration; - - private boolean initialRotationStatus; + RangerHttpConfiguration rangerConfiguration; + boolean initialRotationStatus; } diff --git a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/bundle/HttpServerBundle.java b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/bundle/HttpServerBundle.java new file mode 100644 index 00000000..cd4ab7ce --- /dev/null +++ b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/bundle/HttpServerBundle.java @@ -0,0 +1,97 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.http.server.bundle; + +import com.codahale.metrics.health.HealthCheck; +import com.fasterxml.jackson.core.type.TypeReference; +import com.flipkart.ranger.client.RangerHubClient; +import com.flipkart.ranger.client.http.UnshardedRangerHttpHubClient; +import com.flipkart.ranger.common.server.ShardInfo; +import com.flipkart.ranger.http.config.HttpClientConfig; +import com.flipkart.ranger.http.model.ServiceNodesResponse; +import com.flipkart.ranger.http.server.AppConfiguration; +import com.flipkart.ranger.zk.server.bundle.RangerServerBundle; +import com.flipkart.ranger.zk.server.bundle.model.LifecycleSignal; +import com.google.common.base.Preconditions; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import lombok.val; + +import javax.inject.Singleton; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +@Slf4j +@Singleton +@NoArgsConstructor +public class HttpServerBundle extends RangerServerBundle { + + @Override + protected void verifyPreconditions(AppConfiguration configuration) { + val rangerConfiguration = configuration.getRangerConfiguration(); + Preconditions.checkNotNull(rangerConfiguration, + "ranger configuration can't be null"); + Preconditions.checkNotNull(rangerConfiguration.getNamespace(), + "Namespace can't be null"); + Preconditions.checkArgument(null != rangerConfiguration.getHttpClientConfigs() && !rangerConfiguration.getHttpClientConfigs().isEmpty(), + "Http client config can't be null"); + } + + @Override + protected void preBundle(AppConfiguration configuration) { + /* + Noop + */ + } + + @Override + protected List> withHubs(AppConfiguration configuration) { + val rangerConfiguration = configuration.getRangerConfiguration(); + return rangerConfiguration.getHttpClientConfigs().stream().map(clientConfig -> UnshardedRangerHttpHubClient.builder() + .namespace(rangerConfiguration.getNamespace()) + .mapper(getMapper()) + .clientConfig(clientConfig) + .nodeRefreshIntervalMs(rangerConfiguration.getNodeRefreshTimeMs()) + .deserializer(data -> { + try { + getMapper().readValue(data, new TypeReference>() { + }); + } catch (IOException e) { + log.warn("Error parsing node data with value {}", new String(data)); + } + return null; + }) + .build()).collect(Collectors.toList()); + } + + @Override + protected boolean withInitialRotationStatus(AppConfiguration configuration) { + return configuration.isInitialRotationStatus(); + } + + @Override + protected List withLifecycleSignals(AppConfiguration configuration) { + return Collections.emptyList(); + } + + @Override + protected List withHealthChecks(AppConfiguration configuration) { + return Collections.emptyList(); + } +} diff --git a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/config/RangerHttpConfiguration.java b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/config/RangerHttpConfiguration.java index 0792b871..de04fd1a 100644 --- a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/config/RangerHttpConfiguration.java +++ b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/config/RangerHttpConfiguration.java @@ -16,25 +16,25 @@ package com.flipkart.ranger.http.server.config; import com.flipkart.ranger.http.config.HttpClientConfig; -import lombok.*; +import lombok.Builder; +import lombok.Value; +import lombok.extern.jackson.Jacksonized; import javax.validation.constraints.NotEmpty; import javax.validation.constraints.NotNull; import java.util.List; -@AllArgsConstructor -@NoArgsConstructor -@Getter -@Setter +@Value @Builder +@Jacksonized public class RangerHttpConfiguration { @NotEmpty @NotNull - private String namespace; + String namespace; @NotEmpty @NotNull - private List httpClientConfigs; - private int nodeRefreshTimeMs; + List httpClientConfigs; + int nodeRefreshTimeMs; } diff --git a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/manager/RangerHttpBundleManager.java b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/manager/RangerHttpBundleManager.java deleted file mode 100644 index 58fd127f..00000000 --- a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/manager/RangerHttpBundleManager.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright 2015 Flipkart Internet Pvt. Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.flipkart.ranger.http.server.manager; - -import com.flipkart.ranger.server.bundle.RangerServerBundle; -import io.dropwizard.lifecycle.Managed; -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; - -import javax.inject.Inject; -import javax.inject.Singleton; - -@Slf4j -@Singleton -@Getter -public class RangerHttpBundleManager implements Managed { - - private final RangerServerBundle rangerServerBundle; - - @Inject - public RangerHttpBundleManager(RangerServerBundle rangerServerBundle){ - this.rangerServerBundle = rangerServerBundle; - } - - @Override - public void start() { - log.info("Starting the ranger http bundle manager"); - rangerServerBundle.start(); - log.info("Started the ranger http bundle manager"); - } - - @Override - public void stop() { - log.info("Stopping the ranger http bundle manager"); - rangerServerBundle.stop(); - log.info("Stopped the ranger http bundle manager"); - } -} diff --git a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/util/RangerHttpServerUtils.java b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/util/RangerHttpServerUtils.java deleted file mode 100644 index 64f14164..00000000 --- a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/util/RangerHttpServerUtils.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright 2015 Flipkart Internet Pvt. Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.flipkart.ranger.http.server.util; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.flipkart.ranger.client.http.UnshardedRangerHttpHubClient; -import com.flipkart.ranger.common.server.ShardInfo; -import com.flipkart.ranger.core.model.Criteria; -import com.flipkart.ranger.http.config.HttpClientConfig; -import com.flipkart.ranger.http.model.ServiceNodesResponse; -import com.flipkart.ranger.http.server.config.RangerHttpConfiguration; -import com.google.common.base.Preconditions; -import lombok.extern.slf4j.Slf4j; - -import java.io.IOException; - -@Slf4j -public class RangerHttpServerUtils { - - private RangerHttpServerUtils(){} - - public static void verifyPreconditions(RangerHttpConfiguration rangerConfiguration){ - Preconditions.checkNotNull(rangerConfiguration, - "ranger configuration can't be null"); - Preconditions.checkNotNull(rangerConfiguration.getNamespace(), - "Namespace can't be null"); - Preconditions.checkArgument(null != rangerConfiguration.getHttpClientConfigs() && !rangerConfiguration.getHttpClientConfigs().isEmpty(), - "Http client config can't be null"); - } - - public static UnshardedRangerHttpHubClient> buildRangerHub( - String namespace, - int nodeRefreshTimeMs, - HttpClientConfig clientConfig, - ObjectMapper mapper - ){ - return UnshardedRangerHttpHubClient.>builder() - .namespace(namespace) - .mapper(mapper) - .clientConfig(clientConfig) - .nodeRefreshIntervalMs(nodeRefreshTimeMs) - .deserializer(data -> { - try { - mapper.readValue(data, new TypeReference>() { - }); - } catch (IOException e) { - log.warn("Error parsing node data with value {}", new String(data)); - } - return null; - }) - .build(); - } -} diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/HttpServiceFinderBuilders.java b/ranger-http/src/main/java/com/flipkart/ranger/http/HttpServiceFinderBuilders.java index f578937d..85e51aed 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/HttpServiceFinderBuilders.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/HttpServiceFinderBuilders.java @@ -15,21 +15,20 @@ */ package com.flipkart.ranger.http; -import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.http.servicefinder.HttpShardedServiceFinderBuilder; import com.flipkart.ranger.http.servicefinder.HttpUnshardedServiceFinderBuilider; -public class HttpServiceFinderBuilders { +public final class HttpServiceFinderBuilders { private void HttpServiceProviderBuilders(){ throw new InstantiationError("Must not instantiate this class"); } - public static > HttpShardedServiceFinderBuilder httpShardedServiceFinderBuilder(){ + public static HttpShardedServiceFinderBuilder httpShardedServiceFinderBuilder(){ return new HttpShardedServiceFinderBuilder<>(); } - public static > HttpUnshardedServiceFinderBuilider httpUnshardedServiceFinderBuilider(){ + public static HttpUnshardedServiceFinderBuilider httpUnshardedServiceFinderBuilider(){ return new HttpUnshardedServiceFinderBuilider<>(); } } diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/HttpServiceProviderBuilders.java b/ranger-http/src/main/java/com/flipkart/ranger/http/HttpServiceProviderBuilders.java index 605b3b36..0e0fc6af 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/HttpServiceProviderBuilders.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/HttpServiceProviderBuilders.java @@ -17,7 +17,7 @@ import com.flipkart.ranger.http.serviceprovider.HttpShardedServiceProviderBuilder; -public class HttpServiceProviderBuilders { +public final class HttpServiceProviderBuilders { private HttpServiceProviderBuilders() { throw new InstantiationError("Must not instantiate this class"); diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/config/HttpClientConfig.java b/ranger-http/src/main/java/com/flipkart/ranger/http/config/HttpClientConfig.java index bf1f2f4d..06f6e96d 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/config/HttpClientConfig.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/config/HttpClientConfig.java @@ -15,21 +15,21 @@ */ package com.flipkart.ranger.http.config; -import lombok.*; +import lombok.Builder; +import lombok.Value; +import lombok.extern.jackson.Jacksonized; /** * */ -@Data -@AllArgsConstructor +@Value @Builder -@ToString -@NoArgsConstructor +@Jacksonized public class HttpClientConfig { - private String host; - private int port; - private boolean secure; - private long connectionTimeoutMs; - private long operationTimeoutMs; - private long refreshIntervalMillis; + String host; + int port; + boolean secure; + long connectionTimeoutMs; + long operationTimeoutMs; + long refreshIntervalMillis; } diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceDataSourceResponse.java b/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceDataSourceResponse.java index 33b378a8..6966b729 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceDataSourceResponse.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceDataSourceResponse.java @@ -17,18 +17,19 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.flipkart.ranger.core.model.Service; -import lombok.*; +import lombok.Builder; +import lombok.Value; +import lombok.extern.jackson.Jacksonized; import java.util.List; -@AllArgsConstructor -@NoArgsConstructor -@Getter -@Setter + +@Value +@Jacksonized @Builder @JsonIgnoreProperties(ignoreUnknown = true) public class ServiceDataSourceResponse { - private boolean success; - private String error; - private List data; + boolean success; + String error; + List data; } diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceNodesResponse.java b/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceNodesResponse.java index 5404217c..591c8985 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceNodesResponse.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceNodesResponse.java @@ -17,22 +17,21 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.flipkart.ranger.core.model.ServiceNode; -import lombok.*; +import lombok.Builder; +import lombok.Value; +import lombok.extern.jackson.Jacksonized; -import java.util.Collections; import java.util.List; /** * */ -@AllArgsConstructor -@NoArgsConstructor -@Getter -@Setter +@Value +@Jacksonized @Builder @JsonIgnoreProperties(ignoreUnknown = true) public class ServiceNodesResponse { - private boolean success; - private String error; - private List> data = Collections.emptyList(); + boolean success; + String error; + List> data; } diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceRegistrationResponse.java b/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceRegistrationResponse.java index 61594913..f7fe0bcc 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceRegistrationResponse.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceRegistrationResponse.java @@ -16,16 +16,16 @@ package com.flipkart.ranger.http.model; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import lombok.*; +import lombok.Builder; +import lombok.Value; +import lombok.extern.jackson.Jacksonized; -@AllArgsConstructor -@NoArgsConstructor -@Getter -@Setter +@Value +@Jacksonized @Builder @JsonIgnoreProperties(ignoreUnknown = true) public class ServiceRegistrationResponse { - private boolean success; - private String error; + boolean success; + String error; } diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpNodeDataSource.java b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpNodeDataSource.java index 507c7a0c..28782434 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpNodeDataSource.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpNodeDataSource.java @@ -19,7 +19,6 @@ import com.flipkart.ranger.core.model.NodeDataSource; import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.core.model.ServiceNode; -import com.flipkart.ranger.core.util.Exceptions; import com.flipkart.ranger.core.util.FinderUtils; import com.flipkart.ranger.http.common.HttpNodeDataStoreConnector; import com.flipkart.ranger.http.config.HttpClientConfig; @@ -34,7 +33,6 @@ import java.io.IOException; import java.util.Collections; import java.util.List; -import java.util.Optional; /** * @@ -53,7 +51,7 @@ public HttpNodeDataSource( } @Override - public Optional>> refresh(D deserializer) { + public List> refresh(D deserializer) { Preconditions.checkNotNull(config, "client config has not been set for node data"); Preconditions.checkNotNull(mapper, "mapper has not been set for node data"); val httpUrl = new HttpUrl.Builder() @@ -96,7 +94,7 @@ public Optional>> refresh(D deserializer) { } catch (IOException e) { log.error("Error getting service data from the http endPoint: ", e); } - return Optional.of(serviceNodes); + return serviceNodes; } @Override diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilder.java b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilder.java index bdb3b487..8e7927ec 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilder.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilder.java @@ -18,7 +18,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.flipkart.ranger.core.finder.SimpleShardedServiceFinder; import com.flipkart.ranger.core.finder.SimpleShardedServiceFinderBuilder; -import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.NodeDataSource; import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.http.config.HttpClientConfig; @@ -27,23 +26,23 @@ /** * */ -public class HttpShardedServiceFinderBuilder> extends SimpleShardedServiceFinderBuilder, HTTPResponseDataDeserializer, C> { +public class HttpShardedServiceFinderBuilder extends SimpleShardedServiceFinderBuilder, HTTPResponseDataDeserializer> { private HttpClientConfig clientConfig; private ObjectMapper mapper; - public HttpShardedServiceFinderBuilder withClientConfig(final HttpClientConfig clientConfig) { + public HttpShardedServiceFinderBuilder withClientConfig(final HttpClientConfig clientConfig) { this.clientConfig = clientConfig; return this; } - public HttpShardedServiceFinderBuilder withObjectMapper(final ObjectMapper mapper){ + public HttpShardedServiceFinderBuilder withObjectMapper(final ObjectMapper mapper){ this.mapper = mapper; return this; } @Override - public SimpleShardedServiceFinder build() { + public SimpleShardedServiceFinder build() { return buildFinder(); } diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpUnshardedServiceFinderBuilider.java b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpUnshardedServiceFinderBuilider.java index fbc1ece8..a7ca539e 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpUnshardedServiceFinderBuilider.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpUnshardedServiceFinderBuilider.java @@ -18,30 +18,29 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.flipkart.ranger.core.finder.SimpleUnshardedServiceFinder; import com.flipkart.ranger.core.finder.SimpleUnshardedServiceFinderBuilder; -import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.NodeDataSource; import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.http.config.HttpClientConfig; import com.flipkart.ranger.http.serde.HTTPResponseDataDeserializer; -public class HttpUnshardedServiceFinderBuilider> - extends SimpleUnshardedServiceFinderBuilder, HTTPResponseDataDeserializer, C> { +public class HttpUnshardedServiceFinderBuilider + extends SimpleUnshardedServiceFinderBuilder, HTTPResponseDataDeserializer> { private HttpClientConfig clientConfig; private ObjectMapper mapper; - public HttpUnshardedServiceFinderBuilider withClientConfig(final HttpClientConfig clientConfig) { + public HttpUnshardedServiceFinderBuilider withClientConfig(final HttpClientConfig clientConfig) { this.clientConfig = clientConfig; return this; } - public HttpUnshardedServiceFinderBuilider withObjectMapper(final ObjectMapper mapper) { + public HttpUnshardedServiceFinderBuilider withObjectMapper(final ObjectMapper mapper) { this.mapper = mapper; return this; } @Override - public SimpleUnshardedServiceFinder build() { + public SimpleUnshardedServiceFinder build() { return buildFinder(); } diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceFinderHubBuilder.java b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceFinderHubBuilder.java index 96ccf5c4..5bbc4fc9 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceFinderHubBuilder.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceFinderHubBuilder.java @@ -17,12 +17,11 @@ import com.flipkart.ranger.core.finderhub.ServiceFinderHub; import com.flipkart.ranger.core.finderhub.ServiceFinderHubBuilder; -import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.ServiceRegistry; import lombok.extern.slf4j.Slf4j; @Slf4j -public class HttpServiceFinderHubBuilder, R extends ServiceRegistry> extends ServiceFinderHubBuilder { +public class HttpServiceFinderHubBuilder> extends ServiceFinderHubBuilder { @Override protected void preBuild() { @@ -30,7 +29,7 @@ protected void preBuild() { } @Override - protected void postBuild(ServiceFinderHub serviceFinderHub) { + protected void postBuild(ServiceFinderHub serviceFinderHub) { log.info("No post build actions necessary"); } } diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpShardedServiceFinderFactory.java b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpShardedServiceFinderFactory.java index b4805de1..3a03b6bf 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpShardedServiceFinderFactory.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpShardedServiceFinderFactory.java @@ -19,7 +19,6 @@ import com.flipkart.ranger.core.finder.ServiceFinder; import com.flipkart.ranger.core.finder.serviceregistry.MapBasedServiceRegistry; import com.flipkart.ranger.core.finderhub.ServiceFinderFactory; -import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.core.model.ServiceNodeSelector; import com.flipkart.ranger.core.model.ShardSelector; @@ -31,12 +30,12 @@ import lombok.val; @Getter -public class HttpShardedServiceFinderFactory > implements ServiceFinderFactory> { +public class HttpShardedServiceFinderFactory implements ServiceFinderFactory> { private final HttpClientConfig clientConfig; private final ObjectMapper mapper; private final HTTPResponseDataDeserializer deserializer; - private final ShardSelector> shardSelector; + private final ShardSelector> shardSelector; private final ServiceNodeSelector nodeSelector; private final int nodeRefreshIntervalMs; @@ -45,7 +44,7 @@ public HttpShardedServiceFinderFactory( HttpClientConfig httpClientConfig, ObjectMapper mapper, HTTPResponseDataDeserializer deserializer, - ShardSelector> shardSelector, + ShardSelector> shardSelector, ServiceNodeSelector nodeSelector, int nodeRefreshIntervalMs) { @@ -58,8 +57,8 @@ public HttpShardedServiceFinderFactory( } @Override - public ServiceFinder> buildFinder(Service service) { - val serviceFinder = new HttpShardedServiceFinderBuilder() + public ServiceFinder> buildFinder(Service service) { + val serviceFinder = new HttpShardedServiceFinderBuilder() .withClientConfig(clientConfig) .withObjectMapper(mapper) .withDeserializer(deserializer) diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpUnshardedServiceFinderFactory.java b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpUnshardedServiceFinderFactory.java index eeb63c76..f785fc60 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpUnshardedServiceFinderFactory.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpUnshardedServiceFinderFactory.java @@ -19,7 +19,6 @@ import com.flipkart.ranger.core.finder.ServiceFinder; import com.flipkart.ranger.core.finder.serviceregistry.ListBasedServiceRegistry; import com.flipkart.ranger.core.finderhub.ServiceFinderFactory; -import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.core.model.ServiceNodeSelector; import com.flipkart.ranger.core.model.ShardSelector; @@ -31,12 +30,12 @@ import lombok.val; @Getter -public class HttpUnshardedServiceFinderFactory> implements ServiceFinderFactory> { +public class HttpUnshardedServiceFinderFactory implements ServiceFinderFactory> { private final HttpClientConfig clientConfig; private final ObjectMapper mapper; private final HTTPResponseDataDeserializer deserializer; - private final ShardSelector> shardSelector; + private final ShardSelector> shardSelector; private final ServiceNodeSelector nodeSelector; private final int nodeRefreshIntervalMs; @@ -45,7 +44,7 @@ public HttpUnshardedServiceFinderFactory( HttpClientConfig httpClientConfig, ObjectMapper mapper, HTTPResponseDataDeserializer deserializer, - ShardSelector> shardSelector, + ShardSelector> shardSelector, ServiceNodeSelector nodeSelector, int nodeRefreshIntervalMs) { @@ -58,8 +57,8 @@ public HttpUnshardedServiceFinderFactory( } @Override - public ServiceFinder> buildFinder(Service service) { - val serviceFinder = new HttpUnshardedServiceFinderBuilider() + public ServiceFinder> buildFinder(Service service) { + val serviceFinder = new HttpUnshardedServiceFinderBuilider() .withClientConfig(clientConfig) .withObjectMapper(mapper) .withDeserializer(deserializer) diff --git a/ranger-http/src/test/java/com/flipkart/ranger/http/ResourceHelper.java b/ranger-http/src/test/java/com/flipkart/ranger/http/ResourceHelper.java index 94dcacf0..678761bf 100644 --- a/ranger-http/src/test/java/com/flipkart/ranger/http/ResourceHelper.java +++ b/ranger-http/src/test/java/com/flipkart/ranger/http/ResourceHelper.java @@ -17,9 +17,9 @@ import com.fasterxml.jackson.databind.ObjectMapper; import lombok.SneakyThrows; +import lombok.val; import java.io.BufferedReader; -import java.io.InputStream; import java.io.InputStreamReader; import java.util.stream.Collectors; @@ -28,7 +28,8 @@ public class ResourceHelper { private static final ObjectMapper objectMapper = new ObjectMapper(); public static String getResource(String path) { - final InputStream data = ResourceHelper.class.getClassLoader().getResourceAsStream(path); + val data = ResourceHelper.class.getClassLoader().getResourceAsStream(path); + if(null == data) return null; return new BufferedReader( new InputStreamReader(data)) .lines() @@ -37,7 +38,8 @@ public static String getResource(String path) { @SneakyThrows public static T getResource(String path, Class klass) { - final String data = getResource(path); + val data = getResource(path); + if(null == data) return null; return objectMapper.readValue(data, klass); } diff --git a/ranger-http/src/test/java/com/flipkart/ranger/http/common/HttpNodeDataStoreConnectorTest.java b/ranger-http/src/test/java/com/flipkart/ranger/http/common/HttpNodeDataStoreConnectorTest.java index 7a24e70b..678fe37b 100644 --- a/ranger-http/src/test/java/com/flipkart/ranger/http/common/HttpNodeDataStoreConnectorTest.java +++ b/ranger-http/src/test/java/com/flipkart/ranger/http/common/HttpNodeDataStoreConnectorTest.java @@ -17,6 +17,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.flipkart.ranger.http.config.HttpClientConfig; +import lombok.val; import org.junit.Assert; import org.junit.Test; @@ -24,12 +25,12 @@ public class HttpNodeDataStoreConnectorTest { @Test public void testHttpNodeDataStoreConnector(){ - final ObjectMapper objectMapper = new ObjectMapper(); - final HttpClientConfig httpClientConfig = HttpClientConfig.builder() + val objectMapper = new ObjectMapper(); + val httpClientConfig = HttpClientConfig.builder() .host("localhost-1") .port(80) .build(); - HttpNodeDataStoreConnector httpNodeDataStoreConnector = new HttpNodeDataStoreConnector(httpClientConfig, objectMapper); + val httpNodeDataStoreConnector = new HttpNodeDataStoreConnector<>(httpClientConfig, objectMapper); Assert.assertNotNull(httpNodeDataStoreConnector); Assert.assertTrue(httpNodeDataStoreConnector.isActive()); } diff --git a/ranger-http/src/test/java/com/flipkart/ranger/http/config/HttpClientConfigTest.java b/ranger-http/src/test/java/com/flipkart/ranger/http/config/HttpClientConfigTest.java index b1b0f68f..26b031b6 100644 --- a/ranger-http/src/test/java/com/flipkart/ranger/http/config/HttpClientConfigTest.java +++ b/ranger-http/src/test/java/com/flipkart/ranger/http/config/HttpClientConfigTest.java @@ -16,6 +16,7 @@ package com.flipkart.ranger.http.config; import com.flipkart.ranger.http.ResourceHelper; +import lombok.val; import org.junit.Assert; import org.junit.Test; @@ -23,7 +24,7 @@ public class HttpClientConfigTest { @Test public void testHttpClientConfig(){ - HttpClientConfig resource = ResourceHelper.getResource("fixtures/httpClientConfig.json", HttpClientConfig.class); + val resource = ResourceHelper.getResource("fixtures/httpClientConfig.json", HttpClientConfig.class); Assert.assertEquals("localhost-1", resource.getHost()); Assert.assertEquals(80, resource.getPort()); Assert.assertEquals(10, resource.getConnectionTimeoutMs()); diff --git a/ranger-http/src/test/java/com/flipkart/ranger/http/model/ServiceNodeResponseTest.java b/ranger-http/src/test/java/com/flipkart/ranger/http/model/ServiceNodeResponseTest.java index c3887954..3c29f8bd 100644 --- a/ranger-http/src/test/java/com/flipkart/ranger/http/model/ServiceNodeResponseTest.java +++ b/ranger-http/src/test/java/com/flipkart/ranger/http/model/ServiceNodeResponseTest.java @@ -17,19 +17,21 @@ import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.http.ResourceHelper; -import lombok.*; +import lombok.Builder; +import lombok.Value; +import lombok.extern.jackson.Jacksonized; +import lombok.val; import org.junit.Assert; import org.junit.Test; public class ServiceNodeResponseTest { - @AllArgsConstructor - @NoArgsConstructor - @Getter - @Setter + @Value + @Builder + @Jacksonized static class TestNodeInfo{ - private int shardId; - private String region; + int shardId; + String region; } @Test diff --git a/ranger-http/src/test/java/com/flipkart/ranger/http/model/ServiceRegistrationResponseTest.java b/ranger-http/src/test/java/com/flipkart/ranger/http/model/ServiceRegistrationResponseTest.java index 9949a859..c56b1e14 100644 --- a/ranger-http/src/test/java/com/flipkart/ranger/http/model/ServiceRegistrationResponseTest.java +++ b/ranger-http/src/test/java/com/flipkart/ranger/http/model/ServiceRegistrationResponseTest.java @@ -16,6 +16,7 @@ package com.flipkart.ranger.http.model; import com.flipkart.ranger.http.ResourceHelper; +import lombok.val; import org.junit.Assert; import org.junit.Test; @@ -23,7 +24,7 @@ public class ServiceRegistrationResponseTest { @Test public void testServiceRegistrationResponse(){ - ServiceRegistrationResponse resource = ResourceHelper.getResource("fixtures/serviceResponse.json", ServiceRegistrationResponse.class); + val resource = ResourceHelper.getResource("fixtures/serviceResponse.json", ServiceRegistrationResponse.class); Assert.assertNotNull(resource); Assert.assertTrue(resource.isSuccess()); } diff --git a/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java b/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java index fb9f5202..08bb01ae 100644 --- a/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java +++ b/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java @@ -18,11 +18,9 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import com.flipkart.ranger.core.TestUtils; -import com.flipkart.ranger.core.finder.SimpleShardedServiceFinder; import com.flipkart.ranger.core.healthcheck.HealthcheckStatus; -import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.ServiceNode; +import com.flipkart.ranger.core.utils.TestUtils; import com.flipkart.ranger.http.config.HttpClientConfig; import com.flipkart.ranger.http.model.ServiceNodesResponse; import com.github.tomakehurst.wiremock.junit.WireMockRule; @@ -58,8 +56,8 @@ public NodeData(@JsonProperty("name") String name) { @Test public void testFinder() throws Exception { - final NodeData testNode = new NodeData("testNode"); - final ServiceNode node = new ServiceNode<>("127.0.0.1", 80, testNode); + val testNode = new NodeData("testNode"); + val node = new ServiceNode<>("127.0.0.1", 80, testNode); node.setHealthcheckStatus(HealthcheckStatus.healthy); node.setLastUpdatedTimeStamp(System.currentTimeMillis()); val payload = MAPPER.writeValueAsBytes( @@ -78,7 +76,7 @@ public void testFinder() throws Exception { .operationTimeoutMs(30_000) .build(); - final SimpleShardedServiceFinder> finder = new HttpShardedServiceFinderBuilder>() + val finder = new HttpShardedServiceFinderBuilder() .withClientConfig(clientConfig) .withNamespace("testns") .withServiceName("test") diff --git a/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSourceTest.java b/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSourceTest.java index 5da90812..d94bbc01 100644 --- a/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSourceTest.java +++ b/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSourceTest.java @@ -17,7 +17,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.ObjectMapper; -import com.flipkart.ranger.core.model.Service; +import com.flipkart.ranger.core.utils.RangerTestUtils; import com.flipkart.ranger.http.config.HttpClientConfig; import com.flipkart.ranger.http.model.ServiceDataSourceResponse; import com.github.tomakehurst.wiremock.junit.WireMockRule; @@ -30,7 +30,6 @@ import org.junit.Test; import java.io.IOException; -import java.util.Collection; import static com.github.tomakehurst.wiremock.client.WireMock.*; @@ -56,9 +55,9 @@ public void testServiceDataSource() throws IOException { val responseObj = ServiceDataSourceResponse.builder() .success(true) .data(Lists.newArrayList( - new Service("test-n", "test-s"), - new Service("test-n", "test-s1"), - new Service("test-n", "test-s2") + RangerTestUtils.getService("test-n", "test-s"), + RangerTestUtils.getService("test-n", "test-s1"), + RangerTestUtils.getService("test-n", "test-s2") )) .build(); val response = MAPPER.writeValueAsBytes(responseObj); @@ -72,8 +71,8 @@ public void testServiceDataSource() throws IOException { .connectionTimeoutMs(30_000) .operationTimeoutMs(30_000) .build(); - HttpServiceDataSource httpServiceDataSource = new HttpServiceDataSource<>(clientConfig, MAPPER); - Collection services = httpServiceDataSource.services(); + val httpServiceDataSource = new HttpServiceDataSource<>(clientConfig, MAPPER); + val services = httpServiceDataSource.services(); Assert.assertNotNull(services); Assert.assertFalse(services.isEmpty()); Assert.assertEquals(3, services.size()); diff --git a/ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/RangerServerBundle.java b/ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/RangerServerBundle.java deleted file mode 100644 index dcdf8b12..00000000 --- a/ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/RangerServerBundle.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright 2015 Flipkart Internet Pvt. Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.flipkart.ranger.server.bundle; - -import com.flipkart.ranger.client.RangerHubClient; -import com.flipkart.ranger.core.model.Criteria; -import com.flipkart.ranger.core.model.Deserializer; -import com.flipkart.ranger.server.bundle.resources.RangerResource; -import com.flipkart.ranger.server.bundle.rotation.BirTask; -import com.flipkart.ranger.server.bundle.rotation.OorTask; -import com.flipkart.ranger.server.bundle.rotation.RotationStatus; -import com.google.common.annotations.VisibleForTesting; -import io.dropwizard.Configuration; -import io.dropwizard.ConfiguredBundle; -import io.dropwizard.setup.Bootstrap; -import io.dropwizard.setup.Environment; -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; - -import java.util.List; - -@Slf4j -public abstract class RangerServerBundle< - T, - C extends Criteria, - U extends Configuration> implements ConfiguredBundle{ - - /** - Why are we taking a list of hubs, instead of one? To be able to aggregate from different dataSources if need be - instead of just fetching from a single dataSource. - - Please check the {@link RangerResource} for how a list is getting aggregated. - - You could also define your custom aggregation by using the {@link RangerHubClient} - */ - @Getter - private List> hubs; - @Getter - @VisibleForTesting - private RotationStatus rotationStatus; - - protected abstract List> withHubs(U configuration); - - protected abstract boolean withInitialRotationStatus(U configuration); - - @Override - public void initialize(Bootstrap bootstrap) { - - } - - @Override - public void run(U configuration, Environment environment) { - hubs = withHubs(configuration); - rotationStatus = new RotationStatus(withInitialRotationStatus(configuration)); - - environment.admin() - .addTask(new OorTask(rotationStatus)); - environment.admin() - .addTask(new BirTask(rotationStatus)); - - environment.jersey().register(new RangerResource<>(hubs)); - } - - public void start(){ - log.info("Starting the ranger hubs"); - hubs.forEach(RangerHubClient::start); - } - - public void stop(){ - log.info("Stopping the ranger hub"); - hubs.forEach(RangerHubClient::stop); - } -} diff --git a/ranger-server-bundle/src/main/java/com/flipkart/ranger/zk/server/bundle/RangerServerBundle.java b/ranger-server-bundle/src/main/java/com/flipkart/ranger/zk/server/bundle/RangerServerBundle.java new file mode 100644 index 00000000..b3567f88 --- /dev/null +++ b/ranger-server-bundle/src/main/java/com/flipkart/ranger/zk/server/bundle/RangerServerBundle.java @@ -0,0 +1,111 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.zk.server.bundle; + +import com.codahale.metrics.health.HealthCheck; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.flipkart.ranger.client.RangerHubClient; +import com.flipkart.ranger.zk.server.bundle.model.LifecycleSignal; +import com.flipkart.ranger.zk.server.bundle.resources.RangerResource; +import com.flipkart.ranger.zk.server.bundle.rotation.BirTask; +import com.flipkart.ranger.zk.server.bundle.rotation.OorTask; +import com.flipkart.ranger.zk.server.bundle.rotation.RotationStatus; +import io.dropwizard.Configuration; +import io.dropwizard.ConfiguredBundle; +import io.dropwizard.lifecycle.Managed; +import io.dropwizard.setup.Bootstrap; +import io.dropwizard.setup.Environment; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import lombok.val; + +import java.util.List; + +@Slf4j +public abstract class RangerServerBundle< + T, + U extends Configuration> implements ConfiguredBundle{ + + /** + Why are we taking a list of hubs, instead of one? To be able to aggregate from different dataSources if need be + instead of just fetching from a single dataSource. + + Please check the {@link RangerResource} for how a list is getting aggregated. + + You could also define your custom aggregation by using the {@link RangerHubClient} + */ + @Getter + private List> hubs; + @Getter + private ObjectMapper mapper; + + protected abstract void verifyPreconditions(U configuration); + + protected abstract void preBundle(U configuration); + + protected abstract List> withHubs(U configuration); + + protected abstract boolean withInitialRotationStatus(U configuration); + + protected abstract List withLifecycleSignals(U configuration); + + protected abstract List withHealthChecks(U configuration); + + @Override + public void initialize(Bootstrap bootstrap) { + /* + Nothing to init here! + */ + } + + @Override + public void run(U configuration, Environment environment) { + verifyPreconditions(configuration); + preBundle(configuration); + + mapper = environment.getObjectMapper(); + hubs = withHubs(configuration); + + val rotationStatus = new RotationStatus(withInitialRotationStatus(configuration)); + val lifecycleSignals = withLifecycleSignals(configuration); + val healthChecks = withHealthChecks(configuration); + + environment.admin() + .addTask(new OorTask(rotationStatus)); + environment.admin() + .addTask(new BirTask(rotationStatus)); + + environment.lifecycle().manage(new Managed() { + @Override + public void start() { + log.info("Starting the server manager"); + lifecycleSignals.forEach(LifecycleSignal::start); + hubs.forEach(RangerHubClient::start); + log.info("Started the server manager"); + } + + @Override + public void stop() { + log.info("Stopping the server manager"); + hubs.forEach(RangerHubClient::stop); + lifecycleSignals.forEach(LifecycleSignal::stop); + log.info("Stopped the server manager"); + } + }); + healthChecks.forEach(healthCheck -> environment.healthChecks().register(healthCheck.getClass().getName(), healthCheck)); + environment.jersey().register(new RangerResource<>(hubs)); + } +} diff --git a/ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/model/GenericResponse.java b/ranger-server-bundle/src/main/java/com/flipkart/ranger/zk/server/bundle/model/GenericResponse.java similarity index 76% rename from ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/model/GenericResponse.java rename to ranger-server-bundle/src/main/java/com/flipkart/ranger/zk/server/bundle/model/GenericResponse.java index e680cf34..32b11222 100644 --- a/ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/model/GenericResponse.java +++ b/ranger-server-bundle/src/main/java/com/flipkart/ranger/zk/server/bundle/model/GenericResponse.java @@ -13,18 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.flipkart.ranger.server.bundle.model; +package com.flipkart.ranger.zk.server.bundle.model; -import lombok.*; +import lombok.Builder; +import lombok.Value; +import lombok.extern.jackson.Jacksonized; +@Value @Builder -@Getter -@Setter -@AllArgsConstructor -@NoArgsConstructor +@Jacksonized public class GenericResponse { - - private boolean success; - private String error; - private T data; + boolean success; + String error; + T data; } diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/model/Criteria.java b/ranger-server-bundle/src/main/java/com/flipkart/ranger/zk/server/bundle/model/LifecycleSignal.java similarity index 80% rename from ranger-core/src/main/java/com/flipkart/ranger/core/model/Criteria.java rename to ranger-server-bundle/src/main/java/com/flipkart/ranger/zk/server/bundle/model/LifecycleSignal.java index a9baf215..f96b591d 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/model/Criteria.java +++ b/ranger-server-bundle/src/main/java/com/flipkart/ranger/zk/server/bundle/model/LifecycleSignal.java @@ -1,23 +1,23 @@ /* * Copyright 2015 Flipkart Internet Pvt. Ltd. - *

+ * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

+ * * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ -package com.flipkart.ranger.core.model; +package com.flipkart.ranger.zk.server.bundle.model; -@FunctionalInterface -public interface Criteria { +public interface LifecycleSignal { - boolean apply(T nodeData); + void start(); + void stop(); } diff --git a/ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/resources/RangerResource.java b/ranger-server-bundle/src/main/java/com/flipkart/ranger/zk/server/bundle/resources/RangerResource.java similarity index 83% rename from ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/resources/RangerResource.java rename to ranger-server-bundle/src/main/java/com/flipkart/ranger/zk/server/bundle/resources/RangerResource.java index 46c6c2f8..d4d91ba1 100644 --- a/ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/resources/RangerResource.java +++ b/ranger-server-bundle/src/main/java/com/flipkart/ranger/zk/server/bundle/resources/RangerResource.java @@ -13,14 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.flipkart.ranger.server.bundle.resources; +package com.flipkart.ranger.zk.server.bundle.resources; import com.codahale.metrics.annotation.Metered; import com.flipkart.ranger.client.RangerHubClient; -import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.core.model.ServiceNode; -import com.flipkart.ranger.server.bundle.model.GenericResponse; +import com.flipkart.ranger.zk.server.bundle.model.GenericResponse; import lombok.extern.slf4j.Slf4j; import lombok.val; @@ -31,19 +30,18 @@ import javax.ws.rs.core.MediaType; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.List; @Slf4j @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) @Path("/ranger") -public class RangerResource> { +public class RangerResource { - private final List> rangerHubs; + private final List> rangerHubs; @Inject - public RangerResource(List> rangerHubs){ + public RangerResource(List> rangerHubs){ this.rangerHubs = rangerHubs; } @@ -72,11 +70,11 @@ public GenericResponse>> getNodes( @NotNull @NotEmpty @PathParam("namespace") final String namespace, @NotNull @NotEmpty @PathParam("serviceName") final String serviceName ){ - val service = new Service(namespace, serviceName); + val service = Service.builder().namespace(namespace).serviceName(serviceName).build(); List> serviceNodes = new ArrayList<>(); rangerHubs.forEach(hub -> { try { - serviceNodes.addAll(hub.getAllNodes(service).orElse(Collections.emptyList())); + serviceNodes.addAll(hub.getAllNodes(service)); } catch (Exception e) { log.warn("Call to a hub failed with exception, {}", e.getMessage()); } diff --git a/ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/rotation/BirTask.java b/ranger-server-bundle/src/main/java/com/flipkart/ranger/zk/server/bundle/rotation/BirTask.java similarity index 95% rename from ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/rotation/BirTask.java rename to ranger-server-bundle/src/main/java/com/flipkart/ranger/zk/server/bundle/rotation/BirTask.java index 78ae414b..a2773eb3 100644 --- a/ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/rotation/BirTask.java +++ b/ranger-server-bundle/src/main/java/com/flipkart/ranger/zk/server/bundle/rotation/BirTask.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.flipkart.ranger.server.bundle.rotation; +package com.flipkart.ranger.zk.server.bundle.rotation; import io.dropwizard.servlets.tasks.Task; import lombok.extern.slf4j.Slf4j; diff --git a/ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/rotation/OorTask.java b/ranger-server-bundle/src/main/java/com/flipkart/ranger/zk/server/bundle/rotation/OorTask.java similarity index 95% rename from ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/rotation/OorTask.java rename to ranger-server-bundle/src/main/java/com/flipkart/ranger/zk/server/bundle/rotation/OorTask.java index f1076d42..298972d2 100644 --- a/ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/rotation/OorTask.java +++ b/ranger-server-bundle/src/main/java/com/flipkart/ranger/zk/server/bundle/rotation/OorTask.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.flipkart.ranger.server.bundle.rotation; +package com.flipkart.ranger.zk.server.bundle.rotation; import io.dropwizard.servlets.tasks.Task; import lombok.extern.slf4j.Slf4j; diff --git a/ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/rotation/RotationStatus.java b/ranger-server-bundle/src/main/java/com/flipkart/ranger/zk/server/bundle/rotation/RotationStatus.java similarity index 95% rename from ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/rotation/RotationStatus.java rename to ranger-server-bundle/src/main/java/com/flipkart/ranger/zk/server/bundle/rotation/RotationStatus.java index ce37a56d..7d632d52 100644 --- a/ranger-server-bundle/src/main/java/com/flipkart/ranger/server/bundle/rotation/RotationStatus.java +++ b/ranger-server-bundle/src/main/java/com/flipkart/ranger/zk/server/bundle/rotation/RotationStatus.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.flipkart.ranger.server.bundle.rotation; +package com.flipkart.ranger.zk.server.bundle.rotation; import java.util.concurrent.atomic.AtomicBoolean; diff --git a/ranger-server-bundle/src/test/java/com/flipkart/ranger/server/bundle/RangerServerBundleTest.java b/ranger-server-bundle/src/test/java/com/flipkart/ranger/zk/server/bundle/RangerServerBundleTest.java similarity index 61% rename from ranger-server-bundle/src/test/java/com/flipkart/ranger/server/bundle/RangerServerBundleTest.java rename to ranger-server-bundle/src/test/java/com/flipkart/ranger/zk/server/bundle/RangerServerBundleTest.java index fb9355fa..a61fe421 100644 --- a/ranger-server-bundle/src/test/java/com/flipkart/ranger/server/bundle/RangerServerBundleTest.java +++ b/ranger-server-bundle/src/test/java/com/flipkart/ranger/zk/server/bundle/RangerServerBundleTest.java @@ -1,15 +1,30 @@ -package com.flipkart.ranger.server.bundle; +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.zk.server.bundle; import com.codahale.metrics.MetricRegistry; +import com.codahale.metrics.health.HealthCheck; import com.fasterxml.jackson.databind.ObjectMapper; import com.flipkart.ranger.client.RangerHubClient; -import com.flipkart.ranger.client.stubs.TestDeserializer; -import com.flipkart.ranger.client.stubs.TestShardInfo; import com.flipkart.ranger.client.utils.RangerHubTestUtils; -import com.flipkart.ranger.core.TestUtils; -import com.flipkart.ranger.core.model.Criteria; -import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.core.model.ServiceNode; +import com.flipkart.ranger.core.units.TestNodeData; +import com.flipkart.ranger.core.utils.RangerTestUtils; +import com.flipkart.ranger.core.utils.TestUtils; +import com.flipkart.ranger.zk.server.bundle.model.LifecycleSignal; import com.google.common.collect.Lists; import io.dropwizard.Configuration; import io.dropwizard.jersey.DropwizardResourceConfig; @@ -25,6 +40,7 @@ import org.junit.Before; import org.junit.Test; +import java.util.Collections; import java.util.List; import java.util.Optional; @@ -41,11 +57,25 @@ public class RangerServerBundleTest { private final Bootstrap bootstrap = mock(Bootstrap.class); private final Configuration configuration = mock(Configuration.class); - private final RangerServerBundle, Configuration> - rangerServerBundle = new RangerServerBundle, Configuration>() { + private final RangerServerBundle + rangerServerBundle = new RangerServerBundle() { @Override - protected List>> withHubs(Configuration configuration) { + protected void verifyPreconditions(Configuration configuration) { + /* + Nothing to do + */ + } + + @Override + protected void preBundle(Configuration configuration) { + /* + Noop + */ + } + + @Override + protected List> withHubs(Configuration configuration) { return Lists.newArrayList(RangerHubTestUtils.getTestHub()); } @@ -53,6 +83,16 @@ protected List>> withHubs protected boolean withInitialRotationStatus(Configuration configuration) { return false; } + + @Override + protected List withLifecycleSignals(Configuration configuration) { + return Collections.emptyList(); + } + + @Override + protected List withHealthChecks(Configuration configuration) { + return Collections.emptyList(); + } }; @Before @@ -67,7 +107,6 @@ public void setup() throws Exception { rangerServerBundle.initialize(bootstrap); rangerServerBundle.run(configuration, environment); - rangerServerBundle.start(); for (LifeCycle lifeCycle : lifecycleEnvironment.getManagedObjects()){ lifeCycle.start(); } @@ -78,19 +117,19 @@ public void setup() throws Exception { public void testRangerBundle(){ TestUtils.sleepForSeconds(3); val hub = rangerServerBundle.getHubs().get(0); - Optional> node = hub.getNode(service); + Optional> node = hub.getNode(service); Assert.assertTrue(node.isPresent()); Assert.assertTrue(node.get().getHost().equalsIgnoreCase("localhost")); Assert.assertEquals(9200, node.get().getPort()); Assert.assertEquals(1, node.get().getNodeData().getShardId()); - node = hub.getNode(new Service("test", "test")); + node = hub.getNode(RangerTestUtils.getService("test", "test")); Assert.assertFalse(node.isPresent()); node = hub.getNode(service, nodeData -> nodeData.getShardId() == 2); Assert.assertFalse(node.isPresent()); - node = hub.getNode(new Service("test", "test"), nodeData -> nodeData.getShardId() == 1); + node = hub.getNode(RangerTestUtils.getService("test", "test"), nodeData -> nodeData.getShardId() == 1); Assert.assertFalse(node.isPresent()); } @@ -99,6 +138,5 @@ public void tearDown() throws Exception { for (LifeCycle lifeCycle: lifecycleEnvironment.getManagedObjects()){ lifeCycle.stop(); } - rangerServerBundle.stop(); } } diff --git a/ranger-server-common/src/main/java/com/flipkart/ranger/common/server/ShardInfo.java b/ranger-server-common/src/main/java/com/flipkart/ranger/common/server/ShardInfo.java index ca1ec17a..5377b9de 100644 --- a/ranger-server-common/src/main/java/com/flipkart/ranger/common/server/ShardInfo.java +++ b/ranger-server-common/src/main/java/com/flipkart/ranger/common/server/ShardInfo.java @@ -16,41 +16,18 @@ package com.flipkart.ranger.common.server; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import lombok.*; - -import java.util.Objects; +import lombok.Builder; +import lombok.Value; +import lombok.extern.jackson.Jacksonized; /* An example nodeData with which we have written our ranger-servers. When you write your own servers you could define your own node data! */ -@AllArgsConstructor -@NoArgsConstructor -@Getter -@Setter +@Value @Builder +@Jacksonized @JsonIgnoreProperties(ignoreUnknown = true) public class ShardInfo { - private String environment; - private String region; - - @Override - public boolean equals(Object o) { - if (o == this) - return true; - - if (!(o instanceof ShardInfo)) - return false; - - ShardInfo other = (ShardInfo) o; - return Objects.equals(this.getEnvironment(), other.getEnvironment()) - && Objects.equals(this.getRegion(), other.getRegion()); - } - - @Override - public int hashCode() { - int hash = 5; - hash = hash + (this.environment != null ? this.environment.hashCode() : 0); - hash = hash + (this.region != null ? this.region.hashCode() : 0); - return hash; - } + String environment; + String region; } diff --git a/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/AbstractRangerZKHubClient.java b/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/AbstractRangerZKHubClient.java index 5dac14e2..7d44b6cc 100644 --- a/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/AbstractRangerZKHubClient.java +++ b/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/AbstractRangerZKHubClient.java @@ -20,7 +20,6 @@ import com.flipkart.ranger.core.finderhub.ServiceDataSource; import com.flipkart.ranger.core.finderhub.ServiceFinderHub; import com.flipkart.ranger.core.finderhub.StaticDataSource; -import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.core.model.ServiceRegistry; import com.flipkart.ranger.zookeeper.serde.ZkNodeDataDeserializer; @@ -31,11 +30,12 @@ import org.apache.curator.framework.CuratorFramework; import java.util.List; +import java.util.function.Predicate; @Slf4j @Getter -public abstract class AbstractRangerZKHubClient, R extends ServiceRegistry, D extends ZkNodeDataDeserializer> - extends AbstractRangerHubClient { +public abstract class AbstractRangerZKHubClient, D extends ZkNodeDataDeserializer> + extends AbstractRangerHubClient { private final boolean disablePushUpdaters; private final String connectionString; @@ -53,7 +53,7 @@ public AbstractRangerZKHubClient( boolean disablePushUpdaters, String connectionString, CuratorFramework curatorFramework, - C criteria, + Predicate criteria, D deserializer, List services ) { @@ -66,8 +66,8 @@ public AbstractRangerZKHubClient( } @Override - protected ServiceFinderHub buildHub() { - return new ZkServiceFinderHubBuilder() + protected ServiceFinderHub buildHub() { + return new ZkServiceFinderHubBuilder() .withCuratorFramework(curatorFramework) .withConnectionString(connectionString) .withNamespace(getNamespace()) diff --git a/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/ShardedRangerZKHubClient.java b/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/ShardedRangerZKHubClient.java index 6b0141a8..b31c3aee 100644 --- a/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/ShardedRangerZKHubClient.java +++ b/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/ShardedRangerZKHubClient.java @@ -20,7 +20,6 @@ import com.flipkart.ranger.core.finder.serviceregistry.MapBasedServiceRegistry; import com.flipkart.ranger.core.finder.shardselector.MatchingShardSelector; import com.flipkart.ranger.core.finderhub.ServiceFinderFactory; -import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.zookeeper.serde.ZkNodeDataDeserializer; import com.flipkart.ranger.zookeeper.servicefinderhub.ZkShardedServiceFinderFactory; @@ -29,10 +28,11 @@ import org.apache.curator.framework.CuratorFramework; import java.util.List; +import java.util.function.Predicate; @Slf4j -public class ShardedRangerZKHubClient> - extends AbstractRangerZKHubClient, ZkNodeDataDeserializer> { +public class ShardedRangerZKHubClient + extends AbstractRangerZKHubClient, ZkNodeDataDeserializer> { @Builder public ShardedRangerZKHubClient( @@ -42,7 +42,7 @@ public ShardedRangerZKHubClient( boolean disablePushUpdaters, String connectionString, CuratorFramework curatorFramework, - C criteria, + Predicate criteria, ZkNodeDataDeserializer deserializer, List services ) { @@ -60,8 +60,8 @@ public ShardedRangerZKHubClient( } @Override - protected ServiceFinderFactory> buildFinderFactory() { - return ZkShardedServiceFinderFactory.builder() + protected ServiceFinderFactory> buildFinderFactory() { + return ZkShardedServiceFinderFactory.builder() .curatorFramework(getCuratorFramework()) .connectionString(getConnectionString()) .nodeRefreshIntervalMs(getNodeRefreshTimeMs()) diff --git a/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/SimpleRangerZKClient.java b/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/SimpleRangerZKClient.java index 2e16cefa..c8e67bf5 100644 --- a/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/SimpleRangerZKClient.java +++ b/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/SimpleRangerZKClient.java @@ -20,7 +20,6 @@ import com.flipkart.ranger.client.RangerClientConstants; import com.flipkart.ranger.core.finder.SimpleShardedServiceFinder; import com.flipkart.ranger.core.finder.shardselector.MatchingShardSelector; -import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.zookeeper.ServiceFinderBuilders; import com.flipkart.ranger.zookeeper.serde.ZkNodeDataDeserializer; @@ -34,13 +33,14 @@ import java.util.List; import java.util.Optional; +import java.util.function.Predicate; @Slf4j @Getter -public class SimpleRangerZKClient> implements RangerClient { +public class SimpleRangerZKClient implements RangerClient { - private final SimpleShardedServiceFinder serviceFinder; - private final C criteria; + private final SimpleShardedServiceFinder serviceFinder; + private final Predicate criteria; private final ZkNodeDataDeserializer deserializer; @Builder(builderMethodName = "fromConnectionString", builderClassName = "FromConnectionStringBuilder") @@ -51,7 +51,7 @@ public SimpleRangerZKClient( int nodeRefreshIntervalMs, boolean disableWatchers, String connectionString, - C criteria, + Predicate criteria, ZkNodeDataDeserializer deserializer ){ this( @@ -74,7 +74,7 @@ public SimpleRangerZKClient( int nodeRefreshIntervalMs, boolean disableWatchers, CuratorFramework curatorFramework, - C criteria, + Predicate criteria, ZkNodeDataDeserializer deserializer ){ Preconditions.checkNotNull(mapper, "Mapper can't be null"); @@ -91,7 +91,7 @@ public SimpleRangerZKClient( this.criteria = criteria; this.deserializer = deserializer; - this.serviceFinder = ServiceFinderBuilders.shardedFinderBuilder() + this.serviceFinder = ServiceFinderBuilders.shardedFinderBuilder() .withCuratorFramework(curatorFramework) .withNamespace(namespace) .withServiceName(serviceName) @@ -120,17 +120,17 @@ public Optional> getNode() { } @Override - public Optional>> getAllNodes() { + public List> getAllNodes() { return getAllNodes(criteria); } @Override - public Optional> getNode(C criteria) { + public Optional> getNode(Predicate criteria) { return Optional.ofNullable(this.serviceFinder.get(criteria)); } @Override - public Optional>> getAllNodes(C criteria) { - return Optional.ofNullable(this.serviceFinder.getAll(criteria)); + public List> getAllNodes(Predicate criteria) { + return this.serviceFinder.getAll(criteria); } } diff --git a/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/UnshardedRangerZKHubClient.java b/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/UnshardedRangerZKHubClient.java index d5ce3722..2d935383 100644 --- a/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/UnshardedRangerZKHubClient.java +++ b/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/UnshardedRangerZKHubClient.java @@ -20,7 +20,6 @@ import com.flipkart.ranger.core.finder.serviceregistry.ListBasedServiceRegistry; import com.flipkart.ranger.core.finder.shardselector.ListShardSelector; import com.flipkart.ranger.core.finderhub.ServiceFinderFactory; -import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.zookeeper.serde.ZkNodeDataDeserializer; import com.flipkart.ranger.zookeeper.servicefinderhub.ZKUnshardedServiceFinderFactory; @@ -29,10 +28,11 @@ import org.apache.curator.framework.CuratorFramework; import java.util.List; +import java.util.function.Predicate; @Slf4j -public class UnshardedRangerZKHubClient> - extends AbstractRangerZKHubClient, ZkNodeDataDeserializer> { +public class UnshardedRangerZKHubClient + extends AbstractRangerZKHubClient, ZkNodeDataDeserializer> { @Builder public UnshardedRangerZKHubClient( @@ -42,7 +42,7 @@ public UnshardedRangerZKHubClient( boolean disablePushUpdaters, String connectionString, CuratorFramework curatorFramework, - C criteria, + Predicate criteria, ZkNodeDataDeserializer deserializer, List services ) { @@ -60,8 +60,8 @@ public UnshardedRangerZKHubClient( } @Override - protected ServiceFinderFactory> buildFinderFactory() { - return ZKUnshardedServiceFinderFactory.builder() + protected ServiceFinderFactory> buildFinderFactory() { + return ZKUnshardedServiceFinderFactory.builder() .curatorFramework(getCuratorFramework()) .connectionString(getConnectionString()) .nodeRefreshIntervalMs(getNodeRefreshTimeMs()) diff --git a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/BaseRangerZKClientTest.java b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/BaseRangerZKClientTest.java index b670c1b1..54819312 100644 --- a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/BaseRangerZKClientTest.java +++ b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/BaseRangerZKClientTest.java @@ -28,6 +28,7 @@ import com.flipkart.ranger.zookeeper.serde.ZkNodeDataSerializer; import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import lombok.val; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.retry.ExponentialBackoffRetry; @@ -97,7 +98,7 @@ protected byte[] write(final ServiceNode node) { } protected void initilizeProvider(){ - ExternalTriggeredSignal refreshProviderSignal = new ExternalTriggeredSignal<>( + val refreshProviderSignal = new ExternalTriggeredSignal<>( () -> HealthcheckResult.builder() .status(HealthcheckStatus.healthy) .updatedTime(new Date().getTime()) @@ -108,7 +109,7 @@ protected void initilizeProvider(){ .withNamespace("test-n") .withServiceName("s1") .withSerializer(this::write) - .withNodeData(new TestNodeData(1)) + .withNodeData(TestNodeData.builder().shardId(1).build()) .withHealthcheck(() -> HealthcheckStatus.healthy) .withAdditionalRefreshSignal(refreshProviderSignal) .withCuratorFramework(getCuratorFramework()) diff --git a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/ShardedZKRangerClientTest.java b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/ShardedZKRangerClientTest.java index 135c9632..ac090323 100644 --- a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/ShardedZKRangerClientTest.java +++ b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/ShardedZKRangerClientTest.java @@ -15,11 +15,10 @@ */ package com.flipkart.ranger.client.zk; -import com.flipkart.ranger.core.TestUtils; -import com.flipkart.ranger.core.model.Criteria; -import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.units.TestNodeData; +import com.flipkart.ranger.core.utils.RangerTestUtils; +import com.flipkart.ranger.core.utils.TestUtils; import lombok.extern.slf4j.Slf4j; import lombok.val; import org.junit.Assert; @@ -32,7 +31,7 @@ public class ShardedZKRangerClientTest extends BaseRangerZKClientTest { @Test public void testShardedHub(){ - val zkHubClient = ShardedRangerZKHubClient.>builder() + val zkHubClient = ShardedRangerZKHubClient.builder() .namespace("test-n") .connectionString(getTestingCluster().getConnectString()) .curatorFramework(getCuratorFramework()) @@ -44,11 +43,11 @@ public void testShardedHub(){ zkHubClient.start(); TestUtils.sleepForSeconds(6); - val service = new Service("test-n", "s1"); - Optional> node = zkHubClient.getNode(new Service("test-n", "s1")); + val service = RangerTestUtils.getService("test-n", "s1"); + Optional> node = zkHubClient.getNode(RangerTestUtils.getService("test-n", "s1")); Assert.assertTrue(node.isPresent()); - node = zkHubClient.getNode(service, nodeData -> nodeData.getNodeId() == 1); + node = zkHubClient.getNode(service, nodeData -> nodeData.getShardId() == 1); Assert.assertTrue(node.isPresent()); zkHubClient.stop(); diff --git a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/SimpleRangerZKClientTest.java b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/SimpleRangerZKClientTest.java index 7ef07936..1be52d0b 100644 --- a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/SimpleRangerZKClientTest.java +++ b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/SimpleRangerZKClientTest.java @@ -15,7 +15,6 @@ */ package com.flipkart.ranger.client.zk; -import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.units.TestNodeData; import lombok.val; @@ -28,7 +27,7 @@ public class SimpleRangerZKClientTest extends BaseRangerZKClientTest { @Test public void testBaseClient(){ - val client = SimpleRangerZKClient.>fromCurator() + val client = SimpleRangerZKClient.fromCurator() .curatorFramework(getCuratorFramework()) .deserializer(this::read) .namespace("test-n") @@ -41,10 +40,10 @@ public void testBaseClient(){ Optional> node = client.getNode(); Assert.assertTrue(node.isPresent()); - node = client.getNode(c -> c.getNodeId() == 1); + node = client.getNode(c -> c.getShardId() == 1); Assert.assertTrue(node.isPresent()); - node = client.getNode(c -> c.getNodeId() == 2); + node = client.getNode(c -> c.getShardId() == 2); Assert.assertFalse(node.isPresent()); } } diff --git a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/UnshardedZKRangerClientTest.java b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/UnshardedZKRangerClientTest.java index e29150bb..abeee7a0 100644 --- a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/UnshardedZKRangerClientTest.java +++ b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/UnshardedZKRangerClientTest.java @@ -15,11 +15,10 @@ */ package com.flipkart.ranger.client.zk; -import com.flipkart.ranger.core.TestUtils; -import com.flipkart.ranger.core.model.Criteria; -import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.units.TestNodeData; +import com.flipkart.ranger.core.utils.RangerTestUtils; +import com.flipkart.ranger.core.utils.TestUtils; import lombok.extern.slf4j.Slf4j; import lombok.val; import org.junit.Assert; @@ -32,7 +31,7 @@ public class UnshardedZKRangerClientTest extends BaseRangerZKClientTest { @Test public void testShardedHub(){ - val zkHubClient =UnshardedRangerZKHubClient.>builder() + val zkHubClient =UnshardedRangerZKHubClient.builder() .namespace("test-n") .connectionString(getTestingCluster().getConnectString()) .curatorFramework(getCuratorFramework()) @@ -44,11 +43,11 @@ public void testShardedHub(){ zkHubClient.start(); TestUtils.sleepForSeconds(6); - val service = new Service("test-n", "s1"); - Optional> node = zkHubClient.getNode(new Service("test-n", "s1")); + val service = RangerTestUtils.getService("test-n", "s1"); + Optional> node = zkHubClient.getNode(service); Assert.assertTrue(node.isPresent()); - node = zkHubClient.getNode(service, nodeData -> nodeData.getNodeId() == 1); + node = zkHubClient.getNode(service, nodeData -> nodeData.getShardId() == 1); Assert.assertTrue(node.isPresent()); zkHubClient.stop(); diff --git a/ranger-zk-server/src/main/java/com/flipkart/ranger/server/App.java b/ranger-zk-server/src/main/java/com/flipkart/ranger/server/App.java deleted file mode 100644 index c81a8e60..00000000 --- a/ranger-zk-server/src/main/java/com/flipkart/ranger/server/App.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright 2015 Flipkart Internet Pvt. Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.flipkart.ranger.server; - -import com.flipkart.ranger.client.RangerHubClient; -import com.flipkart.ranger.common.server.ShardInfo; -import com.flipkart.ranger.core.model.Criteria; -import com.flipkart.ranger.server.bundle.RangerServerBundle; -import com.flipkart.ranger.server.healthcheck.RangerHealthCheck; -import com.flipkart.ranger.server.manager.RangerBundleManager; -import com.flipkart.ranger.server.util.RangerServerUtils; -import com.flipkart.ranger.zookeeper.serde.ZkNodeDataDeserializer; -import com.google.common.collect.Lists; -import io.dropwizard.Application; -import io.dropwizard.setup.Bootstrap; -import io.dropwizard.setup.Environment; -import lombok.extern.slf4j.Slf4j; -import lombok.val; - -import java.util.List; - -@Slf4j -public class App extends Application { - - public static void main(String[] args) throws Exception { - new App().run(args); - } - - @Override - public void initialize(Bootstrap bootstrap) { - - } - - /** - * The reason why rangerServerBundle.run is used here instead of initializing in the initialize method and letting run automatically called. - * - * a) RangerServerBundle's closure contains a rangerHubClient within it, now this client could either be a ZK client or the httpClient. - * b) A ZK client would need the curatorFramework and the http client doesn't need anything. - * c) So the RangerServerBundle couldn't contain a curatorFramework and that has to be defined outside, for that we need the AppConfiguration, which - * is available in the run as an arg to be used, - * d) Also don't want to create two bundle managers (one for http and one for ZK) - * - * Also, the verifyPreconditions is different for both zk and http. - * - * @param appConfiguration {@link AppConfiguration} The appConfiguration provided - * @param environment {@link Environment} The environment in scope. - */ - @Override - public void run(AppConfiguration appConfiguration, Environment environment) { - val rangerConfiguration = appConfiguration.getRangerConfiguration(); - RangerServerUtils.verifyPreconditions(rangerConfiguration); - val curatorFramework = RangerServerUtils.buildCurator(rangerConfiguration); - - val rangerServerBundle = new RangerServerBundle, - AppConfiguration>() { - - @Override - protected List>> withHubs(AppConfiguration configuration) { - return Lists.newArrayList( - RangerServerUtils.buildRangerHub(curatorFramework, rangerConfiguration, environment.getObjectMapper()) - ); - } - - @Override - protected boolean withInitialRotationStatus(AppConfiguration configuration) { - return appConfiguration.isInitialRotationStatus(); - } - }; - rangerServerBundle.run(appConfiguration, environment); - - val rangerClientManager = new RangerBundleManager(curatorFramework, rangerServerBundle); - environment.lifecycle().manage(rangerClientManager); - - environment.healthChecks().register( - "ranger-health-check", - new RangerHealthCheck(rangerClientManager.getCuratorFramework()) - ); - } -} diff --git a/ranger-zk-server/src/main/java/com/flipkart/ranger/zk/server/App.java b/ranger-zk-server/src/main/java/com/flipkart/ranger/zk/server/App.java new file mode 100644 index 00000000..f93e176d --- /dev/null +++ b/ranger-zk-server/src/main/java/com/flipkart/ranger/zk/server/App.java @@ -0,0 +1,42 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.zk.server; + +import com.flipkart.ranger.zk.server.bundle.ZKServerBundle; +import io.dropwizard.Application; +import io.dropwizard.setup.Bootstrap; +import io.dropwizard.setup.Environment; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class App extends Application { + + public static void main(String[] args) throws Exception { + new App().run(args); + } + + @Override + public void initialize(Bootstrap bootstrap) { + bootstrap.addBundle(new ZKServerBundle()); + } + + @Override + public void run(AppConfiguration appConfiguration, Environment environment) { + /* + Nothing to do here. + */ + } +} diff --git a/ranger-zk-server/src/main/java/com/flipkart/ranger/server/AppConfiguration.java b/ranger-zk-server/src/main/java/com/flipkart/ranger/zk/server/AppConfiguration.java similarity index 71% rename from ranger-zk-server/src/main/java/com/flipkart/ranger/server/AppConfiguration.java rename to ranger-zk-server/src/main/java/com/flipkart/ranger/zk/server/AppConfiguration.java index 057bba05..838899ca 100644 --- a/ranger-zk-server/src/main/java/com/flipkart/ranger/server/AppConfiguration.java +++ b/ranger-zk-server/src/main/java/com/flipkart/ranger/zk/server/AppConfiguration.java @@ -13,35 +13,33 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.flipkart.ranger.server; +package com.flipkart.ranger.zk.server; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.flipkart.ranger.server.config.RangerConfiguration; +import com.flipkart.ranger.zk.server.config.RangerConfiguration; import io.dropwizard.Configuration; -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; +import lombok.Builder; +import lombok.Value; +import lombok.extern.jackson.Jacksonized; import javax.validation.Valid; import javax.validation.constraints.NotEmpty; import javax.validation.constraints.NotNull; -@Getter -@Setter -@AllArgsConstructor -@NoArgsConstructor +@Value +@Jacksonized +@Builder @JsonIgnoreProperties(ignoreUnknown = true) public class AppConfiguration extends Configuration { @NotEmpty @NotNull - private String name; + String name; @Valid @NotNull - private RangerConfiguration rangerConfiguration; + RangerConfiguration rangerConfiguration; - private boolean initialRotationStatus; + boolean initialRotationStatus; } diff --git a/ranger-zk-server/src/main/java/com/flipkart/ranger/server/util/RangerServerUtils.java b/ranger-zk-server/src/main/java/com/flipkart/ranger/zk/server/bundle/ZKServerBundle.java similarity index 52% rename from ranger-zk-server/src/main/java/com/flipkart/ranger/server/util/RangerServerUtils.java rename to ranger-zk-server/src/main/java/com/flipkart/ranger/zk/server/bundle/ZKServerBundle.java index 95f1540f..5e9b923f 100644 --- a/ranger-zk-server/src/main/java/com/flipkart/ranger/server/util/RangerServerUtils.java +++ b/ranger-zk-server/src/main/java/com/flipkart/ranger/zk/server/bundle/ZKServerBundle.java @@ -13,30 +13,42 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.flipkart.ranger.server.util; +package com.flipkart.ranger.zk.server.bundle; +import com.codahale.metrics.health.HealthCheck; import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; import com.flipkart.ranger.client.RangerClientConstants; +import com.flipkart.ranger.client.RangerHubClient; import com.flipkart.ranger.client.zk.UnshardedRangerZKHubClient; import com.flipkart.ranger.common.server.ShardInfo; -import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.ServiceNode; -import com.flipkart.ranger.server.config.RangerConfiguration; +import com.flipkart.ranger.zk.server.AppConfiguration; +import com.flipkart.ranger.zk.server.bundle.model.LifecycleSignal; +import com.flipkart.ranger.zk.server.healthcheck.RangerHealthCheck; +import com.flipkart.ranger.zk.server.lifecycle.CuratorLifecycle; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; +import lombok.val; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.retry.RetryForever; +import javax.inject.Singleton; import java.io.IOException; +import java.util.List; @Slf4j -public class RangerServerUtils { +@Singleton +@NoArgsConstructor +public class ZKServerBundle extends RangerServerBundle { - private RangerServerUtils(){} + private CuratorFramework curatorFramework; - public static void verifyPreconditions(RangerConfiguration rangerConfiguration){ + @Override + protected void verifyPreconditions(AppConfiguration configuration) { + val rangerConfiguration = configuration.getRangerConfiguration(); Preconditions.checkNotNull(rangerConfiguration, "ranger configuration can't be null"); Preconditions.checkNotNull(rangerConfiguration.getNamespace(), @@ -45,34 +57,50 @@ public static void verifyPreconditions(RangerConfiguration rangerConfiguration){ "Zookeeper can't be null"); } - public static CuratorFramework buildCurator(RangerConfiguration rangerConfiguration){ - return CuratorFrameworkFactory.newClient( - rangerConfiguration.getZookeeper(), + @Override + protected void preBundle(AppConfiguration configuration) { + curatorFramework = CuratorFrameworkFactory.newClient( + configuration.getRangerConfiguration().getZookeeper(), new RetryForever(RangerClientConstants.CONNECTION_RETRY_TIME) ); } - public static UnshardedRangerZKHubClient> buildRangerHub( - CuratorFramework curatorFramework, - RangerConfiguration rangerConfiguration, - ObjectMapper mapper - ){ - return UnshardedRangerZKHubClient.>builder() + @Override + protected List> withHubs(AppConfiguration configuration) { + val rangerConfiguration = configuration.getRangerConfiguration(); + return ImmutableList.of(UnshardedRangerZKHubClient.builder() .namespace(rangerConfiguration.getNamespace()) .connectionString(rangerConfiguration.getZookeeper()) .curatorFramework(curatorFramework) .disablePushUpdaters(rangerConfiguration.isDisablePushUpdaters()) - .mapper(mapper) + .mapper(getMapper()) .refreshTimeMs(rangerConfiguration.getNodeRefreshTimeMs()) .deserializer(data -> { try { - mapper.readValue(data, new TypeReference>() { + getMapper().readValue(data, new TypeReference>() { }); } catch (IOException e) { log.warn("Error parsing node data with value {}", new String(data)); } return null; }) - .build(); + .build()); + } + + @Override + protected boolean withInitialRotationStatus(AppConfiguration configuration) { + return configuration.isInitialRotationStatus(); + } + + @Override + protected List withLifecycleSignals(AppConfiguration configuration) { + return ImmutableList.of( + new CuratorLifecycle(curatorFramework) + ); + } + + @Override + protected List withHealthChecks(AppConfiguration configuration) { + return ImmutableList.of(new RangerHealthCheck(curatorFramework)); } } diff --git a/ranger-zk-server/src/main/java/com/flipkart/ranger/server/config/RangerConfiguration.java b/ranger-zk-server/src/main/java/com/flipkart/ranger/zk/server/config/RangerConfiguration.java similarity index 77% rename from ranger-zk-server/src/main/java/com/flipkart/ranger/server/config/RangerConfiguration.java rename to ranger-zk-server/src/main/java/com/flipkart/ranger/zk/server/config/RangerConfiguration.java index d5aa6fa3..e18c00bd 100644 --- a/ranger-zk-server/src/main/java/com/flipkart/ranger/server/config/RangerConfiguration.java +++ b/ranger-zk-server/src/main/java/com/flipkart/ranger/zk/server/config/RangerConfiguration.java @@ -13,30 +13,30 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.flipkart.ranger.server.config; +package com.flipkart.ranger.zk.server.config; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.flipkart.ranger.client.RangerClientConstants; -import lombok.*; +import lombok.Builder; +import lombok.Value; +import lombok.extern.jackson.Jacksonized; import javax.validation.constraints.Min; import javax.validation.constraints.NotEmpty; import javax.validation.constraints.NotNull; -@Getter -@Setter -@AllArgsConstructor -@NoArgsConstructor +@Value +@Jacksonized @Builder @JsonIgnoreProperties(ignoreUnknown = true) public class RangerConfiguration { @NotEmpty @NotNull - private String namespace; + String namespace; @NotEmpty @NotNull - private String zookeeper; - private boolean disablePushUpdaters; + String zookeeper; + boolean disablePushUpdaters; @Min(1000) - private int nodeRefreshTimeMs = RangerClientConstants.MINIMUM_REFRESH_TIME; + int nodeRefreshTimeMs = RangerClientConstants.MINIMUM_REFRESH_TIME; } diff --git a/ranger-zk-server/src/main/java/com/flipkart/ranger/server/healthcheck/RangerHealthCheck.java b/ranger-zk-server/src/main/java/com/flipkart/ranger/zk/server/healthcheck/RangerHealthCheck.java similarity index 96% rename from ranger-zk-server/src/main/java/com/flipkart/ranger/server/healthcheck/RangerHealthCheck.java rename to ranger-zk-server/src/main/java/com/flipkart/ranger/zk/server/healthcheck/RangerHealthCheck.java index b27968ef..ac5c6c81 100644 --- a/ranger-zk-server/src/main/java/com/flipkart/ranger/server/healthcheck/RangerHealthCheck.java +++ b/ranger-zk-server/src/main/java/com/flipkart/ranger/zk/server/healthcheck/RangerHealthCheck.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.flipkart.ranger.server.healthcheck; +package com.flipkart.ranger.zk.server.healthcheck; import com.codahale.metrics.health.HealthCheck; import lombok.extern.slf4j.Slf4j; @@ -31,7 +31,6 @@ public RangerHealthCheck(CuratorFramework curatorFramework){ this.curatorFramework = curatorFramework; } - @Override protected Result check() { return curatorFramework.getZookeeperClient().isConnected() ? diff --git a/ranger-zk-server/src/main/java/com/flipkart/ranger/server/manager/RangerBundleManager.java b/ranger-zk-server/src/main/java/com/flipkart/ranger/zk/server/lifecycle/CuratorLifecycle.java similarity index 56% rename from ranger-zk-server/src/main/java/com/flipkart/ranger/server/manager/RangerBundleManager.java rename to ranger-zk-server/src/main/java/com/flipkart/ranger/zk/server/lifecycle/CuratorLifecycle.java index bf060353..af00f988 100644 --- a/ranger-zk-server/src/main/java/com/flipkart/ranger/server/manager/RangerBundleManager.java +++ b/ranger-zk-server/src/main/java/com/flipkart/ranger/zk/server/lifecycle/CuratorLifecycle.java @@ -1,50 +1,37 @@ /* * Copyright 2015 Flipkart Internet Pvt. Ltd. - *

+ * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

+ * * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ -package com.flipkart.ranger.server.manager; +package com.flipkart.ranger.zk.server.lifecycle; -import com.flipkart.ranger.server.bundle.RangerServerBundle; -import io.dropwizard.lifecycle.Managed; -import lombok.Getter; +import com.flipkart.ranger.zk.server.bundle.model.LifecycleSignal; +import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.curator.framework.CuratorFramework; -import javax.inject.Inject; import javax.inject.Singleton; -@Singleton -@Getter @Slf4j -public class RangerBundleManager implements Managed { +@Singleton +@AllArgsConstructor +public class CuratorLifecycle implements LifecycleSignal { private final CuratorFramework curatorFramework; - private final RangerServerBundle rangerServerBundle; - - @Inject - public RangerBundleManager( - CuratorFramework curatorFramework, - RangerServerBundle rangerServerBundle - ){ - this.curatorFramework = curatorFramework; - this.rangerServerBundle = rangerServerBundle; - } - @Override public void start() { - log.info("Starting the ranger client manager"); + log.info("Starting the curator"); curatorFramework.start(); try { @@ -53,15 +40,11 @@ public void start() { catch (InterruptedException e) { log.error("Curator block interrupted", e); } - rangerServerBundle.start(); - log.info("Started the ranger client manager"); + log.info("Started the curator"); } @Override public void stop() { - log.info("Stopping the ranger client manager"); - rangerServerBundle.stop(); curatorFramework.close(); - log.info("Stopped the ranger client manager"); } } diff --git a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/ServiceFinderBuilders.java b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/ServiceFinderBuilders.java index 179068c5..021b1618 100644 --- a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/ServiceFinderBuilders.java +++ b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/ServiceFinderBuilders.java @@ -15,7 +15,6 @@ */ package com.flipkart.ranger.zookeeper; -import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.zookeeper.servicefinder.ZkSimpleShardedServiceFinderBuilder; import com.flipkart.ranger.zookeeper.servicefinder.ZkSimpleUnshardedServiceFinderBuilder; @@ -25,11 +24,11 @@ private ServiceFinderBuilders() { throw new InstantiationError("Must not instantiate this class"); } - public static > ZkSimpleShardedServiceFinderBuilder shardedFinderBuilder() { + public static ZkSimpleShardedServiceFinderBuilder shardedFinderBuilder() { return new ZkSimpleShardedServiceFinderBuilder<>(); } - public static > ZkSimpleUnshardedServiceFinderBuilder unshardedFinderBuilder() { + public static ZkSimpleUnshardedServiceFinderBuilder unshardedFinderBuilder() { return new ZkSimpleUnshardedServiceFinderBuilder<>(); } } diff --git a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinder/ZkNodeDataSource.java b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinder/ZkNodeDataSource.java index 2dd34fac..52c6b95e 100644 --- a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinder/ZkNodeDataSource.java +++ b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinder/ZkNodeDataSource.java @@ -18,7 +18,6 @@ import com.flipkart.ranger.core.model.NodeDataSource; import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.core.model.ServiceNode; -import com.flipkart.ranger.core.util.Exceptions; import com.flipkart.ranger.core.util.FinderUtils; import com.flipkart.ranger.zookeeper.common.ZkNodeDataStoreConnector; import com.flipkart.ranger.zookeeper.serde.ZkNodeDataDeserializer; @@ -29,6 +28,7 @@ import org.apache.curator.framework.CuratorFramework; import org.apache.zookeeper.KeeperException; +import java.util.Collections; import java.util.List; import java.util.Optional; @@ -45,20 +45,20 @@ public ZkNodeDataSource( } @Override - public Optional>> refresh(D deserializer) { + public List> refresh(D deserializer) { return checkForUpdateOnZookeeper(deserializer); } - private Optional>> checkForUpdateOnZookeeper(D deserializer) { + private List> checkForUpdateOnZookeeper(D deserializer) { if (!isStarted()) { log.warn("Data source is not yet started for service: {}. No nodes will be returned.", service.getServiceName()); - return Optional.empty(); + return Collections.emptyList(); } if (isStopped()) { log.warn("Data source is stopped already for service: {}. No nodes will be returned.", service.getServiceName()); - return Optional.empty(); + return Collections.emptyList(); } Preconditions.checkNotNull(deserializer, "Deserializer has not been set for node data"); try { @@ -67,7 +67,7 @@ private Optional>> checkForUpdateOnZookeeper(D deserializer) if (!isActive()) { log.warn("ZK connection is not active. Ignoring refresh request for service: {}", service.getServiceName()); - return Optional.empty(); + return Collections.emptyList(); } final String parentPath = PathBuilder.servicePath(service); log.debug("Looking for node list of [{}]", serviceName); @@ -84,12 +84,12 @@ private Optional>> checkForUpdateOnZookeeper(D deserializer) nodes.add(node); } } - return Optional.of(nodes); + return nodes; } catch (Exception e) { log.error("Error getting service data from zookeeper: ", e); } - return Optional.empty(); + return Collections.emptyList(); } private Optional readChild(String parentPath, String child) throws Exception { diff --git a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinder/ZkSimpleShardedServiceFinderBuilder.java b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinder/ZkSimpleShardedServiceFinderBuilder.java index fa055fe0..8c15ddd2 100644 --- a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinder/ZkSimpleShardedServiceFinderBuilder.java +++ b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinder/ZkSimpleShardedServiceFinderBuilder.java @@ -17,7 +17,6 @@ import com.flipkart.ranger.core.finder.SimpleShardedServiceFinder; import com.flipkart.ranger.core.finder.SimpleShardedServiceFinderBuilder; -import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.NodeDataSource; import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.core.signals.Signal; @@ -36,22 +35,22 @@ * */ @Slf4j -public class ZkSimpleShardedServiceFinderBuilder> extends SimpleShardedServiceFinderBuilder, ZkNodeDataDeserializer, C> { +public class ZkSimpleShardedServiceFinderBuilder extends SimpleShardedServiceFinderBuilder, ZkNodeDataDeserializer> { protected CuratorFramework curatorFramework; protected String connectionString; - public ZkSimpleShardedServiceFinderBuilder withCuratorFramework(CuratorFramework curatorFramework) { + public ZkSimpleShardedServiceFinderBuilder withCuratorFramework(CuratorFramework curatorFramework) { this.curatorFramework = curatorFramework; return this; } - public ZkSimpleShardedServiceFinderBuilder withConnectionString(final String connectionString) { + public ZkSimpleShardedServiceFinderBuilder withConnectionString(final String connectionString) { this.connectionString = connectionString; return this; } @Override - public SimpleShardedServiceFinder build() { + public SimpleShardedServiceFinder build() { boolean curatorProvided = curatorFramework != null; if (!curatorProvided) { Preconditions.checkNotNull(connectionString); diff --git a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinder/ZkSimpleUnshardedServiceFinderBuilder.java b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinder/ZkSimpleUnshardedServiceFinderBuilder.java index 640ebb68..e0e92226 100644 --- a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinder/ZkSimpleUnshardedServiceFinderBuilder.java +++ b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinder/ZkSimpleUnshardedServiceFinderBuilder.java @@ -17,7 +17,6 @@ import com.flipkart.ranger.core.finder.SimpleUnshardedServiceFinder; import com.flipkart.ranger.core.finder.SimpleUnshardedServiceFinderBuilder; -import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.NodeDataSource; import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.core.signals.Signal; @@ -36,23 +35,23 @@ * */ @Slf4j -public class ZkSimpleUnshardedServiceFinderBuilder> - extends SimpleUnshardedServiceFinderBuilder, ZkNodeDataDeserializer, C> { +public class ZkSimpleUnshardedServiceFinderBuilder + extends SimpleUnshardedServiceFinderBuilder, ZkNodeDataDeserializer> { private CuratorFramework curatorFramework; private String connectionString; - public ZkSimpleUnshardedServiceFinderBuilder withCuratorFramework(CuratorFramework curatorFramework) { + public ZkSimpleUnshardedServiceFinderBuilder withCuratorFramework(CuratorFramework curatorFramework) { this.curatorFramework = curatorFramework; return this; } - public ZkSimpleUnshardedServiceFinderBuilder withConnectionString(final String connectionString) { + public ZkSimpleUnshardedServiceFinderBuilder withConnectionString(final String connectionString) { this.connectionString = connectionString; return this; } @Override - public SimpleUnshardedServiceFinder build() { + public SimpleUnshardedServiceFinder build() { boolean curatorProvided = curatorFramework != null; if (!curatorProvided) { Preconditions.checkNotNull(connectionString); diff --git a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZKUnshardedServiceFinderFactory.java b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZKUnshardedServiceFinderFactory.java index 8539998b..1de83c51 100644 --- a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZKUnshardedServiceFinderFactory.java +++ b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZKUnshardedServiceFinderFactory.java @@ -18,7 +18,6 @@ import com.flipkart.ranger.core.finder.SimpleUnshardedServiceFinder; import com.flipkart.ranger.core.finder.serviceregistry.ListBasedServiceRegistry; import com.flipkart.ranger.core.finderhub.ServiceFinderFactory; -import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.core.model.ServiceNodeSelector; import com.flipkart.ranger.core.model.ShardSelector; @@ -31,14 +30,14 @@ @Getter -public class ZKUnshardedServiceFinderFactory> implements ServiceFinderFactory>{ +public class ZKUnshardedServiceFinderFactory implements ServiceFinderFactory>{ private final CuratorFramework curatorFramework; private final String connectionString; private final int nodeRefreshIntervalMs; private final boolean disablePushUpdaters; private final ZkNodeDataDeserializer deserializer; - private final ShardSelector> shardSelector; + private final ShardSelector> shardSelector; private final ServiceNodeSelector nodeSelector; @Builder @@ -48,7 +47,7 @@ public ZKUnshardedServiceFinderFactory( int nodeRefreshIntervalMs, boolean disablePushUpdaters, ZkNodeDataDeserializer deserializer, - ShardSelector> shardSelector, + ShardSelector> shardSelector, ServiceNodeSelector nodeSelector) { this.curatorFramework = curatorFramework; this.connectionString = connectionString; @@ -60,8 +59,8 @@ public ZKUnshardedServiceFinderFactory( } @Override - public SimpleUnshardedServiceFinder buildFinder(Service service) { - val finder = new ZkSimpleUnshardedServiceFinderBuilder() + public SimpleUnshardedServiceFinder buildFinder(Service service) { + val finder = new ZkSimpleUnshardedServiceFinderBuilder() .withDeserializer(deserializer) .withNamespace(service.getNamespace()) .withServiceName(service.getServiceName()) diff --git a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZkServiceDataSource.java b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZkServiceDataSource.java index e13fcaf1..224042fe 100644 --- a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZkServiceDataSource.java +++ b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZkServiceDataSource.java @@ -51,8 +51,9 @@ public Collection services() throws Exception { final List children = curatorFramework.getChildren() .forPath("/"); return children.stream() - .map(child -> new Service(namespace, child)) + .map(child -> Service.builder().namespace(namespace).serviceName(child).build()) .collect(Collectors.toSet()); + } @Override diff --git a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZkServiceFinderHubBuilder.java b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZkServiceFinderHubBuilder.java index e0fcc11d..a06772cd 100644 --- a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZkServiceFinderHubBuilder.java +++ b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZkServiceFinderHubBuilder.java @@ -17,7 +17,6 @@ import com.flipkart.ranger.core.finderhub.ServiceFinderHub; import com.flipkart.ranger.core.finderhub.ServiceFinderHubBuilder; -import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.ServiceRegistry; import com.google.common.base.Preconditions; import lombok.extern.slf4j.Slf4j; @@ -29,22 +28,22 @@ * */ @Slf4j -public class ZkServiceFinderHubBuilder, R extends ServiceRegistry> extends ServiceFinderHubBuilder { +public class ZkServiceFinderHubBuilder> extends ServiceFinderHubBuilder { private String namespace; private CuratorFramework curatorFramework; private String connectionString; - public ZkServiceFinderHubBuilder withNamespace(final String namespace) { + public ZkServiceFinderHubBuilder withNamespace(final String namespace) { this.namespace = namespace; return this; } - public ZkServiceFinderHubBuilder withCuratorFramework(CuratorFramework curatorFramework) { + public ZkServiceFinderHubBuilder withCuratorFramework(CuratorFramework curatorFramework) { this.curatorFramework = curatorFramework; return this; } - public ZkServiceFinderHubBuilder withConnectionString(final String connectionString) { + public ZkServiceFinderHubBuilder withConnectionString(final String connectionString) { this.connectionString = connectionString; return this; } @@ -65,7 +64,7 @@ protected void preBuild() { } @Override - protected void postBuild(ServiceFinderHub serviceFinderHub) { + protected void postBuild(ServiceFinderHub serviceFinderHub) { log.debug("No post build steps necessary"); } } diff --git a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZkShardedServiceFinderFactory.java b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZkShardedServiceFinderFactory.java index d36981ff..363279d0 100644 --- a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZkShardedServiceFinderFactory.java +++ b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZkShardedServiceFinderFactory.java @@ -18,7 +18,6 @@ import com.flipkart.ranger.core.finder.SimpleShardedServiceFinder; import com.flipkart.ranger.core.finder.serviceregistry.MapBasedServiceRegistry; import com.flipkart.ranger.core.finderhub.ServiceFinderFactory; -import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.core.model.ServiceNodeSelector; import com.flipkart.ranger.core.model.ShardSelector; @@ -31,13 +30,13 @@ /** * */ -public class ZkShardedServiceFinderFactory> implements ServiceFinderFactory> { +public class ZkShardedServiceFinderFactory implements ServiceFinderFactory> { private final CuratorFramework curatorFramework; private final String connectionString; private final int nodeRefreshIntervalMs; private final boolean disablePushUpdaters; private final ZkNodeDataDeserializer deserializer; - private final ShardSelector> shardSelector; + private final ShardSelector> shardSelector; private final ServiceNodeSelector nodeSelector; @Builder @@ -47,7 +46,7 @@ public ZkShardedServiceFinderFactory( int nodeRefreshIntervalMs, boolean disablePushUpdaters, ZkNodeDataDeserializer deserializer, - ShardSelector> shardSelector, + ShardSelector> shardSelector, ServiceNodeSelector nodeSelector) { this.curatorFramework = curatorFramework; this.connectionString = connectionString; @@ -59,8 +58,8 @@ public ZkShardedServiceFinderFactory( } @Override - public SimpleShardedServiceFinder buildFinder(Service service) { - val finder = new ZkSimpleShardedServiceFinderBuilder() + public SimpleShardedServiceFinder buildFinder(Service service) { + val finder = new ZkSimpleShardedServiceFinderBuilder() .withDeserializer(deserializer) .withNamespace(service.getNamespace()) .withServiceName(service.getServiceName()) diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/ServiceHealthAggregatorTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/ServiceHealthAggregatorTest.java index bc217329..ac6c7f35 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/ServiceHealthAggregatorTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/ServiceHealthAggregatorTest.java @@ -15,12 +15,12 @@ */ package com.flipkart.ranger.zookeeper.healthservice; -import com.flipkart.ranger.core.TestUtils; import com.flipkart.ranger.core.healthcheck.HealthcheckStatus; import com.flipkart.ranger.core.healthservice.ServiceHealthAggregator; import com.flipkart.ranger.core.healthservice.TimeEntity; import com.flipkart.ranger.core.healthservice.monitor.IsolatedHealthMonitor; import com.flipkart.ranger.core.healthservice.monitor.Monitor; +import com.flipkart.ranger.core.utils.TestUtils; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -78,7 +78,7 @@ public void testStaleRun() throws Exception { } - private class TestMonitor extends IsolatedHealthMonitor { + private static class TestMonitor extends IsolatedHealthMonitor { int threadSleep = 2000; public TestMonitor(String name, TimeEntity timeEntity) { diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/ServiceProviderIntegrationTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/ServiceProviderIntegrationTest.java index 3219233f..5e978957 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/ServiceProviderIntegrationTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/ServiceProviderIntegrationTest.java @@ -19,14 +19,13 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import com.flipkart.ranger.core.TestUtils; import com.flipkart.ranger.core.finder.SimpleUnshardedServiceFinder; import com.flipkart.ranger.core.healthcheck.Healthchecks; import com.flipkart.ranger.core.healthservice.TimeEntity; import com.flipkart.ranger.core.healthservice.monitor.sample.RotationStatusMonitor; -import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.util.Exceptions; +import com.flipkart.ranger.core.utils.TestUtils; import com.flipkart.ranger.zookeeper.ServiceFinderBuilders; import com.flipkart.ranger.zookeeper.ServiceProviderBuilders; import lombok.val; @@ -76,7 +75,7 @@ public void startTestCluster() throws Exception { registerService("localhost-4", 9000, 2, anotherFile); - serviceFinder = ServiceFinderBuilders.>unshardedFinderBuilder() + serviceFinder = ServiceFinderBuilders.unshardedFinderBuilder() .withConnectionString(testingCluster.getConnectString()) .withNamespace("test") .withServiceName("test-service") diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/CustomShardSelectorTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/CustomShardSelectorTest.java index d489e6b3..ccbbf1bb 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/CustomShardSelectorTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/CustomShardSelectorTest.java @@ -18,10 +18,8 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import com.flipkart.ranger.core.finder.SimpleShardedServiceFinder; import com.flipkart.ranger.core.finder.serviceregistry.MapBasedServiceRegistry; import com.flipkart.ranger.core.healthcheck.Healthchecks; -import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.model.ShardSelector; import com.flipkart.ranger.core.serviceprovider.ServiceProvider; @@ -30,6 +28,7 @@ import com.flipkart.ranger.zookeeper.serde.ZkNodeDataSerializer; import com.google.common.collect.Lists; import lombok.extern.slf4j.Slf4j; +import lombok.val; import org.apache.curator.test.TestingCluster; import org.junit.After; import org.junit.Assert; @@ -37,8 +36,9 @@ import org.junit.Test; import java.io.IOException; +import java.util.ArrayList; import java.util.List; -import java.util.Map; +import java.util.function.Predicate; @Slf4j @@ -59,9 +59,7 @@ public void startTestCluster() throws Exception { @After public void stopTestCluster() throws Exception { - for(ServiceProvider> serviceProvider : serviceProviders) { - serviceProvider.stop(); - } + serviceProviders.forEach(ServiceProvider::stop); if(null != testingCluster) { testingCluster.close(); } @@ -113,29 +111,29 @@ public int hashCode() { return result; } - private static Criteria getCriteria(int a, int b){ + private static Predicate getCriteria(int a, int b){ return nodeData -> nodeData.getA() == a && nodeData.getB() == b; } } - private static final class TestShardSelector implements ShardSelector, MapBasedServiceRegistry> { + private static final class TestShardSelector implements ShardSelector> { @Override - public List> nodes(Criteria criteria, MapBasedServiceRegistry serviceRegistry) { - List> nodes = Lists.newArrayList(); - for(Map.Entry> entry : serviceRegistry.nodes().entries()) { - TestShardInfo shardInfo = entry.getKey(); - if(criteria.apply(shardInfo)){ + public List> nodes(Predicate criteria, MapBasedServiceRegistry serviceRegistry) { + val nodes = new ArrayList>(); + serviceRegistry.nodes().entries().forEach(entry -> { + val shardInfo = entry.getKey(); + if (criteria.test(shardInfo)) { nodes.add(entry.getValue()); } - } + }); return nodes; } } @Test - public void testBasicDiscovery() throws Exception { - SimpleShardedServiceFinder> serviceFinder = ServiceFinderBuilders.>shardedFinderBuilder() + public void testBasicDiscovery() { + val serviceFinder = ServiceFinderBuilders.shardedFinderBuilder() .withConnectionString(testingCluster.getConnectString()) .withNamespace("test") .withServiceName("test-service") @@ -152,11 +150,11 @@ public void testBasicDiscovery() throws Exception { .build(); serviceFinder.start(); { - ServiceNode node = serviceFinder.get(TestShardInfo.getCriteria(1, 10)); + val node = serviceFinder.get(TestShardInfo.getCriteria(1, 10)); Assert.assertNull(node); } { - ServiceNode node = serviceFinder.get(TestShardInfo.getCriteria(1, 2)); + val node = serviceFinder.get(TestShardInfo.getCriteria(1, 2)); Assert.assertNotNull(node); Assert.assertEquals(new TestShardInfo(1, 2), node.getNodeData()); } @@ -164,7 +162,7 @@ public void testBasicDiscovery() throws Exception { } private void registerService(String host, int port, int a, int b) { - final ServiceProvider> serviceProvider = ServiceProviderBuilders.shardedServiceProviderBuilder() + val serviceProvider = ServiceProviderBuilders.shardedServiceProviderBuilder() .withConnectionString(testingCluster.getConnectString()) .withNamespace("test") .withServiceName("test-service") @@ -178,7 +176,7 @@ private void registerService(String host, int port, int a, int b) { }) .withHostname(host) .withPort(port) - .withNodeData(new TestShardInfo(a,b)) + .withNodeData(new TestShardInfo(a, b)) .withHealthcheck(Healthchecks.defaultHealthyCheck()) .withHealthUpdateIntervalMs(1000) .build(); diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceNoProviderTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceNoProviderTest.java index ebee0623..7fd40580 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceNoProviderTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceNoProviderTest.java @@ -19,8 +19,9 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.flipkart.ranger.core.finder.SimpleShardedServiceFinder; import com.flipkart.ranger.core.finder.nodeselector.RoundRobinServiceNodeSelector; -import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.ServiceNode; +import com.flipkart.ranger.core.units.TestNodeData; +import com.flipkart.ranger.core.utils.RangerTestUtils; import com.flipkart.ranger.zookeeper.ServiceFinderBuilders; import lombok.val; import org.apache.curator.test.TestingCluster; @@ -41,9 +42,6 @@ public void startTestCluster() throws Exception { objectMapper = new ObjectMapper(); testingCluster = new TestingCluster(3); testingCluster.start(); - //registerService("localhost-1", 9000, 1); - //registerService("localhost-2", 9000, 1); - //registerService("localhost-3", 9000, 2); } @After @@ -53,63 +51,16 @@ public void stopTestCluster() throws Exception { } } - private static final class TestShardInfo { - private int shardId; - - public TestShardInfo(int shardId) { - this.shardId = shardId; - } - - public TestShardInfo() { - } - - public int getShardId() { - return shardId; - } - - public void setShardId(int shardId) { - this.shardId = shardId; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - - TestShardInfo that = (TestShardInfo) o; - - if (shardId != that.shardId) { - return false; - } - - return true; - } - - @Override - public int hashCode() { - return shardId; - } - - private static Criteria getCriteria(int shardId){ - return nodeData -> shardId == nodeData.getShardId(); - } - } - - @Test - public void testBasicDiscovery() throws Exception { - SimpleShardedServiceFinder> serviceFinder = ServiceFinderBuilders.>shardedFinderBuilder() + public void testBasicDiscovery() { + SimpleShardedServiceFinder serviceFinder = ServiceFinderBuilders.shardedFinderBuilder() .withConnectionString(testingCluster.getConnectString()) .withNamespace("test") .withServiceName("test-service") .withDeserializer(data -> { try { return objectMapper.readValue(data, - new TypeReference>() { + new TypeReference>() { }); } catch (IOException e) { @@ -119,7 +70,7 @@ public void testBasicDiscovery() throws Exception { }) .build(); serviceFinder.start(); - ServiceNode node = serviceFinder.get(TestShardInfo.getCriteria(1)); + ServiceNode node = serviceFinder.get(RangerTestUtils.getCriteria(1)); Assert.assertNull(node); serviceFinder.stop(); @@ -127,7 +78,7 @@ public void testBasicDiscovery() throws Exception { @Test public void testBasicDiscoveryRR() { - val serviceFinder = ServiceFinderBuilders.>shardedFinderBuilder() + val serviceFinder = ServiceFinderBuilders.shardedFinderBuilder() .withConnectionString(testingCluster.getConnectString()) .withNamespace("test") .withServiceName("test-service") @@ -135,7 +86,7 @@ public void testBasicDiscoveryRR() { .withDeserializer(data -> { try { return objectMapper.readValue(data, - new TypeReference>() { + new TypeReference>() { }); } catch (IOException e) { @@ -145,7 +96,7 @@ public void testBasicDiscoveryRR() { }) .build(); serviceFinder.start(); - ServiceNode node = serviceFinder.get(TestShardInfo.getCriteria(1)); + ServiceNode node = serviceFinder.get(RangerTestUtils.getCriteria(1)); Assert.assertNull(node); serviceFinder.stop(); } diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderExtCuratorTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderExtCuratorTest.java index 68fbb4f6..6ec03f7a 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderExtCuratorTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderExtCuratorTest.java @@ -20,14 +20,16 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.flipkart.ranger.core.finder.SimpleShardedServiceFinder; import com.flipkart.ranger.core.healthcheck.Healthchecks; -import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.serviceprovider.ServiceProvider; +import com.flipkart.ranger.core.units.TestNodeData; +import com.flipkart.ranger.core.utils.RangerTestUtils; import com.flipkart.ranger.zookeeper.ServiceFinderBuilders; import com.flipkart.ranger.zookeeper.ServiceProviderBuilders; import com.flipkart.ranger.zookeeper.serde.ZkNodeDataSerializer; import com.google.common.collect.Lists; import lombok.extern.slf4j.Slf4j; +import lombok.val; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.retry.ExponentialBackoffRetry; @@ -45,7 +47,7 @@ public class ServiceProviderExtCuratorTest { private TestingCluster testingCluster; private ObjectMapper objectMapper; - private List>> serviceProviders = Lists.newArrayList(); + private List>> serviceProviders = Lists.newArrayList(); private CuratorFramework curatorFramework; @Before @@ -65,66 +67,23 @@ public void startTestCluster() throws Exception { @After public void stopTestCluster() throws Exception { - for(ServiceProvider> serviceProvider : serviceProviders) { - serviceProvider.stop(); - } + serviceProviders.forEach(ServiceProvider::stop); curatorFramework.close(); if(null != testingCluster) { testingCluster.close(); } } - private static final class TestShardInfo { - private int shardId; - - public TestShardInfo(int shardId) { - this.shardId = shardId; - } - - public TestShardInfo() { - } - - public int getShardId() { - return shardId; - } - - public void setShardId(int shardId) { - this.shardId = shardId; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - TestShardInfo that = (TestShardInfo) o; - - if (shardId != that.shardId) return false; - - return true; - } - - @Override - public int hashCode() { - return shardId; - } - - private static Criteria getCriteria(int shardId){ - return nodeData -> nodeData.getShardId() == shardId; - } - - } - @Test public void testBasicDiscovery() { - SimpleShardedServiceFinder> serviceFinder = ServiceFinderBuilders.>shardedFinderBuilder() + SimpleShardedServiceFinder serviceFinder = ServiceFinderBuilders.shardedFinderBuilder() .withCuratorFramework(curatorFramework) .withNamespace("test") .withServiceName("test-service") .withDeserializer(data -> { try { return objectMapper.readValue(data, - new TypeReference>() { + new TypeReference>() { }); } catch (IOException e) { e.printStackTrace(); @@ -134,25 +93,25 @@ public void testBasicDiscovery() { .build(); serviceFinder.start(); { - ServiceNode node = serviceFinder.get(TestShardInfo.getCriteria(1)); + ServiceNode node = serviceFinder.get(RangerTestUtils.getCriteria(1)); Assert.assertNotNull(node); Assert.assertEquals(1, node.getNodeData().getShardId()); } { - ServiceNode node = serviceFinder.get(TestShardInfo.getCriteria(1)); + ServiceNode node = serviceFinder.get(RangerTestUtils.getCriteria(1)); Assert.assertNotNull(node); Assert.assertEquals(1, node.getNodeData().getShardId()); } long startTime = System.currentTimeMillis(); for(long i = 0; i <1000000; i++) { - ServiceNode node = serviceFinder.get(TestShardInfo.getCriteria(2)); + ServiceNode node = serviceFinder.get(RangerTestUtils.getCriteria(2)); Assert.assertNotNull(node); Assert.assertEquals(2, node.getNodeData().getShardId()); } log.info("PERF::RandomSelector::Took (ms):" + (System.currentTimeMillis() - startTime)); { - ServiceNode node = serviceFinder.get(TestShardInfo.getCriteria(99)); + ServiceNode node = serviceFinder.get(RangerTestUtils.getCriteria(99)); Assert.assertNull(node); } serviceFinder.stop(); @@ -160,7 +119,7 @@ public void testBasicDiscovery() { } private void registerService(String host, int port, int shardId) { - final ServiceProvider> serviceProvider = ServiceProviderBuilders.shardedServiceProviderBuilder() + val serviceProvider = ServiceProviderBuilders.shardedServiceProviderBuilder() .withCuratorFramework(curatorFramework) .withNamespace("test") .withServiceName("test-service") @@ -174,7 +133,7 @@ private void registerService(String host, int port, int shardId) { }) .withHostname(host) .withPort(port) - .withNodeData(new TestShardInfo(shardId)) + .withNodeData(TestNodeData.builder().shardId(shardId).build()) .withHealthcheck(Healthchecks.defaultHealthyCheck()) .withHealthUpdateIntervalMs(1000) .build(); diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderHealthcheckTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderHealthcheckTest.java index bbeef818..31625988 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderHealthcheckTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderHealthcheckTest.java @@ -19,17 +19,16 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import com.flipkart.ranger.core.TestUtils; -import com.flipkart.ranger.core.finder.SimpleShardedServiceFinder; import com.flipkart.ranger.core.healthcheck.Healthcheck; import com.flipkart.ranger.core.healthcheck.HealthcheckStatus; -import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.ServiceNode; -import com.flipkart.ranger.core.serviceprovider.ServiceProvider; +import com.flipkart.ranger.core.units.TestNodeData; +import com.flipkart.ranger.core.utils.RangerTestUtils; +import com.flipkart.ranger.core.utils.TestUtils; import com.flipkart.ranger.zookeeper.ServiceFinderBuilders; import com.flipkart.ranger.zookeeper.ServiceProviderBuilders; -import com.flipkart.ranger.zookeeper.serde.ZkNodeDataSerializer; import com.google.common.collect.Maps; +import lombok.val; import org.apache.curator.test.TestingCluster; import org.junit.After; import org.junit.Assert; @@ -62,56 +61,16 @@ public void stopTestCluster() throws Exception { } } - private static final class TestShardInfo { - private int shardId; - - public TestShardInfo(int shardId) { - this.shardId = shardId; - } - - public TestShardInfo() { - } - - public int getShardId() { - return shardId; - } - - public void setShardId(int shardId) { - this.shardId = shardId; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - TestShardInfo that = (TestShardInfo) o; - - if (shardId != that.shardId) return false; - - return true; - } - - @Override - public int hashCode() { - return shardId; - } - - private static Criteria getCriteria(int shardId){ - return nodeData -> nodeData.getShardId() == shardId; - } - } - @Test public void testBasicDiscovery() { - SimpleShardedServiceFinder> serviceFinder = ServiceFinderBuilders.>shardedFinderBuilder() + val serviceFinder = ServiceFinderBuilders.shardedFinderBuilder() .withConnectionString(testingCluster.getConnectString()) .withNamespace("test") .withServiceName("test-service") .withDeserializer(data -> { try { return objectMapper.readValue(data, - new TypeReference>() { + new TypeReference>() { }); } catch (IOException e) { e.printStackTrace(); @@ -121,13 +80,13 @@ public void testBasicDiscovery() { .withNodeRefreshIntervalMs(1000) .build(); serviceFinder.start(); - ServiceNode node = serviceFinder.get(TestShardInfo.getCriteria(1)); + ServiceNode node = serviceFinder.get(RangerTestUtils.getCriteria(1)); Assert.assertNotNull(node); Assert.assertEquals("localhost-1", node.getHost()); TestServiceProvider testServiceProvider = serviceProviders.get(node.getHost()); testServiceProvider.oor(); TestUtils.sleepForSeconds(6); - Assert.assertNull(serviceFinder.get(TestShardInfo.getCriteria(1))); + Assert.assertNull(serviceFinder.get(RangerTestUtils.getCriteria(1))); serviceFinder.stop(); } @@ -174,7 +133,7 @@ public void oor() { } public void start() throws Exception { - final ServiceProvider> serviceProvider = ServiceProviderBuilders.shardedServiceProviderBuilder() + val serviceProvider = ServiceProviderBuilders.shardedServiceProviderBuilder() .withConnectionString(connectionString) .withNamespace("test") .withServiceName("test-service") @@ -188,7 +147,7 @@ public void start() throws Exception { }) .withHostname(host) .withPort(port) - .withNodeData(new TestShardInfo(shardId)) + .withNodeData(TestNodeData.builder().shardId(shardId).build()) .withHealthcheck(healthcheck) .withHealthUpdateIntervalMs(1000) .build(); @@ -197,7 +156,7 @@ public void start() throws Exception { } private void registerService(String host, int port, int shardId) throws Exception { - TestServiceProvider serviceProvider = new TestServiceProvider(objectMapper, testingCluster.getConnectString(), host, port, shardId); + val serviceProvider = new TestServiceProvider(objectMapper, testingCluster.getConnectString(), host, port, shardId); serviceProvider.start(); serviceProviders.put(host, serviceProvider); } diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderTest.java index 55041fd0..9f5c8323 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderTest.java @@ -19,17 +19,18 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import com.flipkart.ranger.core.finder.SimpleShardedServiceFinder; import com.flipkart.ranger.core.finder.nodeselector.RoundRobinServiceNodeSelector; import com.flipkart.ranger.core.healthcheck.Healthchecks; -import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.serviceprovider.ServiceProvider; +import com.flipkart.ranger.core.units.TestNodeData; +import com.flipkart.ranger.core.utils.RangerTestUtils; import com.flipkart.ranger.zookeeper.ServiceFinderBuilders; import com.flipkart.ranger.zookeeper.ServiceProviderBuilders; import com.flipkart.ranger.zookeeper.serde.ZkNodeDataSerializer; import com.google.common.collect.Lists; import lombok.extern.slf4j.Slf4j; +import lombok.val; import org.apache.curator.test.TestingCluster; import org.junit.After; import org.junit.Assert; @@ -44,7 +45,7 @@ public class ServiceProviderTest { private TestingCluster testingCluster; private ObjectMapper objectMapper; - private List>> serviceProviders = Lists.newArrayList(); + private List>> serviceProviders = Lists.newArrayList(); @Before public void startTestCluster() throws Exception { @@ -59,69 +60,21 @@ public void startTestCluster() throws Exception { @After public void stopTestCluster() throws Exception { - for (ServiceProvider> serviceProvider : serviceProviders) { - serviceProvider.stop(); - } + serviceProviders.forEach(ServiceProvider::stop); if (null != testingCluster) { testingCluster.close(); } } - private static final class TestShardInfo { - private int shardId; - - public TestShardInfo(int shardId) { - this.shardId = shardId; - } - - public TestShardInfo() { - } - - public int getShardId() { - return shardId; - } - - public void setShardId(int shardId) { - this.shardId = shardId; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - - TestShardInfo that = (TestShardInfo) o; - - if (shardId != that.shardId) { - return false; - } - - return true; - } - - @Override - public int hashCode() { - return shardId; - } - - private static Criteria getCriteria(int shardId){ - return nodeData -> nodeData.getShardId() == shardId; - } - } - @Test - public void testBasicDiscovery() throws Exception { - SimpleShardedServiceFinder> serviceFinder = ServiceFinderBuilders.>shardedFinderBuilder() + public void testBasicDiscovery() { + val serviceFinder = ServiceFinderBuilders.shardedFinderBuilder() .withConnectionString(testingCluster.getConnectString()) .withNamespace("test") .withServiceName("test-service") .withDeserializer(data -> { try { - return objectMapper.readValue(data, new TypeReference>() {}); + return objectMapper.readValue(data, new TypeReference>() {}); } catch (IOException e) { e.printStackTrace(); @@ -131,24 +84,24 @@ public void testBasicDiscovery() throws Exception { .build(); serviceFinder.start(); { - ServiceNode node = serviceFinder.get(TestShardInfo.getCriteria(1)); + val node = serviceFinder.get(RangerTestUtils.getCriteria(1)); Assert.assertNotNull(node); Assert.assertEquals(1, node.getNodeData().getShardId()); } { - ServiceNode node = serviceFinder.get(TestShardInfo.getCriteria(1)); + val node = serviceFinder.get(RangerTestUtils.getCriteria(1)); Assert.assertNotNull(node); Assert.assertEquals(1, node.getNodeData().getShardId()); } long startTime = System.currentTimeMillis(); for (long i = 0; i < 1000000; i++) { - ServiceNode node = serviceFinder.get(TestShardInfo.getCriteria(2)); + val node = serviceFinder.get(RangerTestUtils.getCriteria(2)); Assert.assertNotNull(node); Assert.assertEquals(2, node.getNodeData().getShardId()); } log.info("PERF::RandomSelector::Took (ms):" + (System.currentTimeMillis() - startTime)); { - ServiceNode node = serviceFinder.get(TestShardInfo.getCriteria(99)); + val node = serviceFinder.get(RangerTestUtils.getCriteria(99)); Assert.assertNull(node); } serviceFinder.stop(); @@ -157,19 +110,18 @@ public void testBasicDiscovery() throws Exception { @Test public void testBasicDiscoveryRR() throws Exception { - SimpleShardedServiceFinder> serviceFinder - = ServiceFinderBuilders.>shardedFinderBuilder() + val serviceFinder + = ServiceFinderBuilders.shardedFinderBuilder() .withConnectionString(testingCluster.getConnectString()) .withNamespace("test") .withServiceName("test-service") - .withNodeSelector(new RoundRobinServiceNodeSelector()) + .withNodeSelector(new RoundRobinServiceNodeSelector<>()) .withDeserializer(data -> { try { return objectMapper.readValue(data, - new TypeReference>() { - }); - } - catch (IOException e) { + new TypeReference>() { + }); + } catch (IOException e) { e.printStackTrace(); } return null; @@ -177,24 +129,24 @@ public void testBasicDiscoveryRR() throws Exception { .build(); serviceFinder.start(); { - ServiceNode node = serviceFinder.get(TestShardInfo.getCriteria(1)); + val node = serviceFinder.get(RangerTestUtils.getCriteria(1)); Assert.assertNotNull(node); Assert.assertEquals(1, node.getNodeData().getShardId()); } { - ServiceNode node = serviceFinder.get(TestShardInfo.getCriteria(1)); + val node = serviceFinder.get(RangerTestUtils.getCriteria(1)); Assert.assertNotNull(node); Assert.assertEquals(1, node.getNodeData().getShardId()); } long startTime = System.currentTimeMillis(); for (long i = 0; i < 1000000; i++) { - ServiceNode node = serviceFinder.get(TestShardInfo.getCriteria(2)); + ServiceNode node = serviceFinder.get(RangerTestUtils.getCriteria(2)); Assert.assertNotNull(node); Assert.assertEquals(2, node.getNodeData().getShardId()); } log.info("PERF::RoundRobinSelector::Took (ms):" + (System.currentTimeMillis() - startTime)); { - ServiceNode node = serviceFinder.get(TestShardInfo.getCriteria(99)); + val node = serviceFinder.get(RangerTestUtils.getCriteria(99)); Assert.assertNull(node); } serviceFinder.stop(); @@ -203,16 +155,16 @@ public void testBasicDiscoveryRR() throws Exception { @Test public void testVisibility() { - SimpleShardedServiceFinder> serviceFinder = ServiceFinderBuilders. - >shardedFinderBuilder() + val serviceFinder = ServiceFinderBuilders. + shardedFinderBuilder() .withConnectionString(testingCluster.getConnectString()) .withNamespace("test") .withServiceName("test-service") - .withNodeSelector(new RoundRobinServiceNodeSelector()) + .withNodeSelector(new RoundRobinServiceNodeSelector<>()) .withDeserializer(data -> { try { return objectMapper.readValue(data, - new TypeReference>() { + new TypeReference>() { }); } catch (IOException e) { e.printStackTrace(); @@ -221,33 +173,30 @@ public void testVisibility() { }) .build(); serviceFinder.start(); - List> all = serviceFinder.getAll(TestShardInfo.getCriteria(1)); + List> all = serviceFinder.getAll(RangerTestUtils.getCriteria(1)); log.info("Testing ServiceFinder.getAll()"); - for (ServiceNode serviceNode : all) { - log.info("node = " + serviceNode.getHost() + ":" + serviceNode.getPort() + " " + serviceNode.getHealthcheckStatus() + " " + serviceNode - .getLastUpdatedTimeStamp()); - } + all.stream().map(serviceNode -> "node = " + serviceNode.getHost() + ":" + serviceNode.getPort() + " " + serviceNode.getHealthcheckStatus() + " " + serviceNode + .getLastUpdatedTimeStamp()).forEach(log::info); Assert.assertEquals(3, all.size()); serviceFinder.stop(); } private void registerService(String host, int port, int shardId) { - final ServiceProvider> serviceProvider = ServiceProviderBuilders.shardedServiceProviderBuilder() + val serviceProvider = ServiceProviderBuilders.shardedServiceProviderBuilder() .withConnectionString(testingCluster.getConnectString()) .withNamespace("test") .withServiceName("test-service") .withSerializer(data -> { try { return objectMapper.writeValueAsBytes(data); - } - catch (JsonProcessingException e) { + } catch (JsonProcessingException e) { e.printStackTrace(); } return null; }) .withHostname(host) .withPort(port) - .withNodeData(new TestShardInfo(shardId)) + .withNodeData(TestNodeData.builder().shardId(shardId).build()) .withHealthcheck(Healthchecks.defaultHealthyCheck()) .withHealthUpdateIntervalMs(15000) .build(); diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/SimpleServiceProviderTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/SimpleServiceProviderTest.java index e875c186..934ee8b9 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/SimpleServiceProviderTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/SimpleServiceProviderTest.java @@ -19,9 +19,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import com.flipkart.ranger.core.finder.SimpleUnshardedServiceFinder; import com.flipkart.ranger.core.healthcheck.Healthchecks; -import com.flipkart.ranger.core.model.Criteria; import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.zookeeper.ServiceFinderBuilders; import com.flipkart.ranger.zookeeper.ServiceProviderBuilders; @@ -73,7 +71,7 @@ public boolean equals(Object obj) { @Test public void testBasicDiscovery() { - SimpleUnshardedServiceFinder> serviceFinder = ServiceFinderBuilders.>unshardedFinderBuilder() + val serviceFinder = ServiceFinderBuilders.unshardedFinderBuilder() .withConnectionString(testingCluster.getConnectString()) .withNamespace("test") .withServiceName("test-service") @@ -91,7 +89,7 @@ public void testBasicDiscovery() { .build(); serviceFinder.start(); { - ServiceNode node = serviceFinder.get(null); + val node = serviceFinder.get(null); Assert.assertNotNull(node); System.out.println(node.getHost()); } @@ -99,7 +97,7 @@ public void testBasicDiscovery() { long startTime = System.currentTimeMillis(); for(long i = 0; i <1000000; i++) { - ServiceNode node = serviceFinder.get(null); + val node = serviceFinder.get(null); Assert.assertNotNull(node); frequency.add(node.getHost()); } diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/servicehub/ServiceHubTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/servicehub/ServiceHubTest.java index e1731382..e0d3a745 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/servicehub/ServiceHubTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/servicehub/ServiceHubTest.java @@ -15,23 +15,21 @@ */ package com.flipkart.ranger.zookeeper.servicehub; -import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import com.flipkart.ranger.core.TestUtils; import com.flipkart.ranger.core.finder.serviceregistry.MapBasedServiceRegistry; import com.flipkart.ranger.core.healthcheck.HealthcheckResult; import com.flipkart.ranger.core.healthcheck.HealthcheckStatus; -import com.flipkart.ranger.core.model.Criteria; -import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.signals.ExternalTriggeredSignal; +import com.flipkart.ranger.core.units.TestNodeData; import com.flipkart.ranger.core.util.Exceptions; +import com.flipkart.ranger.core.utils.RangerTestUtils; +import com.flipkart.ranger.core.utils.TestUtils; import com.flipkart.ranger.zookeeper.ServiceProviderBuilders; import com.flipkart.ranger.zookeeper.servicefinderhub.ZkServiceDataSource; import com.flipkart.ranger.zookeeper.servicefinderhub.ZkServiceFinderHubBuilder; import com.flipkart.ranger.zookeeper.servicefinderhub.ZkShardedServiceFinderFactory; -import lombok.Data; import lombok.extern.slf4j.Slf4j; import lombok.val; import org.apache.curator.framework.CuratorFramework; @@ -86,12 +84,12 @@ public void stopTestCluster() throws Exception { @Test public void testHub() { ExternalTriggeredSignal refreshHubSignal = new ExternalTriggeredSignal<>(() -> null, Collections.emptyList()); - val hub = new ZkServiceFinderHubBuilder, MapBasedServiceRegistry>() + val hub = new ZkServiceFinderHubBuilder>() .withCuratorFramework(curatorFramework) .withNamespace("test") .withRefreshFrequencyMs(1000) .withServiceDataSource(new ZkServiceDataSource("test", testingCluster.getConnectString(), curatorFramework)) - .withServiceFinderFactory(ZkShardedServiceFinderFactory.>builder() + .withServiceFinderFactory(ZkShardedServiceFinderFactory.builder() .curatorFramework(curatorFramework) .deserializer(this::read) .build()) @@ -99,18 +97,18 @@ public void testHub() { .build(); hub.start(); - ExternalTriggeredSignal refreshProviderSignal = new ExternalTriggeredSignal<>( + val refreshProviderSignal = new ExternalTriggeredSignal<>( () -> HealthcheckResult.builder() .status(HealthcheckStatus.healthy) .updatedTime(new Date().getTime()) .build(), Collections.emptyList()); - val provider1 = ServiceProviderBuilders.shardedServiceProviderBuilder() + val provider1 = ServiceProviderBuilders.shardedServiceProviderBuilder() .withHostname("localhost") .withPort(1080) .withNamespace(NAMESPACE) .withServiceName("s1") .withSerializer(this::write) - .withNodeData(new TestShardInfo("prod")) + .withNodeData(TestNodeData.builder().shardId(1).build()) .withHealthcheck(() -> HealthcheckStatus.healthy) .withAdditionalRefreshSignal(refreshProviderSignal) .withCuratorFramework(curatorFramework) @@ -121,26 +119,17 @@ public void testHub() { refreshHubSignal.trigger(); TestUtils.sleepForSeconds(3); - val node = hub.finder(new Service(NAMESPACE, "s1")) - .map(finder -> finder.get(nodeData -> nodeData.getEnvironment().equalsIgnoreCase("prod"))) + val node = hub.finder(RangerTestUtils.getService(NAMESPACE, "s1")) + .map(finder -> finder.get(nodeData -> nodeData.getShardId() == 1)) .orElse(null); Assert.assertNotNull(node); hub.stop(); provider1.stop(); } - @Data - private static final class TestShardInfo { - private final String environment; - - private TestShardInfo(@JsonProperty("environment") String environment) { - this.environment = environment; - } - } - - private ServiceNode read(final byte[] data) { + private ServiceNode read(final byte[] data) { try { - return objectMapper.readValue(data, new TypeReference>() {}); + return objectMapper.readValue(data, new TypeReference>() {}); } catch (IOException e) { Exceptions.illegalState(e); @@ -148,7 +137,7 @@ private ServiceNode read(final byte[] data) { return null; } - private byte[] write(final ServiceNode node) { + private byte[] write(final ServiceNode node) { try { return objectMapper.writeValueAsBytes(node); } From 5945a29a4e467b02b8d8eb6a9303c2158c91edc1 Mon Sep 17 00:00:00 2001 From: Koushik R Date: Mon, 20 Dec 2021 14:34:45 +0530 Subject: [PATCH 27/69] a) Uniform lombok usage across the codebase b) Changed the serviceFinder.get to return an optional, uniform optional usage across the codebase c) Uniform lambda usage across the codebase d) Moved the bundle construction inside the ranger-server-bundle by introducing a lifecycle object --- pom.xml | 1 + .../client/AbstractRangerHubClient.java | 17 ++++--- .../ranger/client/RangerHubClient.java | 4 +- .../ranger/client/stubs/RangerTestHub.java | 4 +- .../ranger/core/finder/ServiceFinder.java | 7 +-- .../ServiceRegistryUpdater.java | 2 +- .../core/finderhub/ServiceDataSource.java | 4 +- .../core/finderhub/StaticDataSource.java | 6 +-- .../core/healthcheck/HealthChecker.java | 3 +- .../ServiceHealthAggregator.java | 20 ++++----- .../flipkart/ranger/core/model/Service.java | 7 +-- .../core/finder/SimpleShardFinderTest.java | 2 +- .../finder/UnshardedClusterFinderTest.java | 4 +- .../shardselector/ListShardSelectorTest.java | 2 +- .../shardselector/NoopShardSelectorTest.java | 20 +++++++++ .../client/http/SimpleRangerHttpClient.java | 2 +- .../http/UnshardedRangerHttpHubClient.java | 6 +-- .../client/http/BaseRangerHttpClientTest.java | 8 ++-- ranger-http-model/pom.xml | 22 ++++++++++ .../http/response/model/GenericResponse.java | 37 ++++++++++++++++ .../response/model/RangerResponseCode.java | 15 ++----- .../http/server/bundle/HttpServerBundle.java | 2 - ranger-http/pom.xml | 5 +++ .../common/HttpNodeDataStoreConnector.java | 2 +- .../http/model/ServiceDataSourceResponse.java | 13 ++++-- .../http/model/ServiceNodesResponse.java | 11 ++++- .../model/ServiceRegistrationResponse.java | 9 +++- .../HttpServiceDataSource.java | 7 ++- .../serviceprovider/HttpNodeDataSink.java | 14 +++--- .../ServiceRegistrationResponseTest.java | 5 ++- .../HttpShardedServiceFinderBuilderTest.java | 5 ++- .../HttpServiceDataSourceTest.java | 7 +-- ...HttpShardedServiceProviderBuilderTest.java | 10 +++-- .../fixtures/serviceNodesResponse.json | 2 +- .../resources/fixtures/serviceResponse.json | 2 +- ranger-server-bundle/pom.xml | 6 +++ .../bundle/resources/RangerResource.java | 38 ++++++---------- .../client/zk/AbstractRangerZKHubClient.java | 16 +++---- .../client/zk/ShardedRangerZKHubClient.java | 4 +- .../client/zk/SimpleRangerZKClient.java | 2 +- .../client/zk/UnshardedRangerZKHubClient.java | 4 +- .../servicefinderhub/ZkServiceDataSource.java | 6 +-- .../model/CustomShardSelectorTest.java | 6 +-- .../model/ServiceNoProviderTest.java | 8 ++-- .../model/ServiceProviderExtCuratorTest.java | 25 +++++------ .../model/ServiceProviderHealthcheckTest.java | 10 ++--- .../zookeeper/model/ServiceProviderTest.java | 44 +++++++++---------- .../model/SimpleServiceProviderTest.java | 16 +++---- .../zookeeper/servicehub/ServiceHubTest.java | 6 +-- .../BaseServiceProviderBuilderTest.java | 1 + 50 files changed, 287 insertions(+), 192 deletions(-) create mode 100644 ranger-core/src/test/java/com/flipkart/ranger/core/finder/shardselector/NoopShardSelectorTest.java create mode 100644 ranger-http-model/pom.xml create mode 100644 ranger-http-model/src/main/java/com/flipkart/ranger/http/response/model/GenericResponse.java rename ranger-server-bundle/src/main/java/com/flipkart/ranger/zk/server/bundle/model/GenericResponse.java => ranger-http-model/src/main/java/com/flipkart/ranger/http/response/model/RangerResponseCode.java (71%) diff --git a/pom.xml b/pom.xml index de45d82f..38732e97 100644 --- a/pom.xml +++ b/pom.xml @@ -19,6 +19,7 @@ ranger-server-bundle ranger-zk-server ranger-http-server + ranger-http-model diff --git a/ranger-client/src/main/java/com/flipkart/ranger/client/AbstractRangerHubClient.java b/ranger-client/src/main/java/com/flipkart/ranger/client/AbstractRangerHubClient.java index 7d548c85..266ef666 100644 --- a/ranger-client/src/main/java/com/flipkart/ranger/client/AbstractRangerHubClient.java +++ b/ranger-client/src/main/java/com/flipkart/ranger/client/AbstractRangerHubClient.java @@ -28,10 +28,10 @@ import lombok.extern.slf4j.Slf4j; import lombok.val; -import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.function.Predicate; @Slf4j @@ -44,9 +44,9 @@ public abstract class AbstractRangerHubClient, D private int nodeRefreshTimeMs; private ServiceFinderHub hub; - private Predicate criteria; + private final Predicate criteria; - public AbstractRangerHubClient( + protected AbstractRangerHubClient( String namespace, ObjectMapper mapper, int nodeRefreshTimeMs, @@ -96,7 +96,7 @@ public Optional> getNode( final Predicate criteria ){ val optionalFinder = this.getHub().finder(service); - return optionalFinder.map(trcServiceFinder -> trcServiceFinder.get(criteria)); + return optionalFinder.flatMap(trServiceFinder -> trServiceFinder.get(criteria)); } public List> getAllNodes( @@ -107,8 +107,13 @@ public List> getAllNodes( return optionalFinder.map(trServiceFinder -> trServiceFinder.getAll(criteria)).orElse(Collections.emptyList()); } - public Collection getServices() throws Exception { - return this.getHub().getServiceDataSource().services(); + public Set getServices() { + try{ + return this.getHub().getServiceDataSource().services(); + }catch (Exception e){ + log.warn("Call to a hub failed with exception, {}", e.getMessage()); + return Collections.emptySet(); + } } protected abstract ServiceFinderHub buildHub(); diff --git a/ranger-client/src/main/java/com/flipkart/ranger/client/RangerHubClient.java b/ranger-client/src/main/java/com/flipkart/ranger/client/RangerHubClient.java index 2dde78cd..933d8517 100644 --- a/ranger-client/src/main/java/com/flipkart/ranger/client/RangerHubClient.java +++ b/ranger-client/src/main/java/com/flipkart/ranger/client/RangerHubClient.java @@ -18,9 +18,9 @@ import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.core.model.ServiceNode; -import java.util.Collection; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.function.Predicate; public interface RangerHubClient { @@ -28,7 +28,7 @@ public interface RangerHubClient { void stop(); - Collection getServices() throws Exception; + Set getServices(); Optional> getNode(final Service service); diff --git a/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/RangerTestHub.java b/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/RangerTestHub.java index 059d4ef5..0bb28839 100644 --- a/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/RangerTestHub.java +++ b/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/RangerTestHub.java @@ -21,7 +21,7 @@ import com.flipkart.ranger.core.finder.serviceregistry.ListBasedServiceRegistry; import com.flipkart.ranger.core.finderhub.*; import com.flipkart.ranger.core.units.TestNodeData; -import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import lombok.Builder; import java.util.function.Predicate; @@ -54,7 +54,7 @@ protected void postBuild(ServiceFinderHub get(Predicate criteria) { + public Optional> get(Predicate criteria) { List> nodes = shardSelector.nodes(criteria, serviceRegistry); if (null == nodes || nodes.isEmpty()) { - return null; + return Optional.empty(); } - return nodeSelector.select(nodes); + return Optional.of(nodeSelector.select(nodes)); } public List> getAll(Predicate criteria) { diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/serviceregistry/ServiceRegistryUpdater.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/serviceregistry/ServiceRegistryUpdater.java index ec437e5c..a2147335 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/serviceregistry/ServiceRegistryUpdater.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/serviceregistry/ServiceRegistryUpdater.java @@ -134,7 +134,7 @@ private void updateRegistry() throws InterruptedException { return; } val nodeList = nodeDataSource.refresh(deserializer); - if (!nodeList.isEmpty()) { + if (null != nodeList) { log.debug("Updating nodelist of size: {} for [{}]", nodeList.size(), serviceRegistry.getService().getServiceName()); serviceRegistry.updateNodes(nodeList); diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/ServiceDataSource.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/ServiceDataSource.java index dc579606..5a9e0e50 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/ServiceDataSource.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/ServiceDataSource.java @@ -17,14 +17,14 @@ import com.flipkart.ranger.core.model.Service; -import java.util.Collection; +import java.util.Set; /** * */ public interface ServiceDataSource { - Collection services() throws Exception; + Set services() throws Exception; void start(); void stop(); diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/StaticDataSource.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/StaticDataSource.java index 0d66faea..4977668d 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/StaticDataSource.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/StaticDataSource.java @@ -19,7 +19,7 @@ import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; -import java.util.Collection; +import java.util.Set; /* A static data source to be used when we know the services beforehand and don't have to fetch from a source. @@ -29,10 +29,10 @@ @AllArgsConstructor public class StaticDataSource implements ServiceDataSource{ - private final Collection services; + private final Set services; @Override - public Collection services() throws Exception { + public Set services() throws Exception { return services; } diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/healthcheck/HealthChecker.java b/ranger-core/src/main/java/com/flipkart/ranger/core/healthcheck/HealthChecker.java index 3d7dbd3b..80f8b786 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/healthcheck/HealthChecker.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/healthcheck/HealthChecker.java @@ -16,6 +16,7 @@ package com.flipkart.ranger.core.healthcheck; import lombok.extern.slf4j.Slf4j; +import lombok.val; import java.util.List; import java.util.function.Supplier; @@ -66,7 +67,7 @@ private boolean refreshHealth() { //1. First time //2. Stale update threshold breach //3. Update in health status - long currentTime = System.currentTimeMillis(); + val currentTime = System.currentTimeMillis(); if (lastHealthcheckStatus == null || (currentTime - lastUpdatedTime) > staleUpdateThreshold || lastHealthcheckStatus != healthcheckStatus) { diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/ServiceHealthAggregator.java b/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/ServiceHealthAggregator.java index 2468a633..9b50cb8e 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/ServiceHealthAggregator.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/ServiceHealthAggregator.java @@ -106,18 +106,14 @@ public void start() { } ScheduledExecutorService scheduledExecutorService = Executors.newScheduledThreadPool(isolatedHealthMonitorList.size()); scheduledFutureList = Lists.newArrayListWithCapacity(isolatedHealthMonitorList.size()); - for (IsolatedHealthMonitor isolatedHealthMonitor : isolatedHealthMonitorList) { - final ScheduledFuture scheduledFuture = - scheduledExecutorService.scheduleWithFixedDelay( - isolatedHealthMonitor, - isolatedHealthMonitor.getRunInterval() - .getInitialDelay(), - isolatedHealthMonitor.getRunInterval() - .getTimeInterval(), - isolatedHealthMonitor.getRunInterval() - .getTimeUnit()); - scheduledFutureList.add(scheduledFuture); - } + isolatedHealthMonitorList.stream().map(isolatedHealthMonitor -> scheduledExecutorService.scheduleWithFixedDelay( + isolatedHealthMonitor, + isolatedHealthMonitor.getRunInterval() + .getInitialDelay(), + isolatedHealthMonitor.getRunInterval() + .getTimeInterval(), + isolatedHealthMonitor.getRunInterval() + .getTimeUnit())).forEach(scheduledFuture -> scheduledFutureList.add(scheduledFuture)); running.set(true); } diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/model/Service.java b/ranger-core/src/main/java/com/flipkart/ranger/core/model/Service.java index 164740c7..083ac956 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/model/Service.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/model/Service.java @@ -15,11 +15,12 @@ */ package com.flipkart.ranger.core.model; -import lombok.Builder; -import lombok.Value; +import lombok.*; -@Value +@Data +@NoArgsConstructor +@AllArgsConstructor(access = AccessLevel.PUBLIC) @Builder public class Service { String namespace; diff --git a/ranger-core/src/test/java/com/flipkart/ranger/core/finder/SimpleShardFinderTest.java b/ranger-core/src/test/java/com/flipkart/ranger/core/finder/SimpleShardFinderTest.java index ddd1a71e..83ba898b 100644 --- a/ranger-core/src/test/java/com/flipkart/ranger/core/finder/SimpleShardFinderTest.java +++ b/ranger-core/src/test/java/com/flipkart/ranger/core/finder/SimpleShardFinderTest.java @@ -49,6 +49,6 @@ public void testSimpleShardedFinder(){ serviceRegistry, shardSelector, roundRobinServiceNodeSelector); val testNodeDataServiceNode = simpleShardedFinder.get( RangerTestUtils.getCriteria(2)); - Assert.assertNull(testNodeDataServiceNode); + Assert.assertFalse(testNodeDataServiceNode.isPresent()); } } diff --git a/ranger-core/src/test/java/com/flipkart/ranger/core/finder/UnshardedClusterFinderTest.java b/ranger-core/src/test/java/com/flipkart/ranger/core/finder/UnshardedClusterFinderTest.java index ae4e81d8..2bfb245a 100644 --- a/ranger-core/src/test/java/com/flipkart/ranger/core/finder/UnshardedClusterFinderTest.java +++ b/ranger-core/src/test/java/com/flipkart/ranger/core/finder/UnshardedClusterFinderTest.java @@ -46,7 +46,7 @@ public void unshardedClusterFinder(){ new TestUnshardedNodeSelector() ); val serviceNode = simpleUnshardedServiceFinder.get(RangerTestUtils.getCriteria(1)); - Assert.assertNotNull(serviceNode); - Assert.assertEquals("localhost-1", serviceNode.getHost()); + Assert.assertTrue(serviceNode.isPresent()); + Assert.assertEquals("localhost-1", serviceNode.get().getHost()); } } diff --git a/ranger-core/src/test/java/com/flipkart/ranger/core/finder/shardselector/ListShardSelectorTest.java b/ranger-core/src/test/java/com/flipkart/ranger/core/finder/shardselector/ListShardSelectorTest.java index 187b8901..e27d1c85 100644 --- a/ranger-core/src/test/java/com/flipkart/ranger/core/finder/shardselector/ListShardSelectorTest.java +++ b/ranger-core/src/test/java/com/flipkart/ranger/core/finder/shardselector/ListShardSelectorTest.java @@ -25,7 +25,7 @@ public class ListShardSelectorTest { @Test - public void testNoOpShardSelector(){ + public void testListShardSelector(){ val serviceRegistry = RegistryTestUtils.getUnshardedRegistry(); val shardSelector = new ListShardSelector(); val nodes = shardSelector.nodes(RangerTestUtils.getCriteria(1), serviceRegistry); diff --git a/ranger-core/src/test/java/com/flipkart/ranger/core/finder/shardselector/NoopShardSelectorTest.java b/ranger-core/src/test/java/com/flipkart/ranger/core/finder/shardselector/NoopShardSelectorTest.java new file mode 100644 index 00000000..66d8823a --- /dev/null +++ b/ranger-core/src/test/java/com/flipkart/ranger/core/finder/shardselector/NoopShardSelectorTest.java @@ -0,0 +1,20 @@ +package com.flipkart.ranger.core.finder.shardselector; + +import com.flipkart.ranger.core.units.TestNodeData; +import com.flipkart.ranger.core.utils.RangerTestUtils; +import com.flipkart.ranger.core.utils.RegistryTestUtils; +import lombok.val; +import org.junit.Assert; +import org.junit.Test; + +public class NoopShardSelectorTest { + + @Test + public void testNoOpShardSelector(){ + val serviceRegistry = RegistryTestUtils.getUnshardedRegistry(); + val shardSelector = new NoopShardSelector(); + val nodes = shardSelector.nodes(RangerTestUtils.getCriteria(1), serviceRegistry); + Assert.assertFalse(nodes.isEmpty()); + Assert.assertEquals("localhost-1", nodes.get(0).getHost()); + } +} diff --git a/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/SimpleRangerHttpClient.java b/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/SimpleRangerHttpClient.java index a0c9cb36..b73cbb2e 100644 --- a/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/SimpleRangerHttpClient.java +++ b/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/SimpleRangerHttpClient.java @@ -88,7 +88,7 @@ public List> getAllNodes() { @Override public Optional> getNode(Predicate criteria) { - return Optional.ofNullable(this.serviceFinder.get(criteria)); + return this.serviceFinder.get(criteria); } @Override diff --git a/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/UnshardedRangerHttpHubClient.java b/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/UnshardedRangerHttpHubClient.java index 43331a0c..fad819c5 100644 --- a/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/UnshardedRangerHttpHubClient.java +++ b/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/UnshardedRangerHttpHubClient.java @@ -33,14 +33,14 @@ import lombok.Builder; import lombok.extern.slf4j.Slf4j; -import java.util.List; +import java.util.Set; import java.util.function.Predicate; @Slf4j public class UnshardedRangerHttpHubClient extends AbstractRangerHubClient, HTTPResponseDataDeserializer> { - private final List services; + private final Set services; private final HttpClientConfig clientConfig; @Builder @@ -51,7 +51,7 @@ public UnshardedRangerHttpHubClient( Predicate criteria, HTTPResponseDataDeserializer deserializer, HttpClientConfig clientConfig, - List services + Set services ) { super(namespace, mapper, nodeRefreshIntervalMs, criteria, deserializer); this.clientConfig = clientConfig; diff --git a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/BaseRangerHttpClientTest.java b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/BaseRangerHttpClientTest.java index 98a9a17b..3ed8d871 100644 --- a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/BaseRangerHttpClientTest.java +++ b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/BaseRangerHttpClientTest.java @@ -25,8 +25,10 @@ import com.flipkart.ranger.http.config.HttpClientConfig; import com.flipkart.ranger.http.model.ServiceDataSourceResponse; import com.flipkart.ranger.http.model.ServiceNodesResponse; +import com.flipkart.ranger.http.response.model.RangerResponseCode; import com.github.tomakehurst.wiremock.junit.WireMockRule; import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import lombok.val; @@ -56,7 +58,7 @@ public void startTestCluster() throws Exception { val payload = objectMapper.writeValueAsBytes( ServiceNodesResponse.builder() .data(Lists.newArrayList(node)) - .success(true) + .code(RangerResponseCode.SUCCESS) .build()); server.stubFor(get(urlEqualTo("/ranger/nodes/v1/test-n/test-s")) .willReturn(aResponse() @@ -64,8 +66,8 @@ public void startTestCluster() throws Exception { .withStatus(200))); val responseObj = ServiceDataSourceResponse.builder() - .success(true) - .data(Lists.newArrayList( + .code(RangerResponseCode.SUCCESS) + .data(Sets.newHashSet( RangerTestUtils.getService("test-n", "test-s") )) .build(); diff --git a/ranger-http-model/pom.xml b/ranger-http-model/pom.xml new file mode 100644 index 00000000..027bc178 --- /dev/null +++ b/ranger-http-model/pom.xml @@ -0,0 +1,22 @@ + + + + ranger + com.flipkart.ranger + 1.0-SNAPSHOT + + 4.0.0 + + ranger-http-model + + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + + + + \ No newline at end of file diff --git a/ranger-http-model/src/main/java/com/flipkart/ranger/http/response/model/GenericResponse.java b/ranger-http-model/src/main/java/com/flipkart/ranger/http/response/model/GenericResponse.java new file mode 100644 index 00000000..1b279b9d --- /dev/null +++ b/ranger-http-model/src/main/java/com/flipkart/ranger/http/response/model/GenericResponse.java @@ -0,0 +1,37 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.http.response.model; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import lombok.Builder; +import lombok.Value; +import lombok.extern.jackson.Jacksonized; + +@Value +@Builder +@Jacksonized +@JsonIgnoreProperties(ignoreUnknown = true) +public class GenericResponse { + RangerResponseCode code; + String error; + T data; + + @JsonIgnore + public boolean success(){ + return code == RangerResponseCode.SUCCESS; + } +} diff --git a/ranger-server-bundle/src/main/java/com/flipkart/ranger/zk/server/bundle/model/GenericResponse.java b/ranger-http-model/src/main/java/com/flipkart/ranger/http/response/model/RangerResponseCode.java similarity index 71% rename from ranger-server-bundle/src/main/java/com/flipkart/ranger/zk/server/bundle/model/GenericResponse.java rename to ranger-http-model/src/main/java/com/flipkart/ranger/http/response/model/RangerResponseCode.java index 32b11222..e528e5f6 100644 --- a/ranger-server-bundle/src/main/java/com/flipkart/ranger/zk/server/bundle/model/GenericResponse.java +++ b/ranger-http-model/src/main/java/com/flipkart/ranger/http/response/model/RangerResponseCode.java @@ -13,17 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.flipkart.ranger.zk.server.bundle.model; +package com.flipkart.ranger.http.response.model; -import lombok.Builder; -import lombok.Value; -import lombok.extern.jackson.Jacksonized; +public enum RangerResponseCode { + + SUCCESS; -@Value -@Builder -@Jacksonized -public class GenericResponse { - boolean success; - String error; - T data; } diff --git a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/bundle/HttpServerBundle.java b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/bundle/HttpServerBundle.java index cd4ab7ce..21ebd918 100644 --- a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/bundle/HttpServerBundle.java +++ b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/bundle/HttpServerBundle.java @@ -20,7 +20,6 @@ import com.flipkart.ranger.client.RangerHubClient; import com.flipkart.ranger.client.http.UnshardedRangerHttpHubClient; import com.flipkart.ranger.common.server.ShardInfo; -import com.flipkart.ranger.http.config.HttpClientConfig; import com.flipkart.ranger.http.model.ServiceNodesResponse; import com.flipkart.ranger.http.server.AppConfiguration; import com.flipkart.ranger.zk.server.bundle.RangerServerBundle; @@ -32,7 +31,6 @@ import javax.inject.Singleton; import java.io.IOException; -import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; diff --git a/ranger-http/pom.xml b/ranger-http/pom.xml index 65c92be9..44bfc0e7 100644 --- a/ranger-http/pom.xml +++ b/ranger-http/pom.xml @@ -17,6 +17,11 @@ ranger-core ${project.version} + + com.flipkart.ranger + ranger-http-model + ${project.version} + com.squareup.okhttp3 okhttp diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/common/HttpNodeDataStoreConnector.java b/ranger-http/src/main/java/com/flipkart/ranger/http/common/HttpNodeDataStoreConnector.java index bbaf51a3..abea1db1 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/common/HttpNodeDataStoreConnector.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/common/HttpNodeDataStoreConnector.java @@ -36,7 +36,7 @@ public class HttpNodeDataStoreConnector implements NodeDataStoreConnector public HttpNodeDataStoreConnector( final HttpClientConfig config, - ObjectMapper mapper) { + final ObjectMapper mapper) { this.httpClient = new OkHttpClient.Builder() .callTimeout(config.getOperationTimeoutMs() == 0 ? 3000 diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceDataSourceResponse.java b/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceDataSourceResponse.java index 6966b729..835be5eb 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceDataSourceResponse.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceDataSourceResponse.java @@ -15,13 +15,15 @@ */ package com.flipkart.ranger.http.model; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.flipkart.ranger.core.model.Service; +import com.flipkart.ranger.http.response.model.RangerResponseCode; import lombok.Builder; import lombok.Value; import lombok.extern.jackson.Jacksonized; -import java.util.List; +import java.util.Set; @Value @@ -29,7 +31,12 @@ @Builder @JsonIgnoreProperties(ignoreUnknown = true) public class ServiceDataSourceResponse { - boolean success; + RangerResponseCode code; String error; - List data; + Set data; + + @JsonIgnore + public boolean isSuccess(){ + return code == RangerResponseCode.SUCCESS; + } } diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceNodesResponse.java b/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceNodesResponse.java index 591c8985..72742dd2 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceNodesResponse.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceNodesResponse.java @@ -15,8 +15,10 @@ */ package com.flipkart.ranger.http.model; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.flipkart.ranger.core.model.ServiceNode; +import com.flipkart.ranger.http.response.model.RangerResponseCode; import lombok.Builder; import lombok.Value; import lombok.extern.jackson.Jacksonized; @@ -30,8 +32,13 @@ @Jacksonized @Builder @JsonIgnoreProperties(ignoreUnknown = true) -public class ServiceNodesResponse { - boolean success; +public class ServiceNodesResponse { + RangerResponseCode code; String error; List> data; + + @JsonIgnore + public boolean isSuccess(){ + return code == RangerResponseCode.SUCCESS; + } } diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceRegistrationResponse.java b/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceRegistrationResponse.java index f7fe0bcc..cdcc4949 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceRegistrationResponse.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceRegistrationResponse.java @@ -15,7 +15,9 @@ */ package com.flipkart.ranger.http.model; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.flipkart.ranger.http.response.model.RangerResponseCode; import lombok.Builder; import lombok.Value; import lombok.extern.jackson.Jacksonized; @@ -26,6 +28,11 @@ @Builder @JsonIgnoreProperties(ignoreUnknown = true) public class ServiceRegistrationResponse { - boolean success; + RangerResponseCode code; String error; + + @JsonIgnore + public boolean isSuccess(){ + return code == RangerResponseCode.SUCCESS; + } } diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSource.java b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSource.java index b5bcd7a7..702c8d18 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSource.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSource.java @@ -30,9 +30,8 @@ import okhttp3.ResponseBody; import java.io.IOException; -import java.util.Collection; import java.util.Collections; -import java.util.List; +import java.util.Set; @Slf4j public class HttpServiceDataSource extends HttpNodeDataStoreConnector implements ServiceDataSource { @@ -42,7 +41,7 @@ public HttpServiceDataSource(HttpClientConfig config, ObjectMapper mapper) { } @Override - public Collection services() { + public Set services() { Preconditions.checkNotNull(config, "client config has not been set for node data"); Preconditions.checkNotNull(mapper, "mapper has not been set for node data"); @@ -61,7 +60,7 @@ public Collection services() { .get() .build(); - List services = Collections.emptyList(); + Set services = Collections.emptySet(); try (val response = httpClient.newCall(request).execute()) { if (response.isSuccessful()) { try (final ResponseBody body = response.body()) { diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/serviceprovider/HttpNodeDataSink.java b/ranger-http/src/main/java/com/flipkart/ranger/http/serviceprovider/HttpNodeDataSink.java index 3988b91e..9ce31b41 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/serviceprovider/HttpNodeDataSink.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/serviceprovider/HttpNodeDataSink.java @@ -30,7 +30,6 @@ import okhttp3.HttpUrl; import okhttp3.Request; import okhttp3.RequestBody; -import okhttp3.ResponseBody; import java.io.IOException; import java.util.Optional; @@ -63,7 +62,8 @@ public void updateState(S serializer, ServiceNode serviceNode) { val requestBody = RequestBody.create(serializer.serialize(serviceNode)); val serviceRegistrationResponse = registerService(httpUrl, requestBody); if(!serviceRegistrationResponse.isPresent() || !serviceRegistrationResponse.get().isSuccess()){ - Exceptions.illegalState("Error updating state on the server for nodedata: " + httpUrl); + log.warn("Http call to {} returned a failure response", httpUrl); + Exceptions.illegalState("Error updating state on the server for node data: " + httpUrl); } } @@ -74,17 +74,13 @@ private Optional registerService(HttpUrl httpUrl, R .build(); try (val response = httpClient.newCall(request).execute()) { if (response.isSuccessful()) { - try (final ResponseBody body = response.body()) { + try (val body = response.body()) { if (null == body) { log.warn("HTTP call to {} returned empty body", httpUrl); } else { - final byte[] bytes = body.bytes(); - val serviceRegistrationResponse = mapper.readValue(bytes, ServiceRegistrationResponse.class); - if(!serviceRegistrationResponse.isSuccess()){ - log.warn("Http call to {} returned a failure response with code {} and error {}", httpUrl, response.code(), serviceRegistrationResponse.getError()); - } - return Optional.of(serviceRegistrationResponse); + val bytes = body.bytes(); + return Optional.of(mapper.readValue(bytes, ServiceRegistrationResponse.class)); } } } diff --git a/ranger-http/src/test/java/com/flipkart/ranger/http/model/ServiceRegistrationResponseTest.java b/ranger-http/src/test/java/com/flipkart/ranger/http/model/ServiceRegistrationResponseTest.java index c56b1e14..4381de30 100644 --- a/ranger-http/src/test/java/com/flipkart/ranger/http/model/ServiceRegistrationResponseTest.java +++ b/ranger-http/src/test/java/com/flipkart/ranger/http/model/ServiceRegistrationResponseTest.java @@ -16,6 +16,7 @@ package com.flipkart.ranger.http.model; import com.flipkart.ranger.http.ResourceHelper; +import com.flipkart.ranger.http.response.model.GenericResponse; import lombok.val; import org.junit.Assert; import org.junit.Test; @@ -24,8 +25,8 @@ public class ServiceRegistrationResponseTest { @Test public void testServiceRegistrationResponse(){ - val resource = ResourceHelper.getResource("fixtures/serviceResponse.json", ServiceRegistrationResponse.class); + val resource = ResourceHelper.getResource("fixtures/serviceResponse.json", GenericResponse.class); Assert.assertNotNull(resource); - Assert.assertTrue(resource.isSuccess()); + Assert.assertTrue(resource.success()); } } diff --git a/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java b/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java index 08bb01ae..5515b85e 100644 --- a/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java +++ b/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java @@ -23,6 +23,7 @@ import com.flipkart.ranger.core.utils.TestUtils; import com.flipkart.ranger.http.config.HttpClientConfig; import com.flipkart.ranger.http.model.ServiceNodesResponse; +import com.flipkart.ranger.http.response.model.RangerResponseCode; import com.github.tomakehurst.wiremock.junit.WireMockRule; import com.google.common.collect.Lists; import lombok.Data; @@ -63,7 +64,7 @@ public void testFinder() throws Exception { val payload = MAPPER.writeValueAsBytes( ServiceNodesResponse.builder() .data(Lists.newArrayList(node)) - .success(true) + .code(RangerResponseCode.SUCCESS) .build()); server.stubFor(get(urlEqualTo("/ranger/nodes/v1/testns/test")) .willReturn(aResponse() @@ -93,7 +94,7 @@ public void testFinder() throws Exception { .build(); finder.start(); TestUtils.sleepForSeconds(3); - Assert.assertNotNull(finder.get(nodeData -> true)); + Assert.assertTrue(finder.get(nodeData -> true).isPresent()); } } \ No newline at end of file diff --git a/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSourceTest.java b/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSourceTest.java index d94bbc01..56dfe711 100644 --- a/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSourceTest.java +++ b/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSourceTest.java @@ -20,8 +20,9 @@ import com.flipkart.ranger.core.utils.RangerTestUtils; import com.flipkart.ranger.http.config.HttpClientConfig; import com.flipkart.ranger.http.model.ServiceDataSourceResponse; +import com.flipkart.ranger.http.response.model.RangerResponseCode; import com.github.tomakehurst.wiremock.junit.WireMockRule; -import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import lombok.Builder; import lombok.Data; import lombok.val; @@ -53,8 +54,8 @@ public TestNodeData(@JsonProperty("farmId") String farmId) { @Test public void testServiceDataSource() throws IOException { val responseObj = ServiceDataSourceResponse.builder() - .success(true) - .data(Lists.newArrayList( + .code(RangerResponseCode.SUCCESS) + .data(Sets.newHashSet( RangerTestUtils.getService("test-n", "test-s"), RangerTestUtils.getService("test-n", "test-s1"), RangerTestUtils.getService("test-n", "test-s2") diff --git a/ranger-http/src/test/java/com/flipkart/ranger/http/serviceprovider/HttpShardedServiceProviderBuilderTest.java b/ranger-http/src/test/java/com/flipkart/ranger/http/serviceprovider/HttpShardedServiceProviderBuilderTest.java index 347cc6fd..6969498d 100644 --- a/ranger-http/src/test/java/com/flipkart/ranger/http/serviceprovider/HttpShardedServiceProviderBuilderTest.java +++ b/ranger-http/src/test/java/com/flipkart/ranger/http/serviceprovider/HttpShardedServiceProviderBuilderTest.java @@ -20,11 +20,13 @@ import com.flipkart.ranger.core.healthcheck.Healthchecks; import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.http.config.HttpClientConfig; -import com.flipkart.ranger.http.model.ServiceRegistrationResponse; +import com.flipkart.ranger.http.response.model.GenericResponse; +import com.flipkart.ranger.http.response.model.RangerResponseCode; import com.github.tomakehurst.wiremock.junit.WireMockRule; import lombok.Builder; import lombok.Data; import lombok.val; +import org.junit.Assert; import org.junit.Rule; import org.junit.Test; @@ -52,8 +54,8 @@ public void testProvider() throws Exception { val nm5NodeData = TestNodeData.builder().farmId("nm5").build(); val testNode = new ServiceNode<>("localhost-1", 80, nm5NodeData); val response = MAPPER.writeValueAsBytes( - ServiceRegistrationResponse.builder() - .success(true) + GenericResponse.builder() + .code(RangerResponseCode.SUCCESS) .build()); byte[] requestBytes = MAPPER.writeValueAsBytes(testNode); server.stubFor(post(urlEqualTo("/ranger/nodes/v1/add/testns/test")) @@ -80,6 +82,8 @@ public void testProvider() throws Exception { .withSerializer(node -> requestBytes) .build(); serviceProvider.start(); + Assert.assertNotNull(serviceProvider); + Assert.assertTrue(serviceProvider.getDataSink().isActive()); } } diff --git a/ranger-http/src/test/resources/fixtures/serviceNodesResponse.json b/ranger-http/src/test/resources/fixtures/serviceNodesResponse.json index 7ac55a8e..4a725388 100644 --- a/ranger-http/src/test/resources/fixtures/serviceNodesResponse.json +++ b/ranger-http/src/test/resources/fixtures/serviceNodesResponse.json @@ -1,5 +1,5 @@ { - "success" : true, + "code" : "SUCCESS", "data" : [ { "host" : "localhost-1", diff --git a/ranger-http/src/test/resources/fixtures/serviceResponse.json b/ranger-http/src/test/resources/fixtures/serviceResponse.json index 753c708a..d167d8a8 100644 --- a/ranger-http/src/test/resources/fixtures/serviceResponse.json +++ b/ranger-http/src/test/resources/fixtures/serviceResponse.json @@ -1,3 +1,3 @@ { - "success" : true + "code" : "SUCCESS" } \ No newline at end of file diff --git a/ranger-server-bundle/pom.xml b/ranger-server-bundle/pom.xml index e4788a0f..b0b9300c 100644 --- a/ranger-server-bundle/pom.xml +++ b/ranger-server-bundle/pom.xml @@ -21,6 +21,12 @@ ranger-client ${project.version} + + + com.flipkart.ranger + ranger-http-model + ${project.version} + com.fasterxml.jackson.core jackson-databind diff --git a/ranger-server-bundle/src/main/java/com/flipkart/ranger/zk/server/bundle/resources/RangerResource.java b/ranger-server-bundle/src/main/java/com/flipkart/ranger/zk/server/bundle/resources/RangerResource.java index d4d91ba1..76ff654f 100644 --- a/ranger-server-bundle/src/main/java/com/flipkart/ranger/zk/server/bundle/resources/RangerResource.java +++ b/ranger-server-bundle/src/main/java/com/flipkart/ranger/zk/server/bundle/resources/RangerResource.java @@ -19,7 +19,7 @@ import com.flipkart.ranger.client.RangerHubClient; import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.core.model.ServiceNode; -import com.flipkart.ranger.zk.server.bundle.model.GenericResponse; +import com.flipkart.ranger.http.response.model.GenericResponse; import lombok.extern.slf4j.Slf4j; import lombok.val; @@ -28,9 +28,11 @@ import javax.validation.constraints.NotNull; import javax.ws.rs.*; import javax.ws.rs.core.MediaType; -import java.util.ArrayList; -import java.util.Collection; import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import static com.flipkart.ranger.http.response.model.RangerResponseCode.SUCCESS; @Slf4j @Consumes(MediaType.APPLICATION_JSON) @@ -48,18 +50,11 @@ public RangerResource(List> rangerHubs){ @GET @Path("/services/v1") @Metered - public GenericResponse> getServices() { - Collection services = new ArrayList<>(); - rangerHubs.forEach(hub -> { - try { - services.addAll(hub.getServices()); - } catch (Exception e) { - log.warn("Call to a hub failed with exception, {}", e.getMessage()); - } - }); - return GenericResponse.>builder() - .success(true) - .data(services) + public GenericResponse> getServices() { + return GenericResponse.>builder() + .code(SUCCESS) + .data(rangerHubs.stream().map(RangerHubClient::getServices) + .flatMap(Set::stream).collect(Collectors.toSet())) .build(); } @@ -71,17 +66,10 @@ public GenericResponse>> getNodes( @NotNull @NotEmpty @PathParam("serviceName") final String serviceName ){ val service = Service.builder().namespace(namespace).serviceName(serviceName).build(); - List> serviceNodes = new ArrayList<>(); - rangerHubs.forEach(hub -> { - try { - serviceNodes.addAll(hub.getAllNodes(service)); - } catch (Exception e) { - log.warn("Call to a hub failed with exception, {}", e.getMessage()); - } - }); return GenericResponse.>>builder() - .success(true) - .data(serviceNodes) + .code(SUCCESS) + .data(rangerHubs.stream().map(hub -> hub.getAllNodes(service)) + .flatMap(List::stream).collect(Collectors.toList())) .build(); } } diff --git a/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/AbstractRangerZKHubClient.java b/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/AbstractRangerZKHubClient.java index 7d44b6cc..c9e1fb23 100644 --- a/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/AbstractRangerZKHubClient.java +++ b/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/AbstractRangerZKHubClient.java @@ -29,7 +29,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.curator.framework.CuratorFramework; -import java.util.List; +import java.util.Set; import java.util.function.Predicate; @Slf4j @@ -40,13 +40,13 @@ public abstract class AbstractRangerZKHubClient, private final boolean disablePushUpdaters; private final String connectionString; private final CuratorFramework curatorFramework; - /* - Use this if you don't want the datasource to fetch the entire list of services; - but know beforehand about the services you are going to build the hub for. - */ - private final List services; + /** + Use this if you don't want the datasource to fetch the entire set of services; + but know beforehand about the services you are going to build the hub for. + **/ + private final Set services; - public AbstractRangerZKHubClient( + protected AbstractRangerZKHubClient( String namespace, ObjectMapper mapper, int nodeRefreshIntervalMs, @@ -55,7 +55,7 @@ public AbstractRangerZKHubClient( CuratorFramework curatorFramework, Predicate criteria, D deserializer, - List services + Set services ) { super(namespace, mapper, nodeRefreshIntervalMs, criteria, deserializer); this.disablePushUpdaters = disablePushUpdaters; diff --git a/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/ShardedRangerZKHubClient.java b/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/ShardedRangerZKHubClient.java index b31c3aee..6d137536 100644 --- a/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/ShardedRangerZKHubClient.java +++ b/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/ShardedRangerZKHubClient.java @@ -27,7 +27,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.curator.framework.CuratorFramework; -import java.util.List; +import java.util.Set; import java.util.function.Predicate; @Slf4j @@ -44,7 +44,7 @@ public ShardedRangerZKHubClient( CuratorFramework curatorFramework, Predicate criteria, ZkNodeDataDeserializer deserializer, - List services + Set services ) { super( namespace, diff --git a/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/SimpleRangerZKClient.java b/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/SimpleRangerZKClient.java index c8e67bf5..bc2f226e 100644 --- a/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/SimpleRangerZKClient.java +++ b/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/SimpleRangerZKClient.java @@ -126,7 +126,7 @@ public List> getAllNodes() { @Override public Optional> getNode(Predicate criteria) { - return Optional.ofNullable(this.serviceFinder.get(criteria)); + return this.serviceFinder.get(criteria); } @Override diff --git a/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/UnshardedRangerZKHubClient.java b/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/UnshardedRangerZKHubClient.java index 2d935383..df7b0bde 100644 --- a/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/UnshardedRangerZKHubClient.java +++ b/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/UnshardedRangerZKHubClient.java @@ -27,7 +27,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.curator.framework.CuratorFramework; -import java.util.List; +import java.util.Set; import java.util.function.Predicate; @Slf4j @@ -44,7 +44,7 @@ public UnshardedRangerZKHubClient( CuratorFramework curatorFramework, Predicate criteria, ZkNodeDataDeserializer deserializer, - List services + Set services ) { super( namespace, diff --git a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZkServiceDataSource.java b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZkServiceDataSource.java index 224042fe..24be855f 100644 --- a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZkServiceDataSource.java +++ b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZkServiceDataSource.java @@ -23,8 +23,8 @@ import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.retry.ExponentialBackoffRetry; -import java.util.Collection; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; /** @@ -34,7 +34,7 @@ public class ZkServiceDataSource implements ServiceDataSource { private final String namespace; - private String connectionString; + private final String connectionString; private CuratorFramework curatorFramework; private boolean curatorProvided; @@ -47,7 +47,7 @@ public ZkServiceDataSource(String namespace, } @Override - public Collection services() throws Exception { + public Set services() throws Exception { final List children = curatorFramework.getChildren() .forPath("/"); return children.stream() diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/CustomShardSelectorTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/CustomShardSelectorTest.java index ccbbf1bb..2a6a04f4 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/CustomShardSelectorTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/CustomShardSelectorTest.java @@ -151,12 +151,12 @@ public void testBasicDiscovery() { serviceFinder.start(); { val node = serviceFinder.get(TestShardInfo.getCriteria(1, 10)); - Assert.assertNull(node); + Assert.assertFalse(node.isPresent()); } { val node = serviceFinder.get(TestShardInfo.getCriteria(1, 2)); - Assert.assertNotNull(node); - Assert.assertEquals(new TestShardInfo(1, 2), node.getNodeData()); + Assert.assertTrue(node.isPresent()); + Assert.assertEquals(new TestShardInfo(1, 2), node.get().getNodeData()); } serviceFinder.stop(); } diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceNoProviderTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceNoProviderTest.java index 7fd40580..cc5e25e1 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceNoProviderTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceNoProviderTest.java @@ -70,8 +70,8 @@ public void testBasicDiscovery() { }) .build(); serviceFinder.start(); - ServiceNode node = serviceFinder.get(RangerTestUtils.getCriteria(1)); - Assert.assertNull(node); + val node = serviceFinder.get(RangerTestUtils.getCriteria(1)); + Assert.assertFalse(node.isPresent()); serviceFinder.stop(); } @@ -96,8 +96,8 @@ public void testBasicDiscoveryRR() { }) .build(); serviceFinder.start(); - ServiceNode node = serviceFinder.get(RangerTestUtils.getCriteria(1)); - Assert.assertNull(node); + val node = serviceFinder.get(RangerTestUtils.getCriteria(1)); + Assert.assertFalse(node.isPresent()); serviceFinder.stop(); } diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderExtCuratorTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderExtCuratorTest.java index 6ec03f7a..eca50189 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderExtCuratorTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderExtCuratorTest.java @@ -76,7 +76,7 @@ public void stopTestCluster() throws Exception { @Test public void testBasicDiscovery() { - SimpleShardedServiceFinder serviceFinder = ServiceFinderBuilders.shardedFinderBuilder() + val serviceFinder = ServiceFinderBuilders.shardedFinderBuilder() .withCuratorFramework(curatorFramework) .withNamespace("test") .withServiceName("test-service") @@ -93,29 +93,28 @@ public void testBasicDiscovery() { .build(); serviceFinder.start(); { - ServiceNode node = serviceFinder.get(RangerTestUtils.getCriteria(1)); - Assert.assertNotNull(node); - Assert.assertEquals(1, node.getNodeData().getShardId()); + val node = serviceFinder.get(RangerTestUtils.getCriteria(1)); + Assert.assertTrue(node.isPresent()); + Assert.assertEquals(1, node.get().getNodeData().getShardId()); } { - ServiceNode node = serviceFinder.get(RangerTestUtils.getCriteria(1)); - Assert.assertNotNull(node); - Assert.assertEquals(1, node.getNodeData().getShardId()); + val node = serviceFinder.get(RangerTestUtils.getCriteria(1)); + Assert.assertTrue(node.isPresent()); + Assert.assertEquals(1, node.get().getNodeData().getShardId()); } long startTime = System.currentTimeMillis(); for(long i = 0; i <1000000; i++) { - ServiceNode node = serviceFinder.get(RangerTestUtils.getCriteria(2)); - Assert.assertNotNull(node); - Assert.assertEquals(2, node.getNodeData().getShardId()); + val node = serviceFinder.get(RangerTestUtils.getCriteria(2)); + Assert.assertTrue(node.isPresent()); + Assert.assertEquals(2, node.get().getNodeData().getShardId()); } log.info("PERF::RandomSelector::Took (ms):" + (System.currentTimeMillis() - startTime)); { - ServiceNode node = serviceFinder.get(RangerTestUtils.getCriteria(99)); - Assert.assertNull(node); + val node = serviceFinder.get(RangerTestUtils.getCriteria(99)); + Assert.assertFalse(node.isPresent()); } serviceFinder.stop(); - //while (true); } private void registerService(String host, int port, int shardId) { diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderHealthcheckTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderHealthcheckTest.java index 31625988..485270c5 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderHealthcheckTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderHealthcheckTest.java @@ -80,13 +80,13 @@ public void testBasicDiscovery() { .withNodeRefreshIntervalMs(1000) .build(); serviceFinder.start(); - ServiceNode node = serviceFinder.get(RangerTestUtils.getCriteria(1)); - Assert.assertNotNull(node); - Assert.assertEquals("localhost-1", node.getHost()); - TestServiceProvider testServiceProvider = serviceProviders.get(node.getHost()); + val node = serviceFinder.get(RangerTestUtils.getCriteria(1)); + Assert.assertTrue(node.isPresent()); + Assert.assertEquals("localhost-1", node.get().getHost()); + TestServiceProvider testServiceProvider = serviceProviders.get(node.get().getHost()); testServiceProvider.oor(); TestUtils.sleepForSeconds(6); - Assert.assertNull(serviceFinder.get(RangerTestUtils.getCriteria(1))); + Assert.assertFalse(serviceFinder.get(RangerTestUtils.getCriteria(1)).isPresent()); serviceFinder.stop(); } diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderTest.java index 9f5c8323..c1113917 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderTest.java @@ -39,6 +39,7 @@ import java.io.IOException; import java.util.List; +import java.util.stream.LongStream; @Slf4j public class ServiceProviderTest { @@ -85,31 +86,29 @@ public void testBasicDiscovery() { serviceFinder.start(); { val node = serviceFinder.get(RangerTestUtils.getCriteria(1)); - Assert.assertNotNull(node); - Assert.assertEquals(1, node.getNodeData().getShardId()); + Assert.assertTrue(node.isPresent()); + Assert.assertEquals(1, node.get().getNodeData().getShardId()); } { val node = serviceFinder.get(RangerTestUtils.getCriteria(1)); - Assert.assertNotNull(node); - Assert.assertEquals(1, node.getNodeData().getShardId()); + Assert.assertTrue(node.isPresent()); + Assert.assertEquals(1, node.get().getNodeData().getShardId()); } long startTime = System.currentTimeMillis(); - for (long i = 0; i < 1000000; i++) { - val node = serviceFinder.get(RangerTestUtils.getCriteria(2)); - Assert.assertNotNull(node); - Assert.assertEquals(2, node.getNodeData().getShardId()); - } + LongStream.range(0, 1000000).mapToObj(i -> serviceFinder.get(RangerTestUtils.getCriteria(2))).forEach(node -> { + Assert.assertTrue(node.isPresent()); + Assert.assertEquals(2, node.get().getNodeData().getShardId()); + }); log.info("PERF::RandomSelector::Took (ms):" + (System.currentTimeMillis() - startTime)); { val node = serviceFinder.get(RangerTestUtils.getCriteria(99)); - Assert.assertNull(node); + Assert.assertFalse(node.isPresent()); } serviceFinder.stop(); - //while (true); } @Test - public void testBasicDiscoveryRR() throws Exception { + public void testBasicDiscoveryRR() { val serviceFinder = ServiceFinderBuilders.shardedFinderBuilder() .withConnectionString(testingCluster.getConnectString()) @@ -130,24 +129,23 @@ public void testBasicDiscoveryRR() throws Exception { serviceFinder.start(); { val node = serviceFinder.get(RangerTestUtils.getCriteria(1)); - Assert.assertNotNull(node); - Assert.assertEquals(1, node.getNodeData().getShardId()); + Assert.assertTrue(node.isPresent()); + Assert.assertEquals(1, node.get().getNodeData().getShardId()); } { val node = serviceFinder.get(RangerTestUtils.getCriteria(1)); - Assert.assertNotNull(node); - Assert.assertEquals(1, node.getNodeData().getShardId()); + Assert.assertTrue(node.isPresent()); + Assert.assertEquals(1, node.get().getNodeData().getShardId()); } long startTime = System.currentTimeMillis(); - for (long i = 0; i < 1000000; i++) { - ServiceNode node = serviceFinder.get(RangerTestUtils.getCriteria(2)); - Assert.assertNotNull(node); - Assert.assertEquals(2, node.getNodeData().getShardId()); - } + LongStream.range(0, 1000000).mapToObj(i -> serviceFinder.get(RangerTestUtils.getCriteria(2))).forEach(node -> { + Assert.assertTrue(node.isPresent()); + Assert.assertEquals(2, node.get().getNodeData().getShardId()); + }); log.info("PERF::RoundRobinSelector::Took (ms):" + (System.currentTimeMillis() - startTime)); { val node = serviceFinder.get(RangerTestUtils.getCriteria(99)); - Assert.assertNull(node); + Assert.assertFalse(node.isPresent()); } serviceFinder.stop(); //while (true); @@ -173,7 +171,7 @@ public void testVisibility() { }) .build(); serviceFinder.start(); - List> all = serviceFinder.getAll(RangerTestUtils.getCriteria(1)); + val all = serviceFinder.getAll(RangerTestUtils.getCriteria(1)); log.info("Testing ServiceFinder.getAll()"); all.stream().map(serviceNode -> "node = " + serviceNode.getHost() + ":" + serviceNode.getPort() + " " + serviceNode.getHealthcheckStatus() + " " + serviceNode .getLastUpdatedTimeStamp()).forEach(log::info); diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/SimpleServiceProviderTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/SimpleServiceProviderTest.java index 934ee8b9..00766c8b 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/SimpleServiceProviderTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/SimpleServiceProviderTest.java @@ -33,6 +33,7 @@ import org.junit.Test; import java.io.IOException; +import java.util.stream.LongStream; public class SimpleServiceProviderTest { @@ -90,20 +91,17 @@ public void testBasicDiscovery() { serviceFinder.start(); { val node = serviceFinder.get(null); - Assert.assertNotNull(node); - System.out.println(node.getHost()); + Assert.assertTrue(node.isPresent()); + System.out.println(node.get().getHost()); } Multiset frequency = HashMultiset.create(); long startTime = System.currentTimeMillis(); - for(long i = 0; i <1000000; i++) - { - val node = serviceFinder.get(null); - Assert.assertNotNull(node); - frequency.add(node.getHost()); - } + LongStream.range(0, 1000000).mapToObj(i -> serviceFinder.get(null)).forEach(node -> { + Assert.assertTrue(node.isPresent()); + frequency.add(node.get().getHost()); + }); System.out.println("1 Million lookups and freq counting took (ms):" + (System.currentTimeMillis() -startTime)); System.out.println("Frequency: " + frequency); - //while (true); } private void registerService(String host, int port, int shardId) { diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/servicehub/ServiceHubTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/servicehub/ServiceHubTest.java index e0d3a745..c94f0fae 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/servicehub/ServiceHubTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/servicehub/ServiceHubTest.java @@ -44,6 +44,7 @@ import java.io.IOException; import java.util.Collections; import java.util.Date; +import java.util.function.Predicate; /** * @@ -120,9 +121,8 @@ public void testHub() { TestUtils.sleepForSeconds(3); val node = hub.finder(RangerTestUtils.getService(NAMESPACE, "s1")) - .map(finder -> finder.get(nodeData -> nodeData.getShardId() == 1)) - .orElse(null); - Assert.assertNotNull(node); + .flatMap(finder -> finder.get(nodeData -> nodeData.getShardId() == 1)); + Assert.assertTrue(node.isPresent()); hub.stop(); provider1.stop(); } diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/serviceprovider/BaseServiceProviderBuilderTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/serviceprovider/BaseServiceProviderBuilderTest.java index 06bb8f49..50bce5a5 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/serviceprovider/BaseServiceProviderBuilderTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/serviceprovider/BaseServiceProviderBuilderTest.java @@ -73,6 +73,7 @@ public void testbuilder() { .withPort(port) .withHealthUpdateIntervalMs(1000) .build(); + serviceProvider.start(); } catch (Exception e) { exception = e; } From f4d244cbdb84bda304182a32d24004a12fe95357 Mon Sep 17 00:00:00 2001 From: Koushik R Date: Mon, 20 Dec 2021 15:12:06 +0530 Subject: [PATCH 28/69] a) Renamed getServices to getRegisteredServices() b) Lombok usage in NodeDataSource --- .../flipkart/ranger/client/AbstractRangerHubClient.java | 7 ++----- .../java/com/flipkart/ranger/client/RangerHubClient.java | 3 ++- .../ranger/zk/server/bundle/resources/RangerResource.java | 5 +++-- ranger-zk-server/config/local.yml | 1 - .../ranger/zookeeper/servicefinder/ZkNodeDataSource.java | 5 +++-- .../ranger/zookeeper/serviceprovider/ZkNodeDataSink.java | 5 +++-- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/ranger-client/src/main/java/com/flipkart/ranger/client/AbstractRangerHubClient.java b/ranger-client/src/main/java/com/flipkart/ranger/client/AbstractRangerHubClient.java index 266ef666..9b93a067 100644 --- a/ranger-client/src/main/java/com/flipkart/ranger/client/AbstractRangerHubClient.java +++ b/ranger-client/src/main/java/com/flipkart/ranger/client/AbstractRangerHubClient.java @@ -28,10 +28,7 @@ import lombok.extern.slf4j.Slf4j; import lombok.val; -import java.util.Collections; -import java.util.List; -import java.util.Optional; -import java.util.Set; +import java.util.*; import java.util.function.Predicate; @Slf4j @@ -107,7 +104,7 @@ public List> getAllNodes( return optionalFinder.map(trServiceFinder -> trServiceFinder.getAll(criteria)).orElse(Collections.emptyList()); } - public Set getServices() { + public Collection getRegisteredServices() { try{ return this.getHub().getServiceDataSource().services(); }catch (Exception e){ diff --git a/ranger-client/src/main/java/com/flipkart/ranger/client/RangerHubClient.java b/ranger-client/src/main/java/com/flipkart/ranger/client/RangerHubClient.java index 933d8517..f0a58803 100644 --- a/ranger-client/src/main/java/com/flipkart/ranger/client/RangerHubClient.java +++ b/ranger-client/src/main/java/com/flipkart/ranger/client/RangerHubClient.java @@ -18,6 +18,7 @@ import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.core.model.ServiceNode; +import java.util.Collection; import java.util.List; import java.util.Optional; import java.util.Set; @@ -28,7 +29,7 @@ public interface RangerHubClient { void stop(); - Set getServices(); + Collection getRegisteredServices(); Optional> getNode(final Service service); diff --git a/ranger-server-bundle/src/main/java/com/flipkart/ranger/zk/server/bundle/resources/RangerResource.java b/ranger-server-bundle/src/main/java/com/flipkart/ranger/zk/server/bundle/resources/RangerResource.java index 76ff654f..c0f21373 100644 --- a/ranger-server-bundle/src/main/java/com/flipkart/ranger/zk/server/bundle/resources/RangerResource.java +++ b/ranger-server-bundle/src/main/java/com/flipkart/ranger/zk/server/bundle/resources/RangerResource.java @@ -28,6 +28,7 @@ import javax.validation.constraints.NotNull; import javax.ws.rs.*; import javax.ws.rs.core.MediaType; +import java.util.Collection; import java.util.List; import java.util.Set; import java.util.stream.Collectors; @@ -53,8 +54,8 @@ public RangerResource(List> rangerHubs){ public GenericResponse> getServices() { return GenericResponse.>builder() .code(SUCCESS) - .data(rangerHubs.stream().map(RangerHubClient::getServices) - .flatMap(Set::stream).collect(Collectors.toSet())) + .data(rangerHubs.stream().map(RangerHubClient::getRegisteredServices) + .flatMap(Collection::stream).collect(Collectors.toSet())) .build(); } diff --git a/ranger-zk-server/config/local.yml b/ranger-zk-server/config/local.yml index 837f06d7..b33f9e19 100644 --- a/ranger-zk-server/config/local.yml +++ b/ranger-zk-server/config/local.yml @@ -6,7 +6,6 @@ rangerConfiguration: zookeeper: localhost:2181 disablePushUpdaters: true - server: maxThreads: 128 minThreads: 128 diff --git a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinder/ZkNodeDataSource.java b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinder/ZkNodeDataSource.java index 52c6b95e..f6af1679 100644 --- a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinder/ZkNodeDataSource.java +++ b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinder/ZkNodeDataSource.java @@ -25,6 +25,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import lombok.extern.slf4j.Slf4j; +import lombok.val; import org.apache.curator.framework.CuratorFramework; import org.apache.zookeeper.KeeperException; @@ -76,10 +77,10 @@ private List> checkForUpdateOnZookeeper(D deserializer) { log.debug("Found {} nodes for [{}]", children.size(), serviceName); for (String child : children) { byte[] data = readChild(parentPath, child).orElse(null); - if (data == null) { + if (data == null || data.length <= 0) { continue; } - final ServiceNode node = deserializer.deserialize(data); + val node = deserializer.deserialize(data); if(FinderUtils.isValidNode(service, healthcheckZombieCheckThresholdTime, node)) { nodes.add(node); } diff --git a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/serviceprovider/ZkNodeDataSink.java b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/serviceprovider/ZkNodeDataSink.java index 3792e731..7aeb99e8 100644 --- a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/serviceprovider/ZkNodeDataSink.java +++ b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/serviceprovider/ZkNodeDataSink.java @@ -24,6 +24,7 @@ import com.flipkart.ranger.zookeeper.util.PathBuilder; import com.google.common.base.Preconditions; import lombok.extern.slf4j.Slf4j; +import lombok.val; import org.apache.curator.framework.CuratorFramework; import org.apache.zookeeper.CreateMode; import org.apache.zookeeper.KeeperException; @@ -66,7 +67,7 @@ public void updateState(S serializer, ServiceNode serviceNode) { private synchronized void createPath( ServiceNode serviceNode, S serializer) { - final String instancePath = PathBuilder.instancePath(service, serviceNode); + val instancePath = PathBuilder.instancePath(service, serviceNode); try { if (null == curatorFramework.checkExists().forPath(instancePath)) { curatorFramework.create() @@ -80,7 +81,7 @@ private synchronized void createPath( log.warn("Node already exists.. Race condition?", e); } catch (Exception e) { - final String message = String.format( + val message = String.format( "Could not create node for %s after 60 retries (1 min). " + "This service will not be discoverable. Retry after some time.", service.getServiceName()); log.error(message, e); From 0ceafceb77d1718fbb37b2d55ab2c513eb3dd63a Mon Sep 17 00:00:00 2001 From: Koushik R Date: Mon, 20 Dec 2021 15:21:04 +0530 Subject: [PATCH 29/69] Added a default value of emptySet to services in client inits --- .../ranger/client/http/ShardedRangerHttpHubClient.java | 3 ++- .../ranger/client/http/UnshardedRangerHttpHubClient.java | 3 ++- .../flipkart/ranger/client/zk/AbstractRangerZKHubClient.java | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/ShardedRangerHttpHubClient.java b/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/ShardedRangerHttpHubClient.java index 7620b6e2..cd60dd43 100644 --- a/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/ShardedRangerHttpHubClient.java +++ b/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/ShardedRangerHttpHubClient.java @@ -33,6 +33,7 @@ import lombok.Builder; import lombok.extern.slf4j.Slf4j; +import java.util.Collections; import java.util.Set; import java.util.function.Predicate; @@ -55,7 +56,7 @@ public ShardedRangerHttpHubClient( ) { super(namespace, mapper, nodeRefreshIntervalMs, criteria, deserializer); this.clientConfig = clientConfig; - this.services = services; + this.services = null != services ? services : Collections.emptySet(); } @Override diff --git a/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/UnshardedRangerHttpHubClient.java b/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/UnshardedRangerHttpHubClient.java index fad819c5..6245ba86 100644 --- a/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/UnshardedRangerHttpHubClient.java +++ b/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/UnshardedRangerHttpHubClient.java @@ -33,6 +33,7 @@ import lombok.Builder; import lombok.extern.slf4j.Slf4j; +import java.util.Collections; import java.util.Set; import java.util.function.Predicate; @@ -55,7 +56,7 @@ public UnshardedRangerHttpHubClient( ) { super(namespace, mapper, nodeRefreshIntervalMs, criteria, deserializer); this.clientConfig = clientConfig; - this.services = services; + this.services = null != services ? services : Collections.emptySet(); } @Override diff --git a/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/AbstractRangerZKHubClient.java b/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/AbstractRangerZKHubClient.java index c9e1fb23..c30fc0e6 100644 --- a/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/AbstractRangerZKHubClient.java +++ b/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/AbstractRangerZKHubClient.java @@ -29,6 +29,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.curator.framework.CuratorFramework; +import java.util.Collections; import java.util.Set; import java.util.function.Predicate; @@ -60,7 +61,7 @@ protected AbstractRangerZKHubClient( super(namespace, mapper, nodeRefreshIntervalMs, criteria, deserializer); this.disablePushUpdaters = disablePushUpdaters; this.connectionString = connectionString; - this.services = services; + this.services = null != services ? services : Collections.emptySet(); this.curatorFramework = curatorFramework; } From d3982f552d7b9a7ca4e7433b0a82d086300cac3c Mon Sep 17 00:00:00 2001 From: Koushik R Date: Mon, 20 Dec 2021 15:23:46 +0530 Subject: [PATCH 30/69] Added a null check to assertion in tests --- .../ranger/client/http/ShardedRangerHttpClientTest.java | 6 +++--- .../ranger/client/http/SimpleRangerHttpClientTest.java | 6 +++--- .../ranger/client/http/UnshardedRangerHttpClientTest.java | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/ShardedRangerHttpClientTest.java b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/ShardedRangerHttpClientTest.java index f09baa09..1da4c0d0 100644 --- a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/ShardedRangerHttpClientTest.java +++ b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/ShardedRangerHttpClientTest.java @@ -42,13 +42,13 @@ public void testShardedHttpHubClient(){ val service = RangerTestUtils.getService("test-n", "test-s"); Optional> node = client.getNode(service); - Assert.assertTrue(node.isPresent()); + Assert.assertTrue(null != node && node.isPresent()); node = client.getNode(service, nodeData -> nodeData.getShardId() == 1); - Assert.assertTrue(node.isPresent()); + Assert.assertTrue(null != node && node.isPresent()); node = client.getNode(service, nodeData -> nodeData.getShardId() == 2); - Assert.assertFalse(node.isPresent()); + Assert.assertTrue(null != node && node.isPresent()); client.stop(); } diff --git a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/SimpleRangerHttpClientTest.java b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/SimpleRangerHttpClientTest.java index 92be677f..bfc65455 100644 --- a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/SimpleRangerHttpClientTest.java +++ b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/SimpleRangerHttpClientTest.java @@ -41,13 +41,13 @@ public void testSimpleHttpRangerClient(){ TestUtils.sleepForSeconds(6); Optional> node = client.getNode(); - Assert.assertTrue(node.isPresent()); + Assert.assertTrue(null != node && node.isPresent()); node = client.getNode(nodeData -> nodeData.getShardId() == 1); - Assert.assertTrue(node.isPresent()); + Assert.assertTrue(null != node && node.isPresent()); node = client.getNode(nodeData -> nodeData.getShardId() == 2); - Assert.assertFalse(node.isPresent()); + Assert.assertTrue(null != node && node.isPresent()); client.stop(); } diff --git a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/UnshardedRangerHttpClientTest.java b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/UnshardedRangerHttpClientTest.java index c07d4224..d8795947 100644 --- a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/UnshardedRangerHttpClientTest.java +++ b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/UnshardedRangerHttpClientTest.java @@ -42,13 +42,13 @@ public void testUnshardedRangerHubClient(){ val service = RangerTestUtils.getService("test-n", "test-s"); Optional> node = client.getNode(service); - Assert.assertTrue(node.isPresent()); + Assert.assertTrue(null != node && node.isPresent()); node = client.getNode(service, nodeData -> nodeData.getShardId() == 1); - Assert.assertTrue(node.isPresent()); + Assert.assertTrue(null != node && node.isPresent()); node = client.getNode(service, nodeData -> nodeData.getShardId() == 2); - Assert.assertFalse(node.isPresent()); + Assert.assertTrue(null != node && node.isPresent()); client.stop(); } From 987c9efd9641faaf91c3fe804c716225ee758a53 Mon Sep 17 00:00:00 2001 From: Koushik R Date: Mon, 20 Dec 2021 15:51:46 +0530 Subject: [PATCH 31/69] Changed the optional usage to orElse(null) across all the tests --- .../stubs/TestServiceFinderFactory.java | 7 +----- .../TestSimpleUnshardedServiceFinder.java | 7 +++--- .../http/ShardedRangerHttpClientTest.java | 12 +++++----- .../http/SimpleRangerHttpClientTest.java | 12 +++++----- .../http/UnshardedRangerHttpClientTest.java | 12 +++++----- .../HttpServiceDataSource.java | 4 ++-- .../server/bundle/RangerServerBundleTest.java | 2 +- .../model/CustomShardSelectorTest.java | 10 ++++----- .../model/ServiceNoProviderTest.java | 8 +++---- .../model/ServiceProviderExtCuratorTest.java | 22 +++++++++---------- .../model/ServiceProviderHealthcheckTest.java | 8 +++---- .../zookeeper/model/ServiceProviderTest.java | 22 +++++++++---------- .../model/SimpleServiceProviderTest.java | 12 +++++----- .../zookeeper/servicehub/ServiceHubTest.java | 4 ++-- 14 files changed, 69 insertions(+), 73 deletions(-) diff --git a/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestServiceFinderFactory.java b/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestServiceFinderFactory.java index 504ab91d..34fdcfb4 100644 --- a/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestServiceFinderFactory.java +++ b/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestServiceFinderFactory.java @@ -30,12 +30,7 @@ public ServiceFinder> build val finder = new TestSimpleUnshardedServiceFinder() .withNamespace(service.getNamespace()) .withServiceName(service.getServiceName()) - .withDeserializer(new Deserializer() { - @Override - public int hashCode() { - return super.hashCode(); - } - }) + .withDeserializer(new Deserializer() {}) .build(); finder.start(); return finder; diff --git a/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestSimpleUnshardedServiceFinder.java b/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestSimpleUnshardedServiceFinder.java index bed6c38e..a4452758 100644 --- a/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestSimpleUnshardedServiceFinder.java +++ b/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestSimpleUnshardedServiceFinder.java @@ -25,6 +25,7 @@ import com.google.common.collect.Lists; import lombok.Builder; +import java.util.Collections; import java.util.List; @Builder @@ -45,9 +46,9 @@ static class TestDataSource implements NodeDataSource> refresh(Deserializer deserializer) { - return Lists.newArrayList( - new ServiceNode<>("localhost", 9200, TestNodeData.builder().shardId(1).build()) - ); + return Collections.singletonList( + new ServiceNode<>("localhost", 9200, TestNodeData.builder().shardId(1).build()) + ); } @Override diff --git a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/ShardedRangerHttpClientTest.java b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/ShardedRangerHttpClientTest.java index 1da4c0d0..867a3060 100644 --- a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/ShardedRangerHttpClientTest.java +++ b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/ShardedRangerHttpClientTest.java @@ -41,14 +41,14 @@ public void testShardedHttpHubClient(){ TestUtils.sleepForSeconds(6); val service = RangerTestUtils.getService("test-n", "test-s"); - Optional> node = client.getNode(service); - Assert.assertTrue(null != node && node.isPresent()); + ServiceNode node = client.getNode(service).orElse(null); + Assert.assertNotNull(node); - node = client.getNode(service, nodeData -> nodeData.getShardId() == 1); - Assert.assertTrue(null != node && node.isPresent()); + node = client.getNode(service, nodeData -> nodeData.getShardId() == 1).orElse(null); + Assert.assertNotNull(node); - node = client.getNode(service, nodeData -> nodeData.getShardId() == 2); - Assert.assertTrue(null != node && node.isPresent()); + node = client.getNode(service, nodeData -> nodeData.getShardId() == 2).orElse(null); + Assert.assertNull(node); client.stop(); } diff --git a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/SimpleRangerHttpClientTest.java b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/SimpleRangerHttpClientTest.java index bfc65455..564bdf82 100644 --- a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/SimpleRangerHttpClientTest.java +++ b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/SimpleRangerHttpClientTest.java @@ -40,14 +40,14 @@ public void testSimpleHttpRangerClient(){ TestUtils.sleepForSeconds(6); - Optional> node = client.getNode(); - Assert.assertTrue(null != node && node.isPresent()); + ServiceNode node = client.getNode().orElse(null); + Assert.assertNotNull(node); - node = client.getNode(nodeData -> nodeData.getShardId() == 1); - Assert.assertTrue(null != node && node.isPresent()); + node = client.getNode(nodeData -> nodeData.getShardId() == 1).orElse(null); + Assert.assertNotNull(node); - node = client.getNode(nodeData -> nodeData.getShardId() == 2); - Assert.assertTrue(null != node && node.isPresent()); + node = client.getNode(nodeData -> nodeData.getShardId() == 2).orElse(null); + Assert.assertNull(node); client.stop(); } diff --git a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/UnshardedRangerHttpClientTest.java b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/UnshardedRangerHttpClientTest.java index d8795947..6896a09b 100644 --- a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/UnshardedRangerHttpClientTest.java +++ b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/UnshardedRangerHttpClientTest.java @@ -41,14 +41,14 @@ public void testUnshardedRangerHubClient(){ TestUtils.sleepForSeconds(6); val service = RangerTestUtils.getService("test-n", "test-s"); - Optional> node = client.getNode(service); - Assert.assertTrue(null != node && node.isPresent()); + ServiceNode node = client.getNode(service).orElse(null); + Assert.assertNotNull(node); - node = client.getNode(service, nodeData -> nodeData.getShardId() == 1); - Assert.assertTrue(null != node && node.isPresent()); + node = client.getNode(service, nodeData -> nodeData.getShardId() == 1).orElse(null); + Assert.assertNotNull(node); - node = client.getNode(service, nodeData -> nodeData.getShardId() == 2); - Assert.assertTrue(null != node && node.isPresent()); + node = client.getNode(service, nodeData -> nodeData.getShardId() == 2).orElse(null); + Assert.assertNull(node); client.stop(); } diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSource.java b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSource.java index 702c8d18..88446d58 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSource.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSource.java @@ -63,12 +63,12 @@ public Set services() { Set services = Collections.emptySet(); try (val response = httpClient.newCall(request).execute()) { if (response.isSuccessful()) { - try (final ResponseBody body = response.body()) { + try (val body = response.body()) { if (null == body) { log.warn("HTTP call to {} returned empty body", httpUrl); } else { - final byte[] bytes = body.bytes(); + val bytes = body.bytes(); val serviceDataSourceResponse = mapper.readValue(bytes, ServiceDataSourceResponse.class); if(serviceDataSourceResponse.isSuccess()){ services = serviceDataSourceResponse.getData(); diff --git a/ranger-server-bundle/src/test/java/com/flipkart/ranger/zk/server/bundle/RangerServerBundleTest.java b/ranger-server-bundle/src/test/java/com/flipkart/ranger/zk/server/bundle/RangerServerBundleTest.java index a61fe421..fdaeb089 100644 --- a/ranger-server-bundle/src/test/java/com/flipkart/ranger/zk/server/bundle/RangerServerBundleTest.java +++ b/ranger-server-bundle/src/test/java/com/flipkart/ranger/zk/server/bundle/RangerServerBundleTest.java @@ -76,7 +76,7 @@ protected void preBundle(Configuration configuration) { @Override protected List> withHubs(Configuration configuration) { - return Lists.newArrayList(RangerHubTestUtils.getTestHub()); + return Collections.singletonList(RangerHubTestUtils.getTestHub()); } @Override diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/CustomShardSelectorTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/CustomShardSelectorTest.java index 2a6a04f4..b836191c 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/CustomShardSelectorTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/CustomShardSelectorTest.java @@ -150,13 +150,13 @@ public void testBasicDiscovery() { .build(); serviceFinder.start(); { - val node = serviceFinder.get(TestShardInfo.getCriteria(1, 10)); - Assert.assertFalse(node.isPresent()); + val node = serviceFinder.get(TestShardInfo.getCriteria(1, 10)).orElse(null); + Assert.assertNotNull(node); } { - val node = serviceFinder.get(TestShardInfo.getCriteria(1, 2)); - Assert.assertTrue(node.isPresent()); - Assert.assertEquals(new TestShardInfo(1, 2), node.get().getNodeData()); + val node = serviceFinder.get(TestShardInfo.getCriteria(1, 2)).orElse(null); + Assert.assertNotNull(node); + Assert.assertEquals(new TestShardInfo(1, 2), node.getNodeData()); } serviceFinder.stop(); } diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceNoProviderTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceNoProviderTest.java index cc5e25e1..7e71c6f9 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceNoProviderTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceNoProviderTest.java @@ -70,8 +70,8 @@ public void testBasicDiscovery() { }) .build(); serviceFinder.start(); - val node = serviceFinder.get(RangerTestUtils.getCriteria(1)); - Assert.assertFalse(node.isPresent()); + val node = serviceFinder.get(RangerTestUtils.getCriteria(1)).orElse(null); + Assert.assertNull(node); serviceFinder.stop(); } @@ -96,8 +96,8 @@ public void testBasicDiscoveryRR() { }) .build(); serviceFinder.start(); - val node = serviceFinder.get(RangerTestUtils.getCriteria(1)); - Assert.assertFalse(node.isPresent()); + val node = serviceFinder.get(RangerTestUtils.getCriteria(1)).orElse(null); + Assert.assertNull(node); serviceFinder.stop(); } diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderExtCuratorTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderExtCuratorTest.java index eca50189..f8e805c9 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderExtCuratorTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderExtCuratorTest.java @@ -93,26 +93,26 @@ public void testBasicDiscovery() { .build(); serviceFinder.start(); { - val node = serviceFinder.get(RangerTestUtils.getCriteria(1)); - Assert.assertTrue(node.isPresent()); - Assert.assertEquals(1, node.get().getNodeData().getShardId()); + val node = serviceFinder.get(RangerTestUtils.getCriteria(1)).orElse(null); + Assert.assertNotNull(node); + Assert.assertEquals(1, node.getNodeData().getShardId()); } { - val node = serviceFinder.get(RangerTestUtils.getCriteria(1)); - Assert.assertTrue(node.isPresent()); - Assert.assertEquals(1, node.get().getNodeData().getShardId()); + val node = serviceFinder.get(RangerTestUtils.getCriteria(1)).orElse(null); + Assert.assertNotNull(node); + Assert.assertEquals(1, node.getNodeData().getShardId()); } long startTime = System.currentTimeMillis(); for(long i = 0; i <1000000; i++) { - val node = serviceFinder.get(RangerTestUtils.getCriteria(2)); - Assert.assertTrue(node.isPresent()); - Assert.assertEquals(2, node.get().getNodeData().getShardId()); + val node = serviceFinder.get(RangerTestUtils.getCriteria(2)).orElse(null); + Assert.assertNotNull(node); + Assert.assertEquals(2, node.getNodeData().getShardId()); } log.info("PERF::RandomSelector::Took (ms):" + (System.currentTimeMillis() - startTime)); { - val node = serviceFinder.get(RangerTestUtils.getCriteria(99)); - Assert.assertFalse(node.isPresent()); + val node = serviceFinder.get(RangerTestUtils.getCriteria(99)).orElse(null); + Assert.assertNull(node); } serviceFinder.stop(); } diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderHealthcheckTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderHealthcheckTest.java index 485270c5..611c0dab 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderHealthcheckTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderHealthcheckTest.java @@ -80,10 +80,10 @@ public void testBasicDiscovery() { .withNodeRefreshIntervalMs(1000) .build(); serviceFinder.start(); - val node = serviceFinder.get(RangerTestUtils.getCriteria(1)); - Assert.assertTrue(node.isPresent()); - Assert.assertEquals("localhost-1", node.get().getHost()); - TestServiceProvider testServiceProvider = serviceProviders.get(node.get().getHost()); + val node = serviceFinder.get(RangerTestUtils.getCriteria(1)).orElse(null); + Assert.assertNotNull(node); + Assert.assertEquals("localhost-1", node.getHost()); + TestServiceProvider testServiceProvider = serviceProviders.get(node.getHost()); testServiceProvider.oor(); TestUtils.sleepForSeconds(6); Assert.assertFalse(serviceFinder.get(RangerTestUtils.getCriteria(1)).isPresent()); diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderTest.java index c1113917..6b62e8f6 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderTest.java @@ -85,24 +85,24 @@ public void testBasicDiscovery() { .build(); serviceFinder.start(); { - val node = serviceFinder.get(RangerTestUtils.getCriteria(1)); - Assert.assertTrue(node.isPresent()); - Assert.assertEquals(1, node.get().getNodeData().getShardId()); + val node = serviceFinder.get(RangerTestUtils.getCriteria(1)).orElse(null); + Assert.assertNotNull(node); + Assert.assertEquals(1, node.getNodeData().getShardId()); } { - val node = serviceFinder.get(RangerTestUtils.getCriteria(1)); - Assert.assertTrue(node.isPresent()); - Assert.assertEquals(1, node.get().getNodeData().getShardId()); + val node = serviceFinder.get(RangerTestUtils.getCriteria(1)).orElse(null); + Assert.assertNotNull(node); + Assert.assertEquals(1, node.getNodeData().getShardId()); } long startTime = System.currentTimeMillis(); - LongStream.range(0, 1000000).mapToObj(i -> serviceFinder.get(RangerTestUtils.getCriteria(2))).forEach(node -> { - Assert.assertTrue(node.isPresent()); - Assert.assertEquals(2, node.get().getNodeData().getShardId()); + LongStream.range(0, 1000000).mapToObj(i -> serviceFinder.get(RangerTestUtils.getCriteria(2)).orElse(null)).forEach(node -> { + Assert.assertNotNull(node); + Assert.assertEquals(2, node.getNodeData().getShardId()); }); log.info("PERF::RandomSelector::Took (ms):" + (System.currentTimeMillis() - startTime)); { - val node = serviceFinder.get(RangerTestUtils.getCriteria(99)); - Assert.assertFalse(node.isPresent()); + val node = serviceFinder.get(RangerTestUtils.getCriteria(99)).orElse(null); + Assert.assertNull(node); } serviceFinder.stop(); } diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/SimpleServiceProviderTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/SimpleServiceProviderTest.java index 00766c8b..fe1a5ac7 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/SimpleServiceProviderTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/SimpleServiceProviderTest.java @@ -90,15 +90,15 @@ public void testBasicDiscovery() { .build(); serviceFinder.start(); { - val node = serviceFinder.get(null); - Assert.assertTrue(node.isPresent()); - System.out.println(node.get().getHost()); + val node = serviceFinder.get(null).orElse(null); + Assert.assertNotNull(node); + System.out.println(node.getHost()); } Multiset frequency = HashMultiset.create(); long startTime = System.currentTimeMillis(); - LongStream.range(0, 1000000).mapToObj(i -> serviceFinder.get(null)).forEach(node -> { - Assert.assertTrue(node.isPresent()); - frequency.add(node.get().getHost()); + LongStream.range(0, 1000000).mapToObj(i -> serviceFinder.get(null).orElse(null)).forEach(node -> { + Assert.assertNotNull(node); + frequency.add(node.getHost()); }); System.out.println("1 Million lookups and freq counting took (ms):" + (System.currentTimeMillis() -startTime)); System.out.println("Frequency: " + frequency); diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/servicehub/ServiceHubTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/servicehub/ServiceHubTest.java index c94f0fae..7e7ba496 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/servicehub/ServiceHubTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/servicehub/ServiceHubTest.java @@ -121,8 +121,8 @@ public void testHub() { TestUtils.sleepForSeconds(3); val node = hub.finder(RangerTestUtils.getService(NAMESPACE, "s1")) - .flatMap(finder -> finder.get(nodeData -> nodeData.getShardId() == 1)); - Assert.assertTrue(node.isPresent()); + .flatMap(finder -> finder.get(nodeData -> nodeData.getShardId() == 1)).orElse(null); + Assert.assertNotNull(node); hub.stop(); provider1.stop(); } From bcb4ac87ddc2fe2041fc79ce5cd22cc3f3bf115f Mon Sep 17 00:00:00 2001 From: Koushik R Date: Mon, 20 Dec 2021 15:54:23 +0530 Subject: [PATCH 32/69] Fixed customShardSelectorTest --- .../ranger/zookeeper/model/CustomShardSelectorTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/CustomShardSelectorTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/CustomShardSelectorTest.java index b836191c..f3966dfc 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/CustomShardSelectorTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/CustomShardSelectorTest.java @@ -151,7 +151,7 @@ public void testBasicDiscovery() { serviceFinder.start(); { val node = serviceFinder.get(TestShardInfo.getCriteria(1, 10)).orElse(null); - Assert.assertNotNull(node); + Assert.assertNull(node); } { val node = serviceFinder.get(TestShardInfo.getCriteria(1, 2)).orElse(null); From 9cd3f6bd465a3b7e224bc42a93d1f2540e118e27 Mon Sep 17 00:00:00 2001 From: Koushik R Date: Mon, 20 Dec 2021 16:13:13 +0530 Subject: [PATCH 33/69] scope provided in bundle for dropwizard core --- ranger-server-bundle/pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/ranger-server-bundle/pom.xml b/ranger-server-bundle/pom.xml index b0b9300c..e2ab91d4 100644 --- a/ranger-server-bundle/pom.xml +++ b/ranger-server-bundle/pom.xml @@ -36,6 +36,7 @@ io.dropwizard dropwizard-core ${dropwizard.version} + provided org.mockito From e4f449684ff02374a94cc63ef9f4760d997d2a47 Mon Sep 17 00:00:00 2001 From: Koushik R Date: Mon, 20 Dec 2021 16:17:23 +0530 Subject: [PATCH 34/69] Small fix in the handling of InterruptedException --- .../flipkart/ranger/zk/server/lifecycle/CuratorLifecycle.java | 1 + 1 file changed, 1 insertion(+) diff --git a/ranger-zk-server/src/main/java/com/flipkart/ranger/zk/server/lifecycle/CuratorLifecycle.java b/ranger-zk-server/src/main/java/com/flipkart/ranger/zk/server/lifecycle/CuratorLifecycle.java index af00f988..3b25dbc5 100644 --- a/ranger-zk-server/src/main/java/com/flipkart/ranger/zk/server/lifecycle/CuratorLifecycle.java +++ b/ranger-zk-server/src/main/java/com/flipkart/ranger/zk/server/lifecycle/CuratorLifecycle.java @@ -39,6 +39,7 @@ public void start() { } catch (InterruptedException e) { log.error("Curator block interrupted", e); + Thread.currentThread().interrupt(); } log.info("Started the curator"); } From e9e14aeab9364dc54ff1c5b57b4df5da40a2a9e8 Mon Sep 17 00:00:00 2001 From: Koushik R Date: Mon, 20 Dec 2021 16:19:30 +0530 Subject: [PATCH 35/69] Additional lombok cleanups --- .../ServiceHealthAggregator.java | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/ServiceHealthAggregator.java b/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/ServiceHealthAggregator.java index 9b50cb8e..4f7b4aa8 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/ServiceHealthAggregator.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/ServiceHealthAggregator.java @@ -22,6 +22,7 @@ import com.flipkart.ranger.core.healthservice.monitor.Monitor; import com.google.common.collect.Lists; import lombok.extern.slf4j.Slf4j; +import lombok.val; import java.util.Date; import java.util.List; @@ -147,7 +148,7 @@ public HealthcheckStatus getServiceHealth() { Date currentTime = new Date(); /* check health status of isolated monitors */ - final boolean hasUnhealthyMonitor = isolatedHealthMonitorList.stream() + val hasUnhealthyMonitor = isolatedHealthMonitorList.stream() .filter(isolatedHealthMonitor -> !isolatedHealthMonitor.isDisabled()) .anyMatch(isolatedHealthMonitor -> isIsolatedMonitorHealthy(isolatedHealthMonitor, currentTime)); if (hasUnhealthyMonitor) { @@ -167,7 +168,7 @@ private boolean isIsolatedMonitorHealthy( IsolatedHealthMonitor isolatedHealthMo if (HealthcheckStatus.unhealthy == isolatedHealthMonitor.getHealthStatus()) { return true; } - final boolean hasValidUpdateTime + val hasValidUpdateTime = null != isolatedHealthMonitor.getLastStatusUpdateTime() && hasValidUpdatedTime(isolatedHealthMonitor, currentTime); /* check if the monitor and its last updated time is stale, if so, mark status as unhealthy */ @@ -180,21 +181,13 @@ private boolean isIsolatedMonitorHealthy( IsolatedHealthMonitor isolatedHealthMo } private boolean hasValidUpdatedTime(IsolatedHealthMonitor isolatedHealthMonitor, Date currentTime) { - final long timeDifferenceMillis = currentTime.getTime() - isolatedHealthMonitor.getLastStatusUpdateTime() + val timeDifferenceMillis = currentTime.getTime() - isolatedHealthMonitor.getLastStatusUpdateTime() .getTime(); return timeDifferenceMillis <= isolatedHealthMonitor.getStalenessAllowedInMillis(); } private void processMonitors() { - /* check status of all inline monitors in the same thread */ - for (Monitor healthMonitor : inlineHealthMonitorList) { - if (healthMonitor.isDisabled()) { - continue; - } - final HealthcheckStatus monitorStatus = healthMonitor.monitor(); - if (HealthcheckStatus.unhealthy == monitorStatus) { - healthcheckStatus.set(HealthcheckStatus.unhealthy); - } - } + /* check status of all inline monitors in the same thread */ + inlineHealthMonitorList.stream().filter(healthMonitor -> !healthMonitor.isDisabled()).map(Monitor::monitor).filter(monitorStatus -> HealthcheckStatus.unhealthy == monitorStatus).forEach(monitorStatus -> healthcheckStatus.set(HealthcheckStatus.unhealthy)); } } From b3480579d34ec76764ba929751ccdfe05c9f7aeb Mon Sep 17 00:00:00 2001 From: Koushik R Date: Mon, 20 Dec 2021 16:26:55 +0530 Subject: [PATCH 36/69] Moved to ?: in shardSelectors from the original guardRail if - to maintain consistency with the rest of the codebase --- .../core/finder/serviceregistry/MapBasedServiceRegistry.java | 3 ++- .../ranger/core/finder/shardselector/ListShardSelector.java | 5 +---- .../core/finder/shardselector/MatchingShardSelector.java | 4 ++-- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/serviceregistry/MapBasedServiceRegistry.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/serviceregistry/MapBasedServiceRegistry.java index 8074d8ba..8d5454d6 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/serviceregistry/MapBasedServiceRegistry.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/serviceregistry/MapBasedServiceRegistry.java @@ -19,6 +19,7 @@ import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.model.ServiceRegistry; import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableListMultimap; import com.google.common.collect.ListMultimap; import lombok.val; @@ -43,7 +44,7 @@ public ListMultimap> nodes() { @Override public List> nodeList() { val nodeList = nodes.get(); - return null == nodeList ? Collections.emptyList() : new ArrayList<>(nodeList.values()); + return null == nodeList ? ImmutableList.of() : new ArrayList<>(nodeList.values()); } @Override diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/shardselector/ListShardSelector.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/shardselector/ListShardSelector.java index 1063d406..c7898951 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/shardselector/ListShardSelector.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/shardselector/ListShardSelector.java @@ -27,10 +27,7 @@ public class ListShardSelector implements ShardSelector> nodes(Predicate criteria, ListBasedServiceRegistry serviceRegistry) { - if(null == criteria){ - return serviceRegistry.nodeList(); - } - return serviceRegistry.nodeList() + return null == criteria ? serviceRegistry.nodeList() : serviceRegistry.nodeList() .stream() .filter(node -> criteria.test(node.getNodeData())) .collect(Collectors.toList()); diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/shardselector/MatchingShardSelector.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/shardselector/MatchingShardSelector.java index 360cdb03..099c8687 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/shardselector/MatchingShardSelector.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/shardselector/MatchingShardSelector.java @@ -29,8 +29,8 @@ public class MatchingShardSelector implements ShardSelector> nodes(Predicate criteria, MapBasedServiceRegistry serviceRegistry) { - if(null == criteria) return serviceRegistry.nodeList(); - return serviceRegistry.nodes() + return null == criteria ? serviceRegistry.nodeList() : + serviceRegistry.nodes() .entries() .stream() .filter(e -> criteria.test(e.getKey())) From 0562ef3cc862aa46899d25196452284d31a71397 Mon Sep 17 00:00:00 2001 From: Koushik R Date: Mon, 20 Dec 2021 17:39:56 +0530 Subject: [PATCH 37/69] a) Lombok issues addressed across the codebase b) Formatting changes as well --- .../core/finderhub/ServiceDataSource.java | 3 ++- .../core/finderhub/StaticDataSource.java | 3 ++- .../ranger/core/model/Serializer.java | 3 +-- .../flipkart/ranger/core/model/Service.java | 2 +- .../core/serviceprovider/ServiceProvider.java | 1 - .../core/signals/ExternalTriggeredSignal.java | 2 +- .../core/finder/SimpleShardFinderTest.java | 3 ++- .../finder/UnshardedClusterFinderTest.java | 2 +- .../serviceprovider/ServiceProviderTest.java | 13 +++++++------ .../ranger/core/utils/RegistryTestUtils.java | 19 +++++++++++-------- .../http/HttpServiceFinderBuilders.java | 6 ++---- .../http/HttpServiceProviderBuilders.java | 6 ++---- .../servicefinder/HttpNodeDataSource.java | 16 +++++++++------- .../HttpServiceDataSource.java | 10 +++++----- .../HttpShardedServiceFinderFactory.java | 3 +-- .../HttpUnshardedServiceFinderFactory.java | 1 - .../serviceprovider/HttpNodeDataSink.java | 5 ++++- .../HttpShardedServiceFinderBuilderTest.java | 3 ++- ...HttpShardedServiceProviderBuilderTest.java | 7 +++---- .../ranger/common/server/ShardInfoTest.java | 6 ++++-- .../zookeeper/ServiceFinderBuilders.java | 6 ++---- .../zookeeper/ServiceProviderBuilders.java | 8 +++----- .../ZKUnshardedServiceFinderFactory.java | 1 - .../servicefinderhub/ZkServiceDataSource.java | 16 ++++++++++------ .../ZkServiceProviderBuilder.java | 1 - 25 files changed, 75 insertions(+), 71 deletions(-) diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/ServiceDataSource.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/ServiceDataSource.java index 5a9e0e50..a9ebb51f 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/ServiceDataSource.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/ServiceDataSource.java @@ -17,6 +17,7 @@ import com.flipkart.ranger.core.model.Service; +import java.util.Collection; import java.util.Set; /** @@ -24,7 +25,7 @@ */ public interface ServiceDataSource { - Set services() throws Exception; + Collection services() throws Exception; void start(); void stop(); diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/StaticDataSource.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/StaticDataSource.java index 4977668d..e0de9a01 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/StaticDataSource.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/StaticDataSource.java @@ -19,6 +19,7 @@ import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; +import java.util.Collection; import java.util.Set; /* @@ -32,7 +33,7 @@ public class StaticDataSource implements ServiceDataSource{ private final Set services; @Override - public Set services() throws Exception { + public Collection services() throws Exception { return services; } diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/model/Serializer.java b/ranger-core/src/main/java/com/flipkart/ranger/core/model/Serializer.java index ab37c6ad..cea2dab1 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/model/Serializer.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/model/Serializer.java @@ -13,9 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package com.flipkart.ranger.core.model; public interface Serializer { -// byte[] serialize(final ServiceNode data); + } diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/model/Service.java b/ranger-core/src/main/java/com/flipkart/ranger/core/model/Service.java index 083ac956..2721d2db 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/model/Service.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/model/Service.java @@ -20,7 +20,7 @@ @Data @NoArgsConstructor -@AllArgsConstructor(access = AccessLevel.PUBLIC) +@AllArgsConstructor @Builder public class Service { String namespace; diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/serviceprovider/ServiceProvider.java b/ranger-core/src/main/java/com/flipkart/ranger/core/serviceprovider/ServiceProvider.java index b2f381f0..88bc694a 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/serviceprovider/ServiceProvider.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/serviceprovider/ServiceProvider.java @@ -35,7 +35,6 @@ public class ServiceProvider> { private final Service service; private final ServiceNode serviceNode; private final S serializer; - @Getter private final NodeDataSink dataSink; @Getter private final ExternalTriggeredSignal startSignal = new ExternalTriggeredSignal<>(() -> null, Collections.emptyList()); diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/signals/ExternalTriggeredSignal.java b/ranger-core/src/main/java/com/flipkart/ranger/core/signals/ExternalTriggeredSignal.java index 350a2005..8f822b84 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/signals/ExternalTriggeredSignal.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/signals/ExternalTriggeredSignal.java @@ -30,7 +30,7 @@ public ExternalTriggeredSignal( super(signalDataGenerator, consumers); } - public void trigger() { + public final void trigger() { super.onSignalReceived(); } diff --git a/ranger-core/src/test/java/com/flipkart/ranger/core/finder/SimpleShardFinderTest.java b/ranger-core/src/test/java/com/flipkart/ranger/core/finder/SimpleShardFinderTest.java index 83ba898b..f93cd702 100644 --- a/ranger-core/src/test/java/com/flipkart/ranger/core/finder/SimpleShardFinderTest.java +++ b/ranger-core/src/test/java/com/flipkart/ranger/core/finder/SimpleShardFinderTest.java @@ -27,6 +27,7 @@ import org.junit.Assert; import org.junit.Test; +import java.util.Collections; import java.util.List; import java.util.function.Predicate; @@ -36,7 +37,7 @@ static class TestSimpleShardSelector implements ShardSelector> nodes(Predicate criteria, MapBasedServiceRegistry serviceRegistry) { - return Lists.newArrayList(); + return Collections.emptyList(); } } diff --git a/ranger-core/src/test/java/com/flipkart/ranger/core/finder/UnshardedClusterFinderTest.java b/ranger-core/src/test/java/com/flipkart/ranger/core/finder/UnshardedClusterFinderTest.java index 2bfb245a..64daf491 100644 --- a/ranger-core/src/test/java/com/flipkart/ranger/core/finder/UnshardedClusterFinderTest.java +++ b/ranger-core/src/test/java/com/flipkart/ranger/core/finder/UnshardedClusterFinderTest.java @@ -33,7 +33,7 @@ static class TestUnshardedNodeSelector implements ServiceNodeSelector select(List> serviceNodes) { - return serviceNodes.get(0); + return serviceNodes.isEmpty() ? null : serviceNodes.get(0); } } @Test diff --git a/ranger-core/src/test/java/com/flipkart/ranger/core/serviceprovider/ServiceProviderTest.java b/ranger-core/src/test/java/com/flipkart/ranger/core/serviceprovider/ServiceProviderTest.java index 38503fd2..d0243d29 100644 --- a/ranger-core/src/test/java/com/flipkart/ranger/core/serviceprovider/ServiceProviderTest.java +++ b/ranger-core/src/test/java/com/flipkart/ranger/core/serviceprovider/ServiceProviderTest.java @@ -52,14 +52,14 @@ public byte[] serialize(ServiceNode node) { } } - static class TestNodeDataSink> implements NodeDataSink{ + static class TestNodeDataSink> implements NodeDataSink{ public TestNodeDataSink(){ } @Override public void updateState(S serializer, ServiceNode serviceNode) { - testNodeData = (TestNodeData) serviceNode.getNodeData(); + testNodeData = serviceNode.getNodeData(); } @Override @@ -83,7 +83,8 @@ public boolean isActive() { } } - public class TestServiceProviderBuilder extends BaseServiceProviderBuilder, TestSerializer> { + public static class TestServiceProviderBuilder extends + BaseServiceProviderBuilder, TestSerializer> { @Override public ServiceProvider> build() { @@ -98,7 +99,7 @@ protected NodeDataSink> dataSink(Service service) { @Test(expected = NullPointerException.class) public void testInvalidServiceProvider(){ - new TestServiceProviderBuilder() + new TestServiceProviderBuilder<>() .withServiceName("test-service") .withNamespace("test") .withHostname("localhost-1") @@ -108,7 +109,7 @@ public void testInvalidServiceProvider(){ @Test(expected = IllegalArgumentException.class) public void testInvalidServiceProviderNoHealthCheck(){ - new TestServiceProviderBuilder() + new TestServiceProviderBuilder<>() .withServiceName("test-service") .withNamespace("test") .withHostname("localhost-1") @@ -119,7 +120,7 @@ public void testInvalidServiceProviderNoHealthCheck(){ @Test public void testBuildServiceProvider(){ - val testProvider = new TestServiceProviderBuilder() + val testProvider = new TestServiceProviderBuilder<>() .withServiceName("test-service") .withNamespace("test") .withHostname("localhost-1") diff --git a/ranger-core/src/test/java/com/flipkart/ranger/core/utils/RegistryTestUtils.java b/ranger-core/src/test/java/com/flipkart/ranger/core/utils/RegistryTestUtils.java index c235b4be..e72590f9 100644 --- a/ranger-core/src/test/java/com/flipkart/ranger/core/utils/RegistryTestUtils.java +++ b/ranger-core/src/test/java/com/flipkart/ranger/core/utils/RegistryTestUtils.java @@ -19,6 +19,7 @@ import com.flipkart.ranger.core.finder.serviceregistry.MapBasedServiceRegistry; import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.units.TestNodeData; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import lombok.experimental.UtilityClass; import lombok.val; @@ -31,10 +32,11 @@ public class RegistryTestUtils { public static MapBasedServiceRegistry getServiceRegistry(){ val service = RangerTestUtils.getService(); val serviceRegistry = new MapBasedServiceRegistry(service); - List> serviceNodes = Lists.newArrayList(); - serviceNodes.add(new ServiceNode<>("localhost-1", 9000, TestNodeData.builder().shardId(1).build())); - serviceNodes.add(new ServiceNode<>("localhost-2", 9001, TestNodeData.builder().shardId(2).build())); - serviceNodes.add(new ServiceNode<>("localhost-3", 9002, TestNodeData.builder().shardId(3).build())); + val serviceNodes = ImmutableList.of( + new ServiceNode<>("localhost-1", 9000, TestNodeData.builder().shardId(1).build()), + new ServiceNode<>("localhost-2", 9001, TestNodeData.builder().shardId(2).build()), + new ServiceNode<>("localhost-3", 9002, TestNodeData.builder().shardId(3).build()) + ); serviceRegistry.updateNodes(serviceNodes); return serviceRegistry; } @@ -42,10 +44,11 @@ public static MapBasedServiceRegistry getServiceRegistry(){ public static ListBasedServiceRegistry getUnshardedRegistry(){ val service = RangerTestUtils.getService(); val serviceRegistry = new ListBasedServiceRegistry(service); - List> serviceNodes = Lists.newArrayList(); - serviceNodes.add(new ServiceNode<>("localhost-1", 9000, TestNodeData.builder().shardId(1).build())); - serviceNodes.add(new ServiceNode<>("localhost-2", 9001, TestNodeData.builder().shardId(2).build())); - serviceNodes.add(new ServiceNode<>("localhost-3", 9002, TestNodeData.builder().shardId(3).build())); + val serviceNodes = ImmutableList.of( + new ServiceNode<>("localhost-1", 9000, TestNodeData.builder().shardId(1).build()), + new ServiceNode<>("localhost-2", 9001, TestNodeData.builder().shardId(2).build()), + new ServiceNode<>("localhost-3", 9002, TestNodeData.builder().shardId(3).build()) + ); serviceRegistry.updateNodes(serviceNodes); return serviceRegistry; } diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/HttpServiceFinderBuilders.java b/ranger-http/src/main/java/com/flipkart/ranger/http/HttpServiceFinderBuilders.java index 85e51aed..f4dedf8b 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/HttpServiceFinderBuilders.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/HttpServiceFinderBuilders.java @@ -17,13 +17,11 @@ import com.flipkart.ranger.http.servicefinder.HttpShardedServiceFinderBuilder; import com.flipkart.ranger.http.servicefinder.HttpUnshardedServiceFinderBuilider; +import lombok.experimental.UtilityClass; +@UtilityClass public final class HttpServiceFinderBuilders { - private void HttpServiceProviderBuilders(){ - throw new InstantiationError("Must not instantiate this class"); - } - public static HttpShardedServiceFinderBuilder httpShardedServiceFinderBuilder(){ return new HttpShardedServiceFinderBuilder<>(); } diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/HttpServiceProviderBuilders.java b/ranger-http/src/main/java/com/flipkart/ranger/http/HttpServiceProviderBuilders.java index 0e0fc6af..9fada7f2 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/HttpServiceProviderBuilders.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/HttpServiceProviderBuilders.java @@ -16,13 +16,11 @@ package com.flipkart.ranger.http; import com.flipkart.ranger.http.serviceprovider.HttpShardedServiceProviderBuilder; +import lombok.experimental.UtilityClass; +@UtilityClass public final class HttpServiceProviderBuilders { - private HttpServiceProviderBuilders() { - throw new InstantiationError("Must not instantiate this class"); - } - public static HttpShardedServiceProviderBuilder httpServiceProviderBuilder() { return new HttpShardedServiceProviderBuilder<>(); } diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpNodeDataSource.java b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpNodeDataSource.java index 28782434..864103ae 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpNodeDataSource.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpNodeDataSource.java @@ -54,6 +54,9 @@ public HttpNodeDataSource( public List> refresh(D deserializer) { Preconditions.checkNotNull(config, "client config has not been set for node data"); Preconditions.checkNotNull(mapper, "mapper has not been set for node data"); + val url = String.format("/ranger/nodes/v1/%s/%s", service.getNamespace(), service.getServiceName()); + + log.debug("Refreshing the node list from url {}", url); val httpUrl = new HttpUrl.Builder() .scheme(config.isSecure() ? "https" @@ -62,28 +65,27 @@ public List> refresh(D deserializer) { .port(config.getPort() == 0 ? defaultPort() : config.getPort()) - .encodedPath(String.format("/ranger/nodes/v1/%s/%s", service.getNamespace(), service.getServiceName())) + .encodedPath(url) .build(); val request = new Request.Builder() .url(httpUrl) .get() .build(); - List> serviceNodes = Collections.emptyList(); try (val response = httpClient.newCall(request).execute()) { if (response.isSuccessful()) { - try (final ResponseBody body = response.body()) { + try (val body = response.body()) { if (null == body) { log.warn("HTTP call to {} returned empty body", httpUrl); } else { - final byte[] bytes = body.bytes(); + val bytes = body.bytes(); val serviceNodesResponse = deserializer.deserialize(bytes); if(serviceNodesResponse.isSuccess()){ - serviceNodes = FinderUtils.filterValidNodes( + return FinderUtils.filterValidNodes( service, serviceNodesResponse.getData(), healthcheckZombieCheckThresholdTime(service)); - }else{ + } else{ log.warn("Http call to {} returned a failure response with error {}", httpUrl, serviceNodesResponse.getError()); } } @@ -94,7 +96,7 @@ public List> refresh(D deserializer) { } catch (IOException e) { log.error("Error getting service data from the http endPoint: ", e); } - return serviceNodes; + return Collections.emptyList(); } @Override diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSource.java b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSource.java index 88446d58..14fe59da 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSource.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSource.java @@ -30,6 +30,7 @@ import okhttp3.ResponseBody; import java.io.IOException; +import java.util.Collection; import java.util.Collections; import java.util.Set; @@ -41,7 +42,7 @@ public HttpServiceDataSource(HttpClientConfig config, ObjectMapper mapper) { } @Override - public Set services() { + public Collection services() { Preconditions.checkNotNull(config, "client config has not been set for node data"); Preconditions.checkNotNull(mapper, "mapper has not been set for node data"); @@ -59,8 +60,7 @@ public Set services() { .url(httpUrl) .get() .build(); - - Set services = Collections.emptySet(); + try (val response = httpClient.newCall(request).execute()) { if (response.isSuccessful()) { try (val body = response.body()) { @@ -71,7 +71,7 @@ public Set services() { val bytes = body.bytes(); val serviceDataSourceResponse = mapper.readValue(bytes, ServiceDataSourceResponse.class); if(serviceDataSourceResponse.isSuccess()){ - services = serviceDataSourceResponse.getData(); + return serviceDataSourceResponse.getData(); }else{ log.warn("Http call to {} returned a failure response with error {}", httpUrl, serviceDataSourceResponse.getError()); } @@ -87,6 +87,6 @@ public Set services() { } log.error("No data returned from server: " + httpUrl); - return services; + return Collections.emptySet(); } } diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpShardedServiceFinderFactory.java b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpShardedServiceFinderFactory.java index 3a03b6bf..a4274d61 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpShardedServiceFinderFactory.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpShardedServiceFinderFactory.java @@ -29,8 +29,7 @@ import lombok.Getter; import lombok.val; -@Getter -public class HttpShardedServiceFinderFactory implements ServiceFinderFactory> { +public class HttpShardedServiceFinderFactory implements ServiceFinderFactory> { private final HttpClientConfig clientConfig; private final ObjectMapper mapper; diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpUnshardedServiceFinderFactory.java b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpUnshardedServiceFinderFactory.java index f785fc60..7874d80a 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpUnshardedServiceFinderFactory.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpUnshardedServiceFinderFactory.java @@ -29,7 +29,6 @@ import lombok.Getter; import lombok.val; -@Getter public class HttpUnshardedServiceFinderFactory implements ServiceFinderFactory> { private final HttpClientConfig clientConfig; diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/serviceprovider/HttpNodeDataSink.java b/ranger-http/src/main/java/com/flipkart/ranger/http/serviceprovider/HttpNodeDataSink.java index 9ce31b41..0d876a87 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/serviceprovider/HttpNodeDataSink.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/serviceprovider/HttpNodeDataSink.java @@ -49,6 +49,9 @@ public void updateState(S serializer, ServiceNode serviceNode) { Preconditions.checkNotNull(config, "client config has not been set for node data"); Preconditions.checkNotNull(mapper, "mapper has not been set for node data"); + val url = String.format("/ranger/nodes/v1/add/%s/%s", service.getNamespace(), service.getServiceName()); + log.debug("Updating state at the url {}", url); + val httpUrl = new HttpUrl.Builder() .scheme(config.isSecure() ? "https" @@ -57,7 +60,7 @@ public void updateState(S serializer, ServiceNode serviceNode) { .port(config.getPort() == 0 ? defaultPort() : config.getPort()) - .encodedPath(String.format("/ranger/nodes/v1/add/%s/%s", service.getNamespace(), service.getServiceName())) + .encodedPath(url) .build(); val requestBody = RequestBody.create(serializer.serialize(serviceNode)); val serviceRegistrationResponse = registerService(httpUrl, requestBody); diff --git a/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java b/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java index 5515b85e..aeb8727b 100644 --- a/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java +++ b/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java @@ -33,6 +33,7 @@ import org.junit.Test; import java.io.IOException; +import java.util.Collections; import static com.github.tomakehurst.wiremock.client.WireMock.*; @@ -63,7 +64,7 @@ public void testFinder() throws Exception { node.setLastUpdatedTimeStamp(System.currentTimeMillis()); val payload = MAPPER.writeValueAsBytes( ServiceNodesResponse.builder() - .data(Lists.newArrayList(node)) + .data(Collections.singletonList(node)) .code(RangerResponseCode.SUCCESS) .build()); server.stubFor(get(urlEqualTo("/ranger/nodes/v1/testns/test")) diff --git a/ranger-http/src/test/java/com/flipkart/ranger/http/serviceprovider/HttpShardedServiceProviderBuilderTest.java b/ranger-http/src/test/java/com/flipkart/ranger/http/serviceprovider/HttpShardedServiceProviderBuilderTest.java index 6969498d..a581a51a 100644 --- a/ranger-http/src/test/java/com/flipkart/ranger/http/serviceprovider/HttpShardedServiceProviderBuilderTest.java +++ b/ranger-http/src/test/java/com/flipkart/ranger/http/serviceprovider/HttpShardedServiceProviderBuilderTest.java @@ -51,8 +51,8 @@ public TestNodeData(@JsonProperty("farmId") String farmId) { @Test public void testProvider() throws Exception { - val nm5NodeData = TestNodeData.builder().farmId("nm5").build(); - val testNode = new ServiceNode<>("localhost-1", 80, nm5NodeData); + val farmNodeData = TestNodeData.builder().farmId("farm1").build(); + val testNode = new ServiceNode<>("localhost-1", 80, farmNodeData); val response = MAPPER.writeValueAsBytes( GenericResponse.builder() .code(RangerResponseCode.SUCCESS) @@ -78,12 +78,11 @@ public void testProvider() throws Exception { .withHealthUpdateIntervalMs(1000) .withObjectMapper(MAPPER) .withClientConfiguration(clientConfig) - .withNodeData(nm5NodeData) + .withNodeData(farmNodeData) .withSerializer(node -> requestBytes) .build(); serviceProvider.start(); Assert.assertNotNull(serviceProvider); - Assert.assertTrue(serviceProvider.getDataSink().isActive()); } } diff --git a/ranger-server-common/src/test/java/com/flipkart/ranger/common/server/ShardInfoTest.java b/ranger-server-common/src/test/java/com/flipkart/ranger/common/server/ShardInfoTest.java index b25e830e..39ae6436 100644 --- a/ranger-server-common/src/test/java/com/flipkart/ranger/common/server/ShardInfoTest.java +++ b/ranger-server-common/src/test/java/com/flipkart/ranger/common/server/ShardInfoTest.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import lombok.SneakyThrows; +import lombok.experimental.UtilityClass; import org.junit.Assert; import org.junit.Test; @@ -11,10 +12,11 @@ import java.util.Arrays; import java.util.stream.Collectors; + public class ShardInfoTest { private static final ObjectMapper mapper = new ObjectMapper(); - public static String getResource(String path) { + private String getResource(String path) { final InputStream data = ShardInfoTest.class.getClassLoader().getResourceAsStream(path); return new BufferedReader( new InputStreamReader(data)) @@ -23,7 +25,7 @@ public static String getResource(String path) { } @SneakyThrows - public static T getResource(String path, Class klass) { + private T getResource(String path, Class klass) { final String data = getResource(path); return mapper.readValue(data, klass); } diff --git a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/ServiceFinderBuilders.java b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/ServiceFinderBuilders.java index 021b1618..6591fd67 100644 --- a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/ServiceFinderBuilders.java +++ b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/ServiceFinderBuilders.java @@ -17,13 +17,11 @@ import com.flipkart.ranger.zookeeper.servicefinder.ZkSimpleShardedServiceFinderBuilder; import com.flipkart.ranger.zookeeper.servicefinder.ZkSimpleUnshardedServiceFinderBuilder; +import lombok.experimental.UtilityClass; +@UtilityClass public class ServiceFinderBuilders { - private ServiceFinderBuilders() { - throw new InstantiationError("Must not instantiate this class"); - } - public static ZkSimpleShardedServiceFinderBuilder shardedFinderBuilder() { return new ZkSimpleShardedServiceFinderBuilder<>(); } diff --git a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/ServiceProviderBuilders.java b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/ServiceProviderBuilders.java index e83abfbd..3db2250c 100644 --- a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/ServiceProviderBuilders.java +++ b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/ServiceProviderBuilders.java @@ -17,13 +17,11 @@ package com.flipkart.ranger.zookeeper; import com.flipkart.ranger.zookeeper.serviceprovider.ZkServiceProviderBuilder; +import lombok.experimental.UtilityClass; +@UtilityClass public class ServiceProviderBuilders { - - private ServiceProviderBuilders() { - throw new InstantiationError("Must not instantiate this class"); - } - + public static ZkServiceProviderBuilder shardedServiceProviderBuilder() { return new ZkServiceProviderBuilder<>(); } diff --git a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZKUnshardedServiceFinderFactory.java b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZKUnshardedServiceFinderFactory.java index 1de83c51..c645154c 100644 --- a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZKUnshardedServiceFinderFactory.java +++ b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZKUnshardedServiceFinderFactory.java @@ -29,7 +29,6 @@ import org.apache.curator.framework.CuratorFramework; -@Getter public class ZKUnshardedServiceFinderFactory implements ServiceFinderFactory>{ private final CuratorFramework curatorFramework; diff --git a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZkServiceDataSource.java b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZkServiceDataSource.java index 24be855f..7f32f871 100644 --- a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZkServiceDataSource.java +++ b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZkServiceDataSource.java @@ -19,10 +19,13 @@ import com.flipkart.ranger.core.model.Service; import com.google.common.base.Preconditions; import lombok.extern.slf4j.Slf4j; +import lombok.val; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.retry.ExponentialBackoffRetry; +import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Set; import java.util.stream.Collectors; @@ -47,13 +50,13 @@ public ZkServiceDataSource(String namespace, } @Override - public Set services() throws Exception { - final List children = curatorFramework.getChildren() + public Collection services() throws Exception { + val children = curatorFramework.getChildren() .forPath("/"); - return children.stream() - .map(child -> Service.builder().namespace(namespace).serviceName(child).build()) - .collect(Collectors.toSet()); - + return null == children ? Collections.emptySet() : + children.stream() + .map(child -> Service.builder().namespace(namespace).serviceName(child).build()) + .collect(Collectors.toSet()); } @Override @@ -74,6 +77,7 @@ public void start() { } catch (InterruptedException e) { log.error("Curator block interrupted", e); + Thread.currentThread().interrupt(); } log.info("Service data source started. Curator state is: {}", curatorFramework.getState().name()); } diff --git a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/serviceprovider/ZkServiceProviderBuilder.java b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/serviceprovider/ZkServiceProviderBuilder.java index 1a838324..04ca7f65 100644 --- a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/serviceprovider/ZkServiceProviderBuilder.java +++ b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/serviceprovider/ZkServiceProviderBuilder.java @@ -34,7 +34,6 @@ public class ZkServiceProviderBuilder extends BaseServiceProviderBuilder withCuratorFramework(CuratorFramework curatorFramework) { this.curatorFramework = curatorFramework; return this; From c0184445a8e30a624d38e120ef4d91f442910f27 Mon Sep 17 00:00:00 2001 From: Koushik R Date: Tue, 21 Dec 2021 12:29:20 +0530 Subject: [PATCH 38/69] a) Few more lombok fixes --- .../client/http/ShardedRangerHttpHubClient.java | 2 +- .../client/http/UnshardedRangerHttpHubClient.java | 2 +- ranger-http-server/config/local.yml | 6 +++--- .../ranger/http/server/AppConfiguration.java | 15 +++++++-------- .../http/server/bundle/HttpServerBundle.java | 2 +- .../server/config/RangerHttpConfiguration.java | 15 ++++++++------- .../client/zk/AbstractRangerZKHubClient.java | 2 +- ranger-zk-server/config/local.yml | 4 ++-- .../ranger/zk/server/AppConfiguration.java | 13 +++++-------- .../ranger/zk/server/bundle/ZKServerBundle.java | 2 +- .../zk/server/config/RangerConfiguration.java | 10 ++++------ .../zookeeper/servicefinder/ZkNodeDataSource.java | 10 +++++----- 12 files changed, 39 insertions(+), 44 deletions(-) diff --git a/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/ShardedRangerHttpHubClient.java b/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/ShardedRangerHttpHubClient.java index cd60dd43..92d3e7c6 100644 --- a/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/ShardedRangerHttpHubClient.java +++ b/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/ShardedRangerHttpHubClient.java @@ -74,7 +74,7 @@ protected ServiceFinderHub> buildHub() { */ @Override protected ServiceDataSource buildServiceDataSource() { - return null != services && !services.isEmpty() ? + return !services.isEmpty() ? new StaticDataSource(services) : new HttpServiceDataSource<>(clientConfig, getMapper()); } diff --git a/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/UnshardedRangerHttpHubClient.java b/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/UnshardedRangerHttpHubClient.java index 6245ba86..6cc62802 100644 --- a/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/UnshardedRangerHttpHubClient.java +++ b/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/UnshardedRangerHttpHubClient.java @@ -70,7 +70,7 @@ protected ServiceFinderHub> buildHub() { @Override protected ServiceDataSource buildServiceDataSource() { - return null != services && !services.isEmpty() ? + return !services.isEmpty() ? new StaticDataSource(services) : new HttpServiceDataSource<>(clientConfig, getMapper()); } diff --git a/ranger-http-server/config/local.yml b/ranger-http-server/config/local.yml index e0f6ac70..d2c5c581 100644 --- a/ranger-http-server/config/local.yml +++ b/ranger-http-server/config/local.yml @@ -6,17 +6,17 @@ rangerConfiguration: namespace: test httpClientConfigs: - host: localhost - port: 8080 + port: 7080 server: maxThreads: 128 minThreads: 128 applicationConnectors: - type: http - port: 7080 + port: 6080 adminConnectors: - type: http - port: 7081 + port: 6081 applicationContextPath: / requestLog: appenders: diff --git a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/AppConfiguration.java b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/AppConfiguration.java index 91998166..5f3b3d42 100644 --- a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/AppConfiguration.java +++ b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/AppConfiguration.java @@ -15,22 +15,21 @@ */ package com.flipkart.ranger.http.server; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.flipkart.ranger.http.server.config.RangerHttpConfiguration; import io.dropwizard.Configuration; -import lombok.Builder; -import lombok.Value; -import lombok.extern.jackson.Jacksonized; +import lombok.*; + import javax.validation.Valid; import javax.validation.constraints.NotEmpty; import javax.validation.constraints.NotNull; -@Value -@Jacksonized -@Builder -@JsonIgnoreProperties(ignoreUnknown = true) + +@Data +@AllArgsConstructor +@NoArgsConstructor +@EqualsAndHashCode(callSuper = true) public class AppConfiguration extends Configuration { @NotEmpty @NotNull diff --git a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/bundle/HttpServerBundle.java b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/bundle/HttpServerBundle.java index 21ebd918..a9160869 100644 --- a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/bundle/HttpServerBundle.java +++ b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/bundle/HttpServerBundle.java @@ -68,7 +68,7 @@ protected List> withHubs(AppConfiguration configurati .nodeRefreshIntervalMs(rangerConfiguration.getNodeRefreshTimeMs()) .deserializer(data -> { try { - getMapper().readValue(data, new TypeReference>() { + return getMapper().readValue(data, new TypeReference>() { }); } catch (IOException e) { log.warn("Error parsing node data with value {}", new String(data)); diff --git a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/config/RangerHttpConfiguration.java b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/config/RangerHttpConfiguration.java index de04fd1a..77ac0aa7 100644 --- a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/config/RangerHttpConfiguration.java +++ b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/config/RangerHttpConfiguration.java @@ -15,18 +15,19 @@ */ package com.flipkart.ranger.http.server.config; +import com.flipkart.ranger.client.RangerClientConstants; import com.flipkart.ranger.http.config.HttpClientConfig; -import lombok.Builder; -import lombok.Value; +import lombok.*; import lombok.extern.jackson.Jacksonized; +import javax.validation.constraints.Min; import javax.validation.constraints.NotEmpty; import javax.validation.constraints.NotNull; import java.util.List; -@Value -@Builder -@Jacksonized +@Data +@AllArgsConstructor +@NoArgsConstructor public class RangerHttpConfiguration { @NotEmpty @@ -35,6 +36,6 @@ public class RangerHttpConfiguration { @NotEmpty @NotNull List httpClientConfigs; - int nodeRefreshTimeMs; - + @Min(1000) + int nodeRefreshTimeMs = RangerClientConstants.MINIMUM_REFRESH_TIME;; } diff --git a/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/AbstractRangerZKHubClient.java b/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/AbstractRangerZKHubClient.java index c30fc0e6..01220681 100644 --- a/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/AbstractRangerZKHubClient.java +++ b/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/AbstractRangerZKHubClient.java @@ -80,7 +80,7 @@ protected ServiceFinderHub buildHub() { @Override protected ServiceDataSource buildServiceDataSource() { - return null != services && !services.isEmpty() ? + return !services.isEmpty() ? new StaticDataSource(services) : new ZkServiceDataSource(getNamespace(), connectionString, curatorFramework); } diff --git a/ranger-zk-server/config/local.yml b/ranger-zk-server/config/local.yml index b33f9e19..8123e2d2 100644 --- a/ranger-zk-server/config/local.yml +++ b/ranger-zk-server/config/local.yml @@ -11,10 +11,10 @@ server: minThreads: 128 applicationConnectors: - type: http - port: 8080 + port: 7080 adminConnectors: - type: http - port: 8081 + port: 7081 applicationContextPath: / requestLog: appenders: diff --git a/ranger-zk-server/src/main/java/com/flipkart/ranger/zk/server/AppConfiguration.java b/ranger-zk-server/src/main/java/com/flipkart/ranger/zk/server/AppConfiguration.java index 838899ca..0d5a4ff5 100644 --- a/ranger-zk-server/src/main/java/com/flipkart/ranger/zk/server/AppConfiguration.java +++ b/ranger-zk-server/src/main/java/com/flipkart/ranger/zk/server/AppConfiguration.java @@ -18,28 +18,25 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.flipkart.ranger.zk.server.config.RangerConfiguration; import io.dropwizard.Configuration; -import lombok.Builder; -import lombok.Value; +import lombok.*; import lombok.extern.jackson.Jacksonized; import javax.validation.Valid; import javax.validation.constraints.NotEmpty; import javax.validation.constraints.NotNull; -@Value -@Jacksonized -@Builder -@JsonIgnoreProperties(ignoreUnknown = true) +@Data +@AllArgsConstructor +@NoArgsConstructor +@EqualsAndHashCode(callSuper = true) public class AppConfiguration extends Configuration { @NotEmpty @NotNull String name; - @Valid @NotNull RangerConfiguration rangerConfiguration; - boolean initialRotationStatus; } diff --git a/ranger-zk-server/src/main/java/com/flipkart/ranger/zk/server/bundle/ZKServerBundle.java b/ranger-zk-server/src/main/java/com/flipkart/ranger/zk/server/bundle/ZKServerBundle.java index 5e9b923f..0f32680c 100644 --- a/ranger-zk-server/src/main/java/com/flipkart/ranger/zk/server/bundle/ZKServerBundle.java +++ b/ranger-zk-server/src/main/java/com/flipkart/ranger/zk/server/bundle/ZKServerBundle.java @@ -77,7 +77,7 @@ protected List> withHubs(AppConfiguration configurati .refreshTimeMs(rangerConfiguration.getNodeRefreshTimeMs()) .deserializer(data -> { try { - getMapper().readValue(data, new TypeReference>() { + return getMapper().readValue(data, new TypeReference>() { }); } catch (IOException e) { log.warn("Error parsing node data with value {}", new String(data)); diff --git a/ranger-zk-server/src/main/java/com/flipkart/ranger/zk/server/config/RangerConfiguration.java b/ranger-zk-server/src/main/java/com/flipkart/ranger/zk/server/config/RangerConfiguration.java index e18c00bd..e33ee429 100644 --- a/ranger-zk-server/src/main/java/com/flipkart/ranger/zk/server/config/RangerConfiguration.java +++ b/ranger-zk-server/src/main/java/com/flipkart/ranger/zk/server/config/RangerConfiguration.java @@ -17,18 +17,16 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.flipkart.ranger.client.RangerClientConstants; -import lombok.Builder; -import lombok.Value; +import lombok.*; import lombok.extern.jackson.Jacksonized; import javax.validation.constraints.Min; import javax.validation.constraints.NotEmpty; import javax.validation.constraints.NotNull; -@Value -@Jacksonized -@Builder -@JsonIgnoreProperties(ignoreUnknown = true) +@Data +@AllArgsConstructor +@NoArgsConstructor public class RangerConfiguration { @NotEmpty @NotNull diff --git a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinder/ZkNodeDataSource.java b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinder/ZkNodeDataSource.java index f6af1679..3fcc399d 100644 --- a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinder/ZkNodeDataSource.java +++ b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinder/ZkNodeDataSource.java @@ -63,19 +63,19 @@ private List> checkForUpdateOnZookeeper(D deserializer) { } Preconditions.checkNotNull(deserializer, "Deserializer has not been set for node data"); try { - final long healthcheckZombieCheckThresholdTime = healthcheckZombieCheckThresholdTime(service); //1 Minute - final String serviceName = service.getServiceName(); + val healthcheckZombieCheckThresholdTime = healthcheckZombieCheckThresholdTime(service); //1 Minute + val serviceName = service.getServiceName(); if (!isActive()) { log.warn("ZK connection is not active. Ignoring refresh request for service: {}", service.getServiceName()); return Collections.emptyList(); } - final String parentPath = PathBuilder.servicePath(service); + val parentPath = PathBuilder.servicePath(service); log.debug("Looking for node list of [{}]", serviceName); - List children = curatorFramework.getChildren().forPath(parentPath); + val children = curatorFramework.getChildren().forPath(parentPath); List> nodes = Lists.newArrayListWithCapacity(children.size()); log.debug("Found {} nodes for [{}]", children.size(), serviceName); - for (String child : children) { + for (val child : children) { byte[] data = readChild(parentPath, child).orElse(null); if (data == null || data.length <= 0) { continue; From 14f69f05194061c7a26aa4f1811acb44bf34d25a Mon Sep 17 00:00:00 2001 From: Koushik R Date: Tue, 21 Dec 2021 12:40:14 +0530 Subject: [PATCH 39/69] Optimized unnecessary imports --- .../client/AbstractRangerHubClient.java | 5 +++- .../ranger/client/RangerHubClient.java | 1 - .../TestSimpleUnshardedServiceFinder.java | 1 - .../MapBasedServiceRegistry.java | 1 - .../core/finderhub/ServiceDataSource.java | 1 - .../flipkart/ranger/core/model/Service.java | 5 +++- .../core/finder/SimpleShardFinderTest.java | 1 - .../ranger/core/utils/RegistryTestUtils.java | 3 -- .../http/ShardedRangerHttpClientTest.java | 2 -- .../http/SimpleRangerHttpClientTest.java | 2 -- .../http/UnshardedRangerHttpClientTest.java | 2 -- .../ranger/http/server/AppConfiguration.java | 6 ++-- .../http/server/bundle/HttpServerBundle.java | 4 ++- .../config/RangerHttpConfiguration.java | 5 ++-- .../healthcheck/RangerHttpHealthCheck.java | 30 +++++++++++++++++++ .../servicefinder/HttpNodeDataSource.java | 1 - .../HttpServiceDataSource.java | 2 -- .../HttpShardedServiceFinderFactory.java | 1 - .../HttpUnshardedServiceFinderFactory.java | 1 - .../HttpShardedServiceFinderBuilderTest.java | 1 - .../server/bundle/RangerServerBundleTest.java | 1 - .../ranger/common/server/ShardInfoTest.java | 1 - .../ranger/zk/server/AppConfiguration.java | 7 +++-- .../zk/server/config/RangerConfiguration.java | 6 ++-- .../ZKUnshardedServiceFinderFactory.java | 1 - .../servicefinderhub/ZkServiceDataSource.java | 2 -- .../model/ServiceProviderExtCuratorTest.java | 1 - .../zookeeper/servicehub/ServiceHubTest.java | 1 - 28 files changed, 55 insertions(+), 40 deletions(-) create mode 100644 ranger-http-server/src/main/java/com/flipkart/ranger/http/server/healthcheck/RangerHttpHealthCheck.java diff --git a/ranger-client/src/main/java/com/flipkart/ranger/client/AbstractRangerHubClient.java b/ranger-client/src/main/java/com/flipkart/ranger/client/AbstractRangerHubClient.java index 9b93a067..fc773f11 100644 --- a/ranger-client/src/main/java/com/flipkart/ranger/client/AbstractRangerHubClient.java +++ b/ranger-client/src/main/java/com/flipkart/ranger/client/AbstractRangerHubClient.java @@ -28,7 +28,10 @@ import lombok.extern.slf4j.Slf4j; import lombok.val; -import java.util.*; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Optional; import java.util.function.Predicate; @Slf4j diff --git a/ranger-client/src/main/java/com/flipkart/ranger/client/RangerHubClient.java b/ranger-client/src/main/java/com/flipkart/ranger/client/RangerHubClient.java index f0a58803..8f1d8a1d 100644 --- a/ranger-client/src/main/java/com/flipkart/ranger/client/RangerHubClient.java +++ b/ranger-client/src/main/java/com/flipkart/ranger/client/RangerHubClient.java @@ -21,7 +21,6 @@ import java.util.Collection; import java.util.List; import java.util.Optional; -import java.util.Set; import java.util.function.Predicate; public interface RangerHubClient { diff --git a/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestSimpleUnshardedServiceFinder.java b/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestSimpleUnshardedServiceFinder.java index a4452758..7d1d77a5 100644 --- a/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestSimpleUnshardedServiceFinder.java +++ b/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestSimpleUnshardedServiceFinder.java @@ -22,7 +22,6 @@ import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.units.TestNodeData; -import com.google.common.collect.Lists; import lombok.Builder; import java.util.Collections; diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/serviceregistry/MapBasedServiceRegistry.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/serviceregistry/MapBasedServiceRegistry.java index 8d5454d6..874040f7 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/serviceregistry/MapBasedServiceRegistry.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/serviceregistry/MapBasedServiceRegistry.java @@ -25,7 +25,6 @@ import lombok.val; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.concurrent.atomic.AtomicReference; diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/ServiceDataSource.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/ServiceDataSource.java index a9ebb51f..dc579606 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/ServiceDataSource.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/ServiceDataSource.java @@ -18,7 +18,6 @@ import com.flipkart.ranger.core.model.Service; import java.util.Collection; -import java.util.Set; /** * diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/model/Service.java b/ranger-core/src/main/java/com/flipkart/ranger/core/model/Service.java index 2721d2db..160e80bc 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/model/Service.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/model/Service.java @@ -15,7 +15,10 @@ */ package com.flipkart.ranger.core.model; -import lombok.*; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; @Data diff --git a/ranger-core/src/test/java/com/flipkart/ranger/core/finder/SimpleShardFinderTest.java b/ranger-core/src/test/java/com/flipkart/ranger/core/finder/SimpleShardFinderTest.java index f93cd702..13676f59 100644 --- a/ranger-core/src/test/java/com/flipkart/ranger/core/finder/SimpleShardFinderTest.java +++ b/ranger-core/src/test/java/com/flipkart/ranger/core/finder/SimpleShardFinderTest.java @@ -22,7 +22,6 @@ import com.flipkart.ranger.core.units.TestNodeData; import com.flipkart.ranger.core.utils.RangerTestUtils; import com.flipkart.ranger.core.utils.RegistryTestUtils; -import com.google.common.collect.Lists; import lombok.val; import org.junit.Assert; import org.junit.Test; diff --git a/ranger-core/src/test/java/com/flipkart/ranger/core/utils/RegistryTestUtils.java b/ranger-core/src/test/java/com/flipkart/ranger/core/utils/RegistryTestUtils.java index e72590f9..759a946c 100644 --- a/ranger-core/src/test/java/com/flipkart/ranger/core/utils/RegistryTestUtils.java +++ b/ranger-core/src/test/java/com/flipkart/ranger/core/utils/RegistryTestUtils.java @@ -20,12 +20,9 @@ import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.units.TestNodeData; import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; import lombok.experimental.UtilityClass; import lombok.val; -import java.util.List; - @UtilityClass public class RegistryTestUtils { diff --git a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/ShardedRangerHttpClientTest.java b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/ShardedRangerHttpClientTest.java index 867a3060..31341c08 100644 --- a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/ShardedRangerHttpClientTest.java +++ b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/ShardedRangerHttpClientTest.java @@ -23,8 +23,6 @@ import org.junit.Assert; import org.junit.Test; -import java.util.Optional; - public class ShardedRangerHttpClientTest extends BaseRangerHttpClientTest { @Test diff --git a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/SimpleRangerHttpClientTest.java b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/SimpleRangerHttpClientTest.java index 564bdf82..1486a18b 100644 --- a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/SimpleRangerHttpClientTest.java +++ b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/SimpleRangerHttpClientTest.java @@ -22,8 +22,6 @@ import org.junit.Assert; import org.junit.Test; -import java.util.Optional; - public class SimpleRangerHttpClientTest extends BaseRangerHttpClientTest{ @Test diff --git a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/UnshardedRangerHttpClientTest.java b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/UnshardedRangerHttpClientTest.java index 6896a09b..fdc14a18 100644 --- a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/UnshardedRangerHttpClientTest.java +++ b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/UnshardedRangerHttpClientTest.java @@ -23,8 +23,6 @@ import org.junit.Assert; import org.junit.Test; -import java.util.Optional; - public class UnshardedRangerHttpClientTest extends BaseRangerHttpClientTest { @Test diff --git a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/AppConfiguration.java b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/AppConfiguration.java index 5f3b3d42..e1bb989f 100644 --- a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/AppConfiguration.java +++ b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/AppConfiguration.java @@ -17,8 +17,10 @@ import com.flipkart.ranger.http.server.config.RangerHttpConfiguration; import io.dropwizard.Configuration; -import lombok.*; - +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; import javax.validation.Valid; import javax.validation.constraints.NotEmpty; diff --git a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/bundle/HttpServerBundle.java b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/bundle/HttpServerBundle.java index a9160869..9492ddac 100644 --- a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/bundle/HttpServerBundle.java +++ b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/bundle/HttpServerBundle.java @@ -22,9 +22,11 @@ import com.flipkart.ranger.common.server.ShardInfo; import com.flipkart.ranger.http.model.ServiceNodesResponse; import com.flipkart.ranger.http.server.AppConfiguration; +import com.flipkart.ranger.http.server.healthcheck.RangerHttpHealthCheck; import com.flipkart.ranger.zk.server.bundle.RangerServerBundle; import com.flipkart.ranger.zk.server.bundle.model.LifecycleSignal; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; import lombok.val; @@ -90,6 +92,6 @@ protected List withLifecycleSignals(AppConfiguration configurat @Override protected List withHealthChecks(AppConfiguration configuration) { - return Collections.emptyList(); + return ImmutableList.of(new RangerHttpHealthCheck()); } } diff --git a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/config/RangerHttpConfiguration.java b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/config/RangerHttpConfiguration.java index 77ac0aa7..585a3b02 100644 --- a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/config/RangerHttpConfiguration.java +++ b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/config/RangerHttpConfiguration.java @@ -17,8 +17,9 @@ import com.flipkart.ranger.client.RangerClientConstants; import com.flipkart.ranger.http.config.HttpClientConfig; -import lombok.*; -import lombok.extern.jackson.Jacksonized; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; import javax.validation.constraints.Min; import javax.validation.constraints.NotEmpty; diff --git a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/healthcheck/RangerHttpHealthCheck.java b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/healthcheck/RangerHttpHealthCheck.java new file mode 100644 index 00000000..cc63fa45 --- /dev/null +++ b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/healthcheck/RangerHttpHealthCheck.java @@ -0,0 +1,30 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.http.server.healthcheck; + +import com.codahale.metrics.health.HealthCheck; +import lombok.Data; + +import javax.inject.Singleton; + +@Singleton +@Data +public class RangerHttpHealthCheck extends HealthCheck { + @Override + protected Result check() { + return HealthCheck.Result.healthy("Healthy"); + } +} diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpNodeDataSource.java b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpNodeDataSource.java index 864103ae..5d5c1b22 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpNodeDataSource.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpNodeDataSource.java @@ -28,7 +28,6 @@ import lombok.val; import okhttp3.HttpUrl; import okhttp3.Request; -import okhttp3.ResponseBody; import java.io.IOException; import java.util.Collections; diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSource.java b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSource.java index 14fe59da..ad68d84d 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSource.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSource.java @@ -27,12 +27,10 @@ import lombok.val; import okhttp3.HttpUrl; import okhttp3.Request; -import okhttp3.ResponseBody; import java.io.IOException; import java.util.Collection; import java.util.Collections; -import java.util.Set; @Slf4j public class HttpServiceDataSource extends HttpNodeDataStoreConnector implements ServiceDataSource { diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpShardedServiceFinderFactory.java b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpShardedServiceFinderFactory.java index a4274d61..96894e25 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpShardedServiceFinderFactory.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpShardedServiceFinderFactory.java @@ -26,7 +26,6 @@ import com.flipkart.ranger.http.serde.HTTPResponseDataDeserializer; import com.flipkart.ranger.http.servicefinder.HttpShardedServiceFinderBuilder; import lombok.Builder; -import lombok.Getter; import lombok.val; public class HttpShardedServiceFinderFactory implements ServiceFinderFactory> { diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpUnshardedServiceFinderFactory.java b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpUnshardedServiceFinderFactory.java index 7874d80a..308d7a1d 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpUnshardedServiceFinderFactory.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpUnshardedServiceFinderFactory.java @@ -26,7 +26,6 @@ import com.flipkart.ranger.http.serde.HTTPResponseDataDeserializer; import com.flipkart.ranger.http.servicefinder.HttpUnshardedServiceFinderBuilider; import lombok.Builder; -import lombok.Getter; import lombok.val; public class HttpUnshardedServiceFinderFactory implements ServiceFinderFactory> { diff --git a/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java b/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java index aeb8727b..4030e1ef 100644 --- a/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java +++ b/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java @@ -25,7 +25,6 @@ import com.flipkart.ranger.http.model.ServiceNodesResponse; import com.flipkart.ranger.http.response.model.RangerResponseCode; import com.github.tomakehurst.wiremock.junit.WireMockRule; -import com.google.common.collect.Lists; import lombok.Data; import lombok.val; import org.junit.Assert; diff --git a/ranger-server-bundle/src/test/java/com/flipkart/ranger/zk/server/bundle/RangerServerBundleTest.java b/ranger-server-bundle/src/test/java/com/flipkart/ranger/zk/server/bundle/RangerServerBundleTest.java index fdaeb089..e6974ee2 100644 --- a/ranger-server-bundle/src/test/java/com/flipkart/ranger/zk/server/bundle/RangerServerBundleTest.java +++ b/ranger-server-bundle/src/test/java/com/flipkart/ranger/zk/server/bundle/RangerServerBundleTest.java @@ -25,7 +25,6 @@ import com.flipkart.ranger.core.utils.RangerTestUtils; import com.flipkart.ranger.core.utils.TestUtils; import com.flipkart.ranger.zk.server.bundle.model.LifecycleSignal; -import com.google.common.collect.Lists; import io.dropwizard.Configuration; import io.dropwizard.jersey.DropwizardResourceConfig; import io.dropwizard.jersey.setup.JerseyEnvironment; diff --git a/ranger-server-common/src/test/java/com/flipkart/ranger/common/server/ShardInfoTest.java b/ranger-server-common/src/test/java/com/flipkart/ranger/common/server/ShardInfoTest.java index 39ae6436..98020fc9 100644 --- a/ranger-server-common/src/test/java/com/flipkart/ranger/common/server/ShardInfoTest.java +++ b/ranger-server-common/src/test/java/com/flipkart/ranger/common/server/ShardInfoTest.java @@ -2,7 +2,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import lombok.SneakyThrows; -import lombok.experimental.UtilityClass; import org.junit.Assert; import org.junit.Test; diff --git a/ranger-zk-server/src/main/java/com/flipkart/ranger/zk/server/AppConfiguration.java b/ranger-zk-server/src/main/java/com/flipkart/ranger/zk/server/AppConfiguration.java index 0d5a4ff5..857c6484 100644 --- a/ranger-zk-server/src/main/java/com/flipkart/ranger/zk/server/AppConfiguration.java +++ b/ranger-zk-server/src/main/java/com/flipkart/ranger/zk/server/AppConfiguration.java @@ -15,11 +15,12 @@ */ package com.flipkart.ranger.zk.server; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.flipkart.ranger.zk.server.config.RangerConfiguration; import io.dropwizard.Configuration; -import lombok.*; -import lombok.extern.jackson.Jacksonized; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; import javax.validation.Valid; import javax.validation.constraints.NotEmpty; diff --git a/ranger-zk-server/src/main/java/com/flipkart/ranger/zk/server/config/RangerConfiguration.java b/ranger-zk-server/src/main/java/com/flipkart/ranger/zk/server/config/RangerConfiguration.java index e33ee429..c258c628 100644 --- a/ranger-zk-server/src/main/java/com/flipkart/ranger/zk/server/config/RangerConfiguration.java +++ b/ranger-zk-server/src/main/java/com/flipkart/ranger/zk/server/config/RangerConfiguration.java @@ -15,10 +15,10 @@ */ package com.flipkart.ranger.zk.server.config; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.flipkart.ranger.client.RangerClientConstants; -import lombok.*; -import lombok.extern.jackson.Jacksonized; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; import javax.validation.constraints.Min; import javax.validation.constraints.NotEmpty; diff --git a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZKUnshardedServiceFinderFactory.java b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZKUnshardedServiceFinderFactory.java index c645154c..3924757f 100644 --- a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZKUnshardedServiceFinderFactory.java +++ b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZKUnshardedServiceFinderFactory.java @@ -24,7 +24,6 @@ import com.flipkart.ranger.zookeeper.serde.ZkNodeDataDeserializer; import com.flipkart.ranger.zookeeper.servicefinder.ZkSimpleUnshardedServiceFinderBuilder; import lombok.Builder; -import lombok.Getter; import lombok.val; import org.apache.curator.framework.CuratorFramework; diff --git a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZkServiceDataSource.java b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZkServiceDataSource.java index 7f32f871..08058b78 100644 --- a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZkServiceDataSource.java +++ b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZkServiceDataSource.java @@ -26,8 +26,6 @@ import java.util.Collection; import java.util.Collections; -import java.util.List; -import java.util.Set; import java.util.stream.Collectors; /** diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderExtCuratorTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderExtCuratorTest.java index f8e805c9..7b3ad669 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderExtCuratorTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderExtCuratorTest.java @@ -18,7 +18,6 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import com.flipkart.ranger.core.finder.SimpleShardedServiceFinder; import com.flipkart.ranger.core.healthcheck.Healthchecks; import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.serviceprovider.ServiceProvider; diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/servicehub/ServiceHubTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/servicehub/ServiceHubTest.java index 7e7ba496..afa6c4b1 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/servicehub/ServiceHubTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/servicehub/ServiceHubTest.java @@ -44,7 +44,6 @@ import java.io.IOException; import java.util.Collections; import java.util.Date; -import java.util.function.Predicate; /** * From b009ee389c903fb326f3ae16c48e5e483f0844f1 Mon Sep 17 00:00:00 2001 From: Koushik R Date: Tue, 21 Dec 2021 12:41:29 +0530 Subject: [PATCH 40/69] val fixes in FinderHub --- .../com/flipkart/ranger/core/finderhub/ServiceFinderHub.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/ServiceFinderHub.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/ServiceFinderHub.java index 16ca280d..b8399446 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/ServiceFinderHub.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/ServiceFinderHub.java @@ -144,8 +144,8 @@ private void updateRegistry() { alreadyUpdating.set(true); final Map> updatedFinders = new HashMap<>(); try { - final Collection services = serviceDataSource.services(); - final Map> knownServiceFinders = finders.get(); + val services = serviceDataSource.services(); + val knownServiceFinders = finders.get(); val newFinders = services.stream() .filter(service -> !knownServiceFinders.containsKey(service)) .collect(Collectors.toMap(Function.identity(), finderFactory::buildFinder)); From a8a9c6b58c1b807dd624349fe01d699f25108fe5 Mon Sep 17 00:00:00 2001 From: Koushik R Date: Tue, 21 Dec 2021 13:16:54 +0530 Subject: [PATCH 41/69] The bug in PathBuilder is fixed. The servicePath and instancePath should both contain namespace and that is missing --- .../common/ZkNodeDataStoreConnector.java | 14 +++++++++++-- .../ranger/zookeeper/common/ZkStoreType.java | 20 +++++++++++++++++++ .../servicefinder/ZkNodeDataSource.java | 3 ++- .../servicefinderhub/ZkServiceDataSource.java | 3 ++- .../serviceprovider/ZkNodeDataSink.java | 3 ++- .../ranger/zookeeper/util/PathBuilder.java | 4 ++-- 6 files changed, 40 insertions(+), 7 deletions(-) create mode 100644 ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/common/ZkStoreType.java diff --git a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/common/ZkNodeDataStoreConnector.java b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/common/ZkNodeDataStoreConnector.java index 79261dee..a708e3c5 100644 --- a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/common/ZkNodeDataStoreConnector.java +++ b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/common/ZkNodeDataStoreConnector.java @@ -23,6 +23,7 @@ import lombok.AccessLevel; import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import lombok.val; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.imps.CuratorFrameworkState; import org.apache.zookeeper.KeeperException; @@ -40,6 +41,7 @@ public class ZkNodeDataStoreConnector implements NodeDataStoreConnector { protected final Service service; @Getter(AccessLevel.PROTECTED) protected final CuratorFramework curatorFramework; + protected final ZkStoreType storeType; private final AtomicBoolean started = new AtomicBoolean(false); private final AtomicBoolean stopped = new AtomicBoolean(false); @@ -60,19 +62,27 @@ public void onRetry(Attempt attempt) { protected ZkNodeDataStoreConnector( final Service service, - final CuratorFramework curatorFramework) { + final CuratorFramework curatorFramework, + final ZkStoreType storeType) { this.service = service; this.curatorFramework = curatorFramework; + this.storeType = storeType; } @Override public void start() { + if(storeType == ZkStoreType.SOURCE){ + log.info("Start called on a data source will not do anything, since we don't have to create paths for services found in source. Ignoring after setting started"); + started.set(true); + return; + } + if (started.get()) { log.info("Start called on already initialized data source for service {}. Ignoring.", service.getServiceName()); return; } - final String path = PathBuilder.servicePath(service); + val path = PathBuilder.servicePath(service); try { curatorFramework.blockUntilConnected(); log.info("Connected to zookeeper cluster for {}", service.getServiceName()); diff --git a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/common/ZkStoreType.java b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/common/ZkStoreType.java new file mode 100644 index 00000000..c5271cc2 --- /dev/null +++ b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/common/ZkStoreType.java @@ -0,0 +1,20 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.zookeeper.common; + +public enum ZkStoreType { + SOURCE, SINK; +} diff --git a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinder/ZkNodeDataSource.java b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinder/ZkNodeDataSource.java index 3fcc399d..733f1c9b 100644 --- a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinder/ZkNodeDataSource.java +++ b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinder/ZkNodeDataSource.java @@ -20,6 +20,7 @@ import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.util.FinderUtils; import com.flipkart.ranger.zookeeper.common.ZkNodeDataStoreConnector; +import com.flipkart.ranger.zookeeper.common.ZkStoreType; import com.flipkart.ranger.zookeeper.serde.ZkNodeDataDeserializer; import com.flipkart.ranger.zookeeper.util.PathBuilder; import com.google.common.base.Preconditions; @@ -42,7 +43,7 @@ public class ZkNodeDataSource> extends Zk public ZkNodeDataSource( Service service, CuratorFramework curatorFramework) { - super(service, curatorFramework); + super(service, curatorFramework, ZkStoreType.SOURCE); } @Override diff --git a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZkServiceDataSource.java b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZkServiceDataSource.java index 08058b78..e6658166 100644 --- a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZkServiceDataSource.java +++ b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZkServiceDataSource.java @@ -17,6 +17,7 @@ import com.flipkart.ranger.core.finderhub.ServiceDataSource; import com.flipkart.ranger.core.model.Service; +import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import lombok.extern.slf4j.Slf4j; import lombok.val; @@ -50,7 +51,7 @@ public ZkServiceDataSource(String namespace, @Override public Collection services() throws Exception { val children = curatorFramework.getChildren() - .forPath("/"); + .forPath("/" + namespace); return null == children ? Collections.emptySet() : children.stream() .map(child -> Service.builder().namespace(namespace).serviceName(child).build()) diff --git a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/serviceprovider/ZkNodeDataSink.java b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/serviceprovider/ZkNodeDataSink.java index 7aeb99e8..83ad3ae5 100644 --- a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/serviceprovider/ZkNodeDataSink.java +++ b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/serviceprovider/ZkNodeDataSink.java @@ -20,6 +20,7 @@ import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.util.Exceptions; import com.flipkart.ranger.zookeeper.common.ZkNodeDataStoreConnector; +import com.flipkart.ranger.zookeeper.common.ZkStoreType; import com.flipkart.ranger.zookeeper.serde.ZkNodeDataSerializer; import com.flipkart.ranger.zookeeper.util.PathBuilder; import com.google.common.base.Preconditions; @@ -37,7 +38,7 @@ public class ZkNodeDataSink> extends ZkNode public ZkNodeDataSink( Service service, CuratorFramework curatorFramework) { - super(service, curatorFramework); + super(service, curatorFramework, ZkStoreType.SINK); } @Override diff --git a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/util/PathBuilder.java b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/util/PathBuilder.java index 3474c749..c02e2342 100644 --- a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/util/PathBuilder.java +++ b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/util/PathBuilder.java @@ -25,10 +25,10 @@ public class PathBuilder { public static String servicePath(final Service service) { - return String.format("/%s", service.getServiceName()); + return String.format("/%s/%s", service.getNamespace(), service.getServiceName()); } public static String instancePath(final Service service, final ServiceNode node) { - return String.format("/%s/%s", service.getServiceName(), node.representation()); + return String.format("/%s/%s/%s", service.getNamespace(), service.getServiceName(), node.representation()); } } From a9ca546d983c0df075e0593fdde302552f6c8bb0 Mon Sep 17 00:00:00 2001 From: Koushik R Date: Tue, 21 Dec 2021 13:18:38 +0530 Subject: [PATCH 42/69] Pathbug in datsource is also fixed --- .../zookeeper/servicefinderhub/ZkServiceDataSource.java | 3 ++- .../com/flipkart/ranger/zookeeper/util/PathBuilder.java | 7 ++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZkServiceDataSource.java b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZkServiceDataSource.java index e6658166..adff53bc 100644 --- a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZkServiceDataSource.java +++ b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZkServiceDataSource.java @@ -17,6 +17,7 @@ import com.flipkart.ranger.core.finderhub.ServiceDataSource; import com.flipkart.ranger.core.model.Service; +import com.flipkart.ranger.zookeeper.util.PathBuilder; import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import lombok.extern.slf4j.Slf4j; @@ -51,7 +52,7 @@ public ZkServiceDataSource(String namespace, @Override public Collection services() throws Exception { val children = curatorFramework.getChildren() - .forPath("/" + namespace); + .forPath(PathBuilder.registeredServicesPath(namespace)); return null == children ? Collections.emptySet() : children.stream() .map(child -> Service.builder().namespace(namespace).serviceName(child).build()) diff --git a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/util/PathBuilder.java b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/util/PathBuilder.java index c02e2342..0686b27d 100644 --- a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/util/PathBuilder.java +++ b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/util/PathBuilder.java @@ -20,10 +20,15 @@ import com.flipkart.ranger.core.model.ServiceNode; import lombok.AccessLevel; import lombok.NoArgsConstructor; +import lombok.experimental.UtilityClass; -@NoArgsConstructor(access = AccessLevel.PRIVATE) +@UtilityClass public class PathBuilder { + public static String registeredServicesPath(final String namespace){ + return String.format("/%s", namespace); + } + public static String servicePath(final Service service) { return String.format("/%s/%s", service.getNamespace(), service.getServiceName()); } From 7bb110388672aeb98cb88c44907c56392afb0d75 Mon Sep 17 00:00:00 2001 From: Koushik R Date: Thu, 23 Dec 2021 11:20:08 +0530 Subject: [PATCH 43/69] Added a feature to merge the initial criteria if required with the argCriteria in the rangerClients - simple & hub --- .../ranger/client/AbstractRangerClient.java | 63 +++++++++++++++++++ .../client/AbstractRangerHubClient.java | 24 ++++--- .../ranger/client/utils/CriteriaUtils.java | 39 ++++++++++++ .../ranger/client/stubs/RangerTestHub.java | 2 +- .../http/ShardedRangerHttpHubClient.java | 5 +- .../client/http/SimpleRangerHttpClient.java | 40 ++++-------- .../http/UnshardedRangerHttpHubClient.java | 5 +- .../client/zk/AbstractRangerZKHubClient.java | 5 +- .../client/zk/ShardedRangerZKHubClient.java | 6 +- .../client/zk/SimpleRangerZKClient.java | 47 +++++--------- .../client/zk/UnshardedRangerZKHubClient.java | 6 +- .../servicefinderhub/ZkServiceDataSource.java | 1 - .../ranger/zookeeper/util/PathBuilder.java | 2 - 13 files changed, 161 insertions(+), 84 deletions(-) create mode 100644 ranger-client/src/main/java/com/flipkart/ranger/client/AbstractRangerClient.java create mode 100644 ranger-client/src/main/java/com/flipkart/ranger/client/utils/CriteriaUtils.java diff --git a/ranger-client/src/main/java/com/flipkart/ranger/client/AbstractRangerClient.java b/ranger-client/src/main/java/com/flipkart/ranger/client/AbstractRangerClient.java new file mode 100644 index 00000000..58e836a0 --- /dev/null +++ b/ranger-client/src/main/java/com/flipkart/ranger/client/AbstractRangerClient.java @@ -0,0 +1,63 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.client; + +import com.flipkart.ranger.client.utils.CriteriaUtils; +import com.flipkart.ranger.core.finder.ServiceFinder; +import com.flipkart.ranger.core.model.ServiceNode; +import com.flipkart.ranger.core.model.ServiceRegistry; +import lombok.val; + +import java.util.List; +import java.util.Optional; +import java.util.function.Predicate; + +public abstract class AbstractRangerClient> implements RangerClient { + + private final Predicate initialCriteria; + private final boolean alwaysUseInitialCriteria; + + protected AbstractRangerClient(Predicate initialCriteria, boolean alwaysUseInitialCriteria){ + this.initialCriteria = initialCriteria; + this.alwaysUseInitialCriteria = alwaysUseInitialCriteria; + } + + public abstract ServiceFinder getServiceFinder(); + + @Override + public Optional> getNode() { + val finder = getServiceFinder(); + return finder.get(initialCriteria); + } + + @Override + public Optional> getNode(Predicate criteria) { + val finder = getServiceFinder(); + return finder.get(CriteriaUtils.getCriteria(alwaysUseInitialCriteria, initialCriteria, criteria)); + } + + @Override + public List> getAllNodes() { + val finder = getServiceFinder(); + return finder.getAll(initialCriteria); + } + + @Override + public List> getAllNodes(Predicate criteria) { + val finder = getServiceFinder(); + return finder.getAll(CriteriaUtils.getCriteria(alwaysUseInitialCriteria, initialCriteria, criteria)); + } +} diff --git a/ranger-client/src/main/java/com/flipkart/ranger/client/AbstractRangerHubClient.java b/ranger-client/src/main/java/com/flipkart/ranger/client/AbstractRangerHubClient.java index fc773f11..1a71eebf 100644 --- a/ranger-client/src/main/java/com/flipkart/ranger/client/AbstractRangerHubClient.java +++ b/ranger-client/src/main/java/com/flipkart/ranger/client/AbstractRangerHubClient.java @@ -16,6 +16,7 @@ package com.flipkart.ranger.client; import com.fasterxml.jackson.databind.ObjectMapper; +import com.flipkart.ranger.client.utils.CriteriaUtils; import com.flipkart.ranger.core.finderhub.ServiceDataSource; import com.flipkart.ranger.core.finderhub.ServiceFinderFactory; import com.flipkart.ranger.core.finderhub.ServiceFinderHub; @@ -44,20 +45,23 @@ public abstract class AbstractRangerHubClient, D private int nodeRefreshTimeMs; private ServiceFinderHub hub; - private final Predicate criteria; + private final Predicate initialCriteria; + private final boolean alwaysUseInitialCriteria; protected AbstractRangerHubClient( String namespace, ObjectMapper mapper, int nodeRefreshTimeMs, - Predicate criteria, - D deserializer + Predicate initialCriteria, + D deserializer, + boolean alwaysUseInitialCriteria ){ this.namespace = namespace; this.mapper = mapper; this.nodeRefreshTimeMs = nodeRefreshTimeMs; - this.criteria = criteria; + this.initialCriteria = initialCriteria; this.deserializer = deserializer; + this.alwaysUseInitialCriteria = alwaysUseInitialCriteria; } public void start(){ @@ -82,13 +86,13 @@ public void stop(){ public Optional> getNode( final Service service ){ - return getNode(service, criteria); + return getNode(service, initialCriteria); } public List> getAllNodes( final Service service ){ - return getAllNodes(service, criteria); + return getAllNodes(service, initialCriteria); } public Optional> getNode( @@ -96,7 +100,9 @@ public Optional> getNode( final Predicate criteria ){ val optionalFinder = this.getHub().finder(service); - return optionalFinder.flatMap(trServiceFinder -> trServiceFinder.get(criteria)); + return optionalFinder.flatMap(trServiceFinder -> trServiceFinder.get( + CriteriaUtils.getCriteria(alwaysUseInitialCriteria, initialCriteria, criteria)) + ); } public List> getAllNodes( @@ -104,7 +110,9 @@ public List> getAllNodes( final Predicate criteria ){ val optionalFinder = this.getHub().finder(service); - return optionalFinder.map(trServiceFinder -> trServiceFinder.getAll(criteria)).orElse(Collections.emptyList()); + return optionalFinder.map(trServiceFinder -> trServiceFinder.getAll( + CriteriaUtils.getCriteria(alwaysUseInitialCriteria, initialCriteria, criteria)) + ).orElse(Collections.emptyList()); } public Collection getRegisteredServices() { diff --git a/ranger-client/src/main/java/com/flipkart/ranger/client/utils/CriteriaUtils.java b/ranger-client/src/main/java/com/flipkart/ranger/client/utils/CriteriaUtils.java new file mode 100644 index 00000000..eae89c7f --- /dev/null +++ b/ranger-client/src/main/java/com/flipkart/ranger/client/utils/CriteriaUtils.java @@ -0,0 +1,39 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.client.utils; + +import lombok.experimental.UtilityClass; + +import java.util.function.Predicate; + +@UtilityClass +public class CriteriaUtils { + + /* + We use this merge only when the explicit arg criteria is passed to the clients. + The idea is to check if this has to be merged with the initialCriteria or not. + + So the default return is always argCriteria, in case the initial criteria is null! + */ + public static Predicate getCriteria( + boolean useInitialCriteria, + Predicate initialCriteria, + Predicate argCriteria + ){ + return null != initialCriteria && null != argCriteria && useInitialCriteria ? + initialCriteria.and(argCriteria) : argCriteria; + } +} diff --git a/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/RangerTestHub.java b/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/RangerTestHub.java index 0bb28839..058dc5b9 100644 --- a/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/RangerTestHub.java +++ b/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/RangerTestHub.java @@ -32,7 +32,7 @@ public class RangerTestHub extends AbstractRangerHubClient criteria, TestDeserializer deserilizer) { - super(namespace, mapper, nodeRefreshTimeMs, criteria, deserilizer); + super(namespace, mapper, nodeRefreshTimeMs, criteria, deserilizer, false); } @Override diff --git a/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/ShardedRangerHttpHubClient.java b/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/ShardedRangerHttpHubClient.java index 92d3e7c6..e88f787c 100644 --- a/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/ShardedRangerHttpHubClient.java +++ b/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/ShardedRangerHttpHubClient.java @@ -52,9 +52,10 @@ public ShardedRangerHttpHubClient( Predicate criteria, HTTPResponseDataDeserializer deserializer, HttpClientConfig clientConfig, - Set services + Set services, + boolean alwaysUseInitialCriteria ) { - super(namespace, mapper, nodeRefreshIntervalMs, criteria, deserializer); + super(namespace, mapper, nodeRefreshIntervalMs, criteria, deserializer, alwaysUseInitialCriteria); this.clientConfig = clientConfig; this.services = null != services ? services : Collections.emptySet(); } diff --git a/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/SimpleRangerHttpClient.java b/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/SimpleRangerHttpClient.java index b73cbb2e..7510b662 100644 --- a/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/SimpleRangerHttpClient.java +++ b/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/SimpleRangerHttpClient.java @@ -16,25 +16,24 @@ package com.flipkart.ranger.client.http; import com.fasterxml.jackson.databind.ObjectMapper; -import com.flipkart.ranger.client.RangerClient; +import com.flipkart.ranger.client.AbstractRangerClient; import com.flipkart.ranger.core.finder.SimpleUnshardedServiceFinder; +import com.flipkart.ranger.core.finder.serviceregistry.ListBasedServiceRegistry; import com.flipkart.ranger.core.finder.shardselector.ListShardSelector; -import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.http.HttpServiceFinderBuilders; import com.flipkart.ranger.http.config.HttpClientConfig; import com.flipkart.ranger.http.serde.HTTPResponseDataDeserializer; import com.google.common.base.Preconditions; import lombok.Builder; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; -import java.util.List; -import java.util.Optional; import java.util.function.Predicate; @Slf4j -public class SimpleRangerHttpClient implements RangerClient { +public class SimpleRangerHttpClient extends AbstractRangerClient> { - private final Predicate criteria; + @Getter private final SimpleUnshardedServiceFinder serviceFinder; @Builder @@ -44,15 +43,17 @@ public SimpleRangerHttpClient( ObjectMapper mapper, int nodeRefreshIntervalMs, HttpClientConfig clientConfig, - Predicate criteria, - HTTPResponseDataDeserializer deserializer + Predicate initialCriteria, + HTTPResponseDataDeserializer deserializer, + boolean alwaysUseInitialCriteria ) { + + super(initialCriteria, alwaysUseInitialCriteria); + Preconditions.checkNotNull(mapper, "Mapper can't be null"); Preconditions.checkNotNull(namespace, "namespace can't be null"); Preconditions.checkNotNull(deserializer, "deserializer can't be null"); - this.criteria = criteria; - this.serviceFinder = HttpServiceFinderBuilders.httpUnshardedServiceFinderBuilider() .withClientConfig(clientConfig) .withServiceName(serviceName) @@ -76,24 +77,5 @@ public void stop() { this.serviceFinder.stop(); } - @Override - public Optional> getNode() { - return getNode(criteria); - } - - @Override - public List> getAllNodes() { - return getAllNodes(criteria); - } - - @Override - public Optional> getNode(Predicate criteria) { - return this.serviceFinder.get(criteria); - } - - @Override - public List> getAllNodes(Predicate criteria) { - return this.serviceFinder.getAll(criteria); - } } diff --git a/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/UnshardedRangerHttpHubClient.java b/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/UnshardedRangerHttpHubClient.java index 6cc62802..40025b87 100644 --- a/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/UnshardedRangerHttpHubClient.java +++ b/ranger-http-client/src/main/java/com/flipkart/ranger/client/http/UnshardedRangerHttpHubClient.java @@ -52,9 +52,10 @@ public UnshardedRangerHttpHubClient( Predicate criteria, HTTPResponseDataDeserializer deserializer, HttpClientConfig clientConfig, - Set services + Set services, + boolean alwaysUseInitialCriteria ) { - super(namespace, mapper, nodeRefreshIntervalMs, criteria, deserializer); + super(namespace, mapper, nodeRefreshIntervalMs, criteria, deserializer, alwaysUseInitialCriteria); this.clientConfig = clientConfig; this.services = null != services ? services : Collections.emptySet(); } diff --git a/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/AbstractRangerZKHubClient.java b/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/AbstractRangerZKHubClient.java index 01220681..63fb6858 100644 --- a/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/AbstractRangerZKHubClient.java +++ b/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/AbstractRangerZKHubClient.java @@ -56,9 +56,10 @@ protected AbstractRangerZKHubClient( CuratorFramework curatorFramework, Predicate criteria, D deserializer, - Set services + Set services, + boolean alwaysUseInitialCriteria ) { - super(namespace, mapper, nodeRefreshIntervalMs, criteria, deserializer); + super(namespace, mapper, nodeRefreshIntervalMs, criteria, deserializer, alwaysUseInitialCriteria); this.disablePushUpdaters = disablePushUpdaters; this.connectionString = connectionString; this.services = null != services ? services : Collections.emptySet(); diff --git a/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/ShardedRangerZKHubClient.java b/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/ShardedRangerZKHubClient.java index 6d137536..856f0239 100644 --- a/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/ShardedRangerZKHubClient.java +++ b/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/ShardedRangerZKHubClient.java @@ -44,7 +44,8 @@ public ShardedRangerZKHubClient( CuratorFramework curatorFramework, Predicate criteria, ZkNodeDataDeserializer deserializer, - Set services + Set services, + boolean alwaysUseInitialCriteria ) { super( namespace, @@ -55,7 +56,8 @@ public ShardedRangerZKHubClient( curatorFramework, criteria, deserializer, - services + services, + alwaysUseInitialCriteria ); } diff --git a/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/SimpleRangerZKClient.java b/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/SimpleRangerZKClient.java index bc2f226e..0c418bf3 100644 --- a/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/SimpleRangerZKClient.java +++ b/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/SimpleRangerZKClient.java @@ -16,11 +16,11 @@ package com.flipkart.ranger.client.zk; import com.fasterxml.jackson.databind.ObjectMapper; -import com.flipkart.ranger.client.RangerClient; +import com.flipkart.ranger.client.AbstractRangerClient; import com.flipkart.ranger.client.RangerClientConstants; import com.flipkart.ranger.core.finder.SimpleShardedServiceFinder; +import com.flipkart.ranger.core.finder.serviceregistry.MapBasedServiceRegistry; import com.flipkart.ranger.core.finder.shardselector.MatchingShardSelector; -import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.zookeeper.ServiceFinderBuilders; import com.flipkart.ranger.zookeeper.serde.ZkNodeDataDeserializer; import com.google.common.base.Preconditions; @@ -31,16 +31,13 @@ import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.retry.RetryForever; -import java.util.List; -import java.util.Optional; import java.util.function.Predicate; @Slf4j @Getter -public class SimpleRangerZKClient implements RangerClient { +public class SimpleRangerZKClient extends AbstractRangerClient> { private final SimpleShardedServiceFinder serviceFinder; - private final Predicate criteria; private final ZkNodeDataDeserializer deserializer; @Builder(builderMethodName = "fromConnectionString", builderClassName = "FromConnectionStringBuilder") @@ -51,8 +48,9 @@ public SimpleRangerZKClient( int nodeRefreshIntervalMs, boolean disableWatchers, String connectionString, - Predicate criteria, - ZkNodeDataDeserializer deserializer + Predicate initialCriteria, + ZkNodeDataDeserializer deserializer, + boolean alwaysUseInitialCriteria ){ this( namespace, @@ -61,8 +59,9 @@ public SimpleRangerZKClient( nodeRefreshIntervalMs, disableWatchers, CuratorFrameworkFactory.newClient(connectionString, new RetryForever(RangerClientConstants.CONNECTION_RETRY_TIME)), - criteria, - deserializer + initialCriteria, + deserializer, + alwaysUseInitialCriteria ); } @@ -74,9 +73,12 @@ public SimpleRangerZKClient( int nodeRefreshIntervalMs, boolean disableWatchers, CuratorFramework curatorFramework, - Predicate criteria, - ZkNodeDataDeserializer deserializer + Predicate initialCriteria, + ZkNodeDataDeserializer deserializer, + boolean alwaysUseInitialCriteria ){ + super(initialCriteria, alwaysUseInitialCriteria); + Preconditions.checkNotNull(mapper, "Mapper can't be null"); Preconditions.checkNotNull(namespace, "namespace can't be null"); Preconditions.checkNotNull(deserializer, "deserializer can't be null"); @@ -89,7 +91,6 @@ public SimpleRangerZKClient( RangerClientConstants.MINIMUM_REFRESH_TIME); } - this.criteria = criteria; this.deserializer = deserializer; this.serviceFinder = ServiceFinderBuilders.shardedFinderBuilder() .withCuratorFramework(curatorFramework) @@ -113,24 +114,4 @@ public void stop() { log.info("Stopping the service finder"); this.serviceFinder.stop(); } - - @Override - public Optional> getNode() { - return getNode(criteria); - } - - @Override - public List> getAllNodes() { - return getAllNodes(criteria); - } - - @Override - public Optional> getNode(Predicate criteria) { - return this.serviceFinder.get(criteria); - } - - @Override - public List> getAllNodes(Predicate criteria) { - return this.serviceFinder.getAll(criteria); - } } diff --git a/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/UnshardedRangerZKHubClient.java b/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/UnshardedRangerZKHubClient.java index df7b0bde..fc44533b 100644 --- a/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/UnshardedRangerZKHubClient.java +++ b/ranger-zk-client/src/main/java/com/flipkart/ranger/client/zk/UnshardedRangerZKHubClient.java @@ -44,7 +44,8 @@ public UnshardedRangerZKHubClient( CuratorFramework curatorFramework, Predicate criteria, ZkNodeDataDeserializer deserializer, - Set services + Set services, + boolean alwaysUseInitialCriteria ) { super( namespace, @@ -55,7 +56,8 @@ public UnshardedRangerZKHubClient( curatorFramework, criteria, deserializer, - services + services, + alwaysUseInitialCriteria ); } diff --git a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZkServiceDataSource.java b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZkServiceDataSource.java index adff53bc..3694f287 100644 --- a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZkServiceDataSource.java +++ b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinderhub/ZkServiceDataSource.java @@ -18,7 +18,6 @@ import com.flipkart.ranger.core.finderhub.ServiceDataSource; import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.zookeeper.util.PathBuilder; -import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import lombok.extern.slf4j.Slf4j; import lombok.val; diff --git a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/util/PathBuilder.java b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/util/PathBuilder.java index 0686b27d..34bcdd72 100644 --- a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/util/PathBuilder.java +++ b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/util/PathBuilder.java @@ -18,8 +18,6 @@ import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.core.model.ServiceNode; -import lombok.AccessLevel; -import lombok.NoArgsConstructor; import lombok.experimental.UtilityClass; @UtilityClass From 8fcac419c81b55dff654f812a6fc314adcd67816 Mon Sep 17 00:00:00 2001 From: Koushik R Date: Sun, 26 Dec 2021 09:56:11 +0530 Subject: [PATCH 44/69] Fixed a few more lombok usages, brought in var as a replacement for all mutable types in the codebase --- .../ranger/client/AbstractRangerClient.java | 12 ++--- .../client/AbstractRangerHubClient.java | 6 +-- .../ranger/client/utils/CriteriaUtils.java | 3 ++ .../client/AbstractRangerHubClientTest.java | 25 ++++------ .../ranger/client/CriteriaUtilsTest.java | 49 +++++++++++++++++++ .../ranger/core/finder/ServiceFinder.java | 9 ++-- .../ListBasedServiceRegistry.java | 1 + .../MapBasedServiceRegistry.java | 5 +- .../ServiceRegistryUpdater.java | 3 +- .../core/healthcheck/HealthcheckResult.java | 7 +-- .../ranger/core/healthcheck/Healthchecks.java | 10 ++-- .../ServiceHealthAggregator.java | 9 ++-- .../core/healthservice/monitor/Monitors.java | 9 ++-- .../monitor/sample/CountMonitor.java | 3 +- .../monitor/sample/DiskSpaceMonitor.java | 3 +- .../monitor/sample/PingCheckMonitor.java | 17 +++---- .../monitor/sample/RotationStatusMonitor.java | 9 ++-- .../RoundRobinServiceNodeSelectorTest.java | 10 ++-- .../http/ShardedRangerHttpClientTest.java | 14 ++---- .../http/SimpleRangerHttpClientTest.java | 14 ++---- .../http/UnshardedRangerHttpClientTest.java | 12 ++--- .../server/bundle/RangerServerBundleTest.java | 25 ++++------ .../ranger/common/server/ShardInfoTest.java | 11 ++--- .../client/zk/ShardedZKRangerClientTest.java | 10 +--- .../client/zk/SimpleRangerZKClientTest.java | 12 ++--- .../zk/UnshardedZKRangerClientTest.java | 10 +--- .../ZkSimpleShardedServiceFinderBuilder.java | 3 +- ...ZkSimpleUnshardedServiceFinderBuilder.java | 3 +- .../serviceprovider/ZkNodeDataSink.java | 2 +- .../ServiceProviderIntegrationTest.java | 5 +- .../monitor/RollingWindowHealthQueueTest.java | 29 +++++------ .../monitor/sample/DiskSpaceMonitorTest.java | 2 +- .../monitor/sample/PingCheckMonitorTest.java | 11 ++--- .../sample/RotationStatusMonitorTest.java | 9 ++-- .../model/ServiceProviderExtCuratorTest.java | 10 ++-- .../zookeeper/model/ServiceProviderTest.java | 3 +- .../model/SimpleServiceProviderTest.java | 4 +- .../BaseServiceProviderBuilderTest.java | 4 +- 38 files changed, 185 insertions(+), 198 deletions(-) create mode 100644 ranger-client/src/test/java/com/flipkart/ranger/client/CriteriaUtilsTest.java diff --git a/ranger-client/src/main/java/com/flipkart/ranger/client/AbstractRangerClient.java b/ranger-client/src/main/java/com/flipkart/ranger/client/AbstractRangerClient.java index 58e836a0..5cc792c3 100644 --- a/ranger-client/src/main/java/com/flipkart/ranger/client/AbstractRangerClient.java +++ b/ranger-client/src/main/java/com/flipkart/ranger/client/AbstractRangerClient.java @@ -39,25 +39,21 @@ protected AbstractRangerClient(Predicate initialCriteria, boolean alwaysUseIn @Override public Optional> getNode() { - val finder = getServiceFinder(); - return finder.get(initialCriteria); + return getServiceFinder().get(initialCriteria); } @Override public Optional> getNode(Predicate criteria) { - val finder = getServiceFinder(); - return finder.get(CriteriaUtils.getCriteria(alwaysUseInitialCriteria, initialCriteria, criteria)); + return getServiceFinder().get(CriteriaUtils.getCriteria(alwaysUseInitialCriteria, initialCriteria, criteria)); } @Override public List> getAllNodes() { - val finder = getServiceFinder(); - return finder.getAll(initialCriteria); + return getServiceFinder().getAll(initialCriteria); } @Override public List> getAllNodes(Predicate criteria) { - val finder = getServiceFinder(); - return finder.getAll(CriteriaUtils.getCriteria(alwaysUseInitialCriteria, initialCriteria, criteria)); + return getServiceFinder().getAll(CriteriaUtils.getCriteria(alwaysUseInitialCriteria, initialCriteria, criteria)); } } diff --git a/ranger-client/src/main/java/com/flipkart/ranger/client/AbstractRangerHubClient.java b/ranger-client/src/main/java/com/flipkart/ranger/client/AbstractRangerHubClient.java index 1a71eebf..0330685a 100644 --- a/ranger-client/src/main/java/com/flipkart/ranger/client/AbstractRangerHubClient.java +++ b/ranger-client/src/main/java/com/flipkart/ranger/client/AbstractRangerHubClient.java @@ -99,8 +99,7 @@ public Optional> getNode( final Service service, final Predicate criteria ){ - val optionalFinder = this.getHub().finder(service); - return optionalFinder.flatMap(trServiceFinder -> trServiceFinder.get( + return this.getHub().finder(service).flatMap(trServiceFinder -> trServiceFinder.get( CriteriaUtils.getCriteria(alwaysUseInitialCriteria, initialCriteria, criteria)) ); } @@ -109,8 +108,7 @@ public List> getAllNodes( final Service service, final Predicate criteria ){ - val optionalFinder = this.getHub().finder(service); - return optionalFinder.map(trServiceFinder -> trServiceFinder.getAll( + return this.getHub().finder(service).map(trServiceFinder -> trServiceFinder.getAll( CriteriaUtils.getCriteria(alwaysUseInitialCriteria, initialCriteria, criteria)) ).orElse(Collections.emptyList()); } diff --git a/ranger-client/src/main/java/com/flipkart/ranger/client/utils/CriteriaUtils.java b/ranger-client/src/main/java/com/flipkart/ranger/client/utils/CriteriaUtils.java index eae89c7f..af04b862 100644 --- a/ranger-client/src/main/java/com/flipkart/ranger/client/utils/CriteriaUtils.java +++ b/ranger-client/src/main/java/com/flipkart/ranger/client/utils/CriteriaUtils.java @@ -27,6 +27,9 @@ public class CriteriaUtils { The idea is to check if this has to be merged with the initialCriteria or not. So the default return is always argCriteria, in case the initial criteria is null! + + So when useInitialCriteria is true, the initial criteria defined will always be used, no matter what, + it is a predicate and. (Not to be confused with or) */ public static Predicate getCriteria( boolean useInitialCriteria, diff --git a/ranger-client/src/test/java/com/flipkart/ranger/client/AbstractRangerHubClientTest.java b/ranger-client/src/test/java/com/flipkart/ranger/client/AbstractRangerHubClientTest.java index 30ec5bac..f359a8d9 100644 --- a/ranger-client/src/test/java/com/flipkart/ranger/client/AbstractRangerHubClientTest.java +++ b/ranger-client/src/test/java/com/flipkart/ranger/client/AbstractRangerHubClientTest.java @@ -23,6 +23,7 @@ import com.flipkart.ranger.core.utils.TestUtils; import lombok.extern.slf4j.Slf4j; import lombok.val; +import lombok.var; import org.junit.Assert; import org.junit.Test; @@ -37,23 +38,15 @@ public class AbstractRangerHubClientTest { public void testAbstractHubClient() { val testAbstractHub = RangerHubTestUtils.getTestHub(); testAbstractHub.start(); - TestUtils.sleepForSeconds(3); - Optional> node = testAbstractHub.getNode(service); - Assert.assertTrue(node.isPresent()); - Assert.assertTrue(node.get().getHost().equalsIgnoreCase("localhost")); - Assert.assertEquals(9200, node.get().getPort()); - Assert.assertEquals(1, node.get().getNodeData().getShardId()); - - node = testAbstractHub.getNode(RangerTestUtils.getService("test", "test")); - Assert.assertFalse(node.isPresent()); - - node = testAbstractHub.getNode(service, nodeData -> nodeData.getShardId() == 2); - Assert.assertFalse(node.isPresent()); - - node = testAbstractHub.getNode(RangerTestUtils.getService("test", "test"), nodeData -> nodeData.getShardId() == 1); - Assert.assertFalse(node.isPresent()); - + var node = testAbstractHub.getNode(service).orElse(null); + Assert.assertNotNull(node); + Assert.assertTrue(node.getHost().equalsIgnoreCase("localhost")); + Assert.assertEquals(9200, node.getPort()); + Assert.assertEquals(1, node.getNodeData().getShardId()); + Assert.assertFalse(testAbstractHub.getNode(RangerTestUtils.getService("test", "test")).isPresent()); + Assert.assertFalse(testAbstractHub.getNode(service, nodeData -> nodeData.getShardId() == 2).isPresent()); + Assert.assertFalse(testAbstractHub.getNode(RangerTestUtils.getService("test", "test"), nodeData -> nodeData.getShardId() == 1).isPresent()); testAbstractHub.stop(); } } diff --git a/ranger-client/src/test/java/com/flipkart/ranger/client/CriteriaUtilsTest.java b/ranger-client/src/test/java/com/flipkart/ranger/client/CriteriaUtilsTest.java new file mode 100644 index 00000000..755e8112 --- /dev/null +++ b/ranger-client/src/test/java/com/flipkart/ranger/client/CriteriaUtilsTest.java @@ -0,0 +1,49 @@ +/* + * Copyright 2015 Flipkart Internet Pvt. Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.flipkart.ranger.client; + +import com.flipkart.ranger.client.utils.CriteriaUtils; +import com.flipkart.ranger.core.units.TestNodeData; +import lombok.val; +import lombok.var; +import org.junit.Assert; +import org.junit.Test; + +import java.util.function.Predicate; + +public class CriteriaUtilsTest { + + private Predicate getCriteria(int shardId){ + return new Predicate() { + @Override + public boolean test(TestNodeData testNodeData) { + return testNodeData.getShardId() == shardId; + } + }; + } + + @Test + public void testGetCriteria(){ + val initialCriteria = getCriteria(1); + val argCriteria = getCriteria(2); + var mergedCriteria = CriteriaUtils.getCriteria(true, initialCriteria, argCriteria); + Assert.assertFalse(mergedCriteria.test(TestNodeData.builder().shardId(1).build())); + Assert.assertFalse(mergedCriteria.test(TestNodeData.builder().shardId(2).build())); + mergedCriteria = CriteriaUtils.getCriteria(false, initialCriteria, argCriteria); + Assert.assertFalse(mergedCriteria.test(TestNodeData.builder().shardId(1).build())); + Assert.assertTrue(mergedCriteria.test(TestNodeData.builder().shardId(2).build())); + } +} diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/ServiceFinder.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/ServiceFinder.java index 78ac52ce..553f741b 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/ServiceFinder.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/ServiceFinder.java @@ -23,7 +23,9 @@ import com.flipkart.ranger.core.signals.ExternalTriggeredSignal; import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import lombok.var; +import javax.swing.text.html.Option; import java.util.Collections; import java.util.List; import java.util.Optional; @@ -50,11 +52,8 @@ protected ServiceFinder( } public Optional> get(Predicate criteria) { - List> nodes = shardSelector.nodes(criteria, serviceRegistry); - if (null == nodes || nodes.isEmpty()) { - return Optional.empty(); - } - return Optional.of(nodeSelector.select(nodes)); + var nodes = shardSelector.nodes(criteria, serviceRegistry); + return null == nodes || nodes.isEmpty() ? Optional.empty() : Optional.of(nodeSelector.select(nodes)); } public List> getAll(Predicate criteria) { diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/serviceregistry/ListBasedServiceRegistry.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/serviceregistry/ListBasedServiceRegistry.java index 66e2123f..5a02014b 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/serviceregistry/ListBasedServiceRegistry.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/serviceregistry/ListBasedServiceRegistry.java @@ -21,6 +21,7 @@ import com.google.common.collect.ImmutableList; import lombok.val; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.concurrent.atomic.AtomicReference; diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/serviceregistry/MapBasedServiceRegistry.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/serviceregistry/MapBasedServiceRegistry.java index 874040f7..2eb84cbf 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/serviceregistry/MapBasedServiceRegistry.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/serviceregistry/MapBasedServiceRegistry.java @@ -49,10 +49,7 @@ public List> nodeList() { @Override public void updateNodes(List> nodes) { ListMultimap> serviceNodes = ArrayListMultimap.create(); - for (ServiceNode serviceNode : nodes) { - serviceNodes.put(serviceNode.getNodeData(), serviceNode); - } + nodes.forEach(serviceNode -> serviceNodes.put(serviceNode.getNodeData(), serviceNode)); this.nodes.set(ImmutableListMultimap.copyOf(serviceNodes)); } - } diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/serviceregistry/ServiceRegistryUpdater.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/serviceregistry/ServiceRegistryUpdater.java index a2147335..726df353 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/serviceregistry/ServiceRegistryUpdater.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/serviceregistry/ServiceRegistryUpdater.java @@ -25,6 +25,7 @@ import com.google.common.base.Stopwatch; import lombok.extern.slf4j.Slf4j; import lombok.val; +import lombok.var; import java.util.List; import java.util.concurrent.ExecutorService; @@ -68,7 +69,7 @@ public void start() { log.info("Started updater for [{}]. Triggering initial update.", serviceName); checkForUpdate(null); log.info("Waiting for initial update to complete for: {}", serviceName); - Stopwatch stopwatch = Stopwatch.createStarted(); + var stopwatch = Stopwatch.createStarted(); try { RetryerBuilder.newBuilder() .retryIfResult(r -> null == r || !r) diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/healthcheck/HealthcheckResult.java b/ranger-core/src/main/java/com/flipkart/ranger/core/healthcheck/HealthcheckResult.java index e7b38167..65d0cfe4 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/healthcheck/HealthcheckResult.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/healthcheck/HealthcheckResult.java @@ -17,13 +17,14 @@ import lombok.Builder; import lombok.Data; +import lombok.Value; /** * */ -@Data +@Value @Builder public class HealthcheckResult { - private final HealthcheckStatus status; - private final long updatedTime; + HealthcheckStatus status; + long updatedTime; } diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/healthcheck/Healthchecks.java b/ranger-core/src/main/java/com/flipkart/ranger/core/healthcheck/Healthchecks.java index a07cc271..5862ca74 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/healthcheck/Healthchecks.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/healthcheck/Healthchecks.java @@ -15,6 +15,8 @@ */ package com.flipkart.ranger.core.healthcheck; +import lombok.val; + import java.io.File; /** @@ -27,12 +29,8 @@ private Healthchecks() { public static Healthcheck fileExistanceCheck(final String filePath) { return () -> { - File file = new File(filePath); - if (file.exists()) { - return HealthcheckStatus.healthy; - } else { - return HealthcheckStatus.unhealthy; - } + val file = new File(filePath); + return file.exists() ? HealthcheckStatus.healthy : HealthcheckStatus.unhealthy; }; } diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/ServiceHealthAggregator.java b/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/ServiceHealthAggregator.java index 4f7b4aa8..30eae894 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/ServiceHealthAggregator.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/ServiceHealthAggregator.java @@ -105,7 +105,7 @@ public void start() { log.info("Service aggregator is already running"); return; } - ScheduledExecutorService scheduledExecutorService = Executors.newScheduledThreadPool(isolatedHealthMonitorList.size()); + val scheduledExecutorService = Executors.newScheduledThreadPool(isolatedHealthMonitorList.size()); scheduledFutureList = Lists.newArrayListWithCapacity(isolatedHealthMonitorList.size()); isolatedHealthMonitorList.stream().map(isolatedHealthMonitor -> scheduledExecutorService.scheduleWithFixedDelay( isolatedHealthMonitor, @@ -127,9 +127,7 @@ public synchronized void stop() { log.error("Service aggregator is currently not running, cannot stop.."); return; } - for (ScheduledFuture scheduledFuture : scheduledFutureList) { - scheduledFuture.cancel(true); - } + scheduledFutureList.forEach(scheduledFuture -> scheduledFuture.cancel(true)); running.set(false); } @@ -145,8 +143,7 @@ public HealthcheckStatus getServiceHealth() { "Please start the aggregator before trying to get health");*/ } healthcheckStatus.set(HealthcheckStatus.healthy); - Date currentTime = new Date(); - + val currentTime = new Date(); /* check health status of isolated monitors */ val hasUnhealthyMonitor = isolatedHealthMonitorList.stream() .filter(isolatedHealthMonitor -> !isolatedHealthMonitor.isDisabled()) diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/Monitors.java b/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/Monitors.java index 769ae8cd..0a64974d 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/Monitors.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/Monitors.java @@ -16,6 +16,7 @@ package com.flipkart.ranger.core.healthservice.monitor; import com.flipkart.ranger.core.healthcheck.HealthcheckStatus; +import lombok.val; import java.io.File; @@ -32,12 +33,8 @@ public static Monitor fileExistanceCheckMonitor(final String return new Monitor() { @Override public HealthcheckStatus monitor() { - File file = new File(filePath); - if (file.exists()) { - return HealthcheckStatus.healthy; - } else { - return HealthcheckStatus.unhealthy; - } + val file = new File(filePath); + return file.exists() ? HealthcheckStatus.healthy : HealthcheckStatus.unhealthy; } @Override diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/sample/CountMonitor.java b/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/sample/CountMonitor.java index de3e1551..e297a0bd 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/sample/CountMonitor.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/sample/CountMonitor.java @@ -18,6 +18,7 @@ import com.flipkart.ranger.core.healthcheck.HealthcheckStatus; import com.flipkart.ranger.core.healthservice.TimeEntity; import com.flipkart.ranger.core.healthservice.monitor.IsolatedHealthMonitor; +import lombok.val; /** * A monitor that can be used as a counting monitor to check if any countable entity breaches a threashhold @@ -56,7 +57,7 @@ public CountMonitor(String name, CheckSign checkSign, Integer threshold, TimeEnt @Override public HealthcheckStatus monitor() { - final long count = getCount().longValue(); + val count = getCount().longValue(); switch (checkSign) { case LESSER_THAN: if (count < threshold) { diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/sample/DiskSpaceMonitor.java b/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/sample/DiskSpaceMonitor.java index 10cca535..959c87dc 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/sample/DiskSpaceMonitor.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/sample/DiskSpaceMonitor.java @@ -16,6 +16,7 @@ package com.flipkart.ranger.core.healthservice.monitor.sample; import com.flipkart.ranger.core.healthservice.TimeEntity; +import lombok.val; import java.io.File; @@ -49,7 +50,7 @@ public DiskSpaceMonitor(String partition, String name, Integer threshhold, TimeE @Override public Number getCount() { - File file = new File(partition); + val file = new File(partition); return file.getFreeSpace(); } } diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/sample/PingCheckMonitor.java b/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/sample/PingCheckMonitor.java index b9385320..db754bc3 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/sample/PingCheckMonitor.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/sample/PingCheckMonitor.java @@ -20,6 +20,7 @@ import com.flipkart.ranger.core.healthservice.monitor.IsolatedHealthMonitor; import com.flipkart.ranger.core.healthservice.monitor.RollingWindowHealthQueue; import lombok.extern.slf4j.Slf4j; +import lombok.val; import org.apache.http.HttpHost; import org.apache.http.HttpRequest; import org.apache.http.HttpStatus; @@ -70,7 +71,7 @@ public PingCheckMonitor( this.port = port; this.rollingWindowHealthQueue = new RollingWindowHealthQueue(pingWindowSize, maxFailures); this.executorService = Executors.newSingleThreadExecutor(); - PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager(); + val connectionManager = new PoolingHttpClientConnectionManager(); connectionManager.setMaxPerRoute(new HttpRoute(new HttpHost(host, port)), 2); this.httpClient = HttpClients.custom() .setConnectionManager(connectionManager) @@ -80,15 +81,13 @@ public PingCheckMonitor( @Override public HealthcheckStatus monitor() { log.debug("Running ping monitor :{} with HttpRequest:{} on host:{} port:{}", name, httpRequest, host, port); - Future futurePingResponse = executorService.submit(this::healthPing); + val futurePingResponse = executorService.submit(this::healthPing); try { - final Boolean pingSuccessful = futurePingResponse.get(pingTimeoutInMilliseconds, TimeUnit.MILLISECONDS); - if (!pingSuccessful) { - return getRollingWindowHealthcheckStatus(HealthcheckStatus.unhealthy); - } else { - return getRollingWindowHealthcheckStatus(HealthcheckStatus.healthy); - } + val pingSuccessful = futurePingResponse.get(pingTimeoutInMilliseconds, TimeUnit.MILLISECONDS); + return getRollingWindowHealthcheckStatus( + pingSuccessful ? HealthcheckStatus.healthy : HealthcheckStatus.unhealthy + ); } catch (InterruptedException | ExecutionException | TimeoutException e) { log.error("Ping monitor failed:{} with HttpRequest:{} on host:{} port:{}", name, httpRequest, host, port); log.error("Error running ping monitor: ", e); @@ -109,7 +108,7 @@ private HealthcheckStatus getRollingWindowHealthcheckStatus(HealthcheckStatus he private boolean healthPing() { try { log.debug("executing http HttpRequest: {}, host:{}, port:{}", httpRequest, host, port); - CloseableHttpResponse response = httpClient.execute(new HttpHost(host, port), httpRequest); + val response = httpClient.execute(new HttpHost(host, port), httpRequest); if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) { log.error("Error while executing Ping Test. HttpRequest: {}, host:{}, port:{}, reason:{}", httpRequest, host, port, response.getStatusLine().getReasonPhrase()); response.close(); diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/sample/RotationStatusMonitor.java b/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/sample/RotationStatusMonitor.java index b0bc1c48..f293502b 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/sample/RotationStatusMonitor.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/sample/RotationStatusMonitor.java @@ -19,6 +19,7 @@ import com.flipkart.ranger.core.healthcheck.HealthcheckStatus; import com.flipkart.ranger.core.healthservice.TimeEntity; import com.flipkart.ranger.core.healthservice.monitor.IsolatedHealthMonitor; +import lombok.val; import java.io.File; import java.util.concurrent.TimeUnit; @@ -73,11 +74,7 @@ public RotationStatusMonitor(TimeEntity timeEntity, long stalenessAllowedInMilli */ @Override public HealthcheckStatus monitor() { - File file = new File(filePath); - if (file.exists()) { - return HealthcheckStatus.healthy; - } else { - return HealthcheckStatus.unhealthy; - } + val file = new File(filePath); + return file.exists() ? HealthcheckStatus.healthy : HealthcheckStatus.unhealthy; } } diff --git a/ranger-core/src/test/java/com/flipkart/ranger/core/finder/nodeselector/RoundRobinServiceNodeSelectorTest.java b/ranger-core/src/test/java/com/flipkart/ranger/core/finder/nodeselector/RoundRobinServiceNodeSelectorTest.java index 177048ea..15960ae3 100644 --- a/ranger-core/src/test/java/com/flipkart/ranger/core/finder/nodeselector/RoundRobinServiceNodeSelectorTest.java +++ b/ranger-core/src/test/java/com/flipkart/ranger/core/finder/nodeselector/RoundRobinServiceNodeSelectorTest.java @@ -18,6 +18,7 @@ import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.units.TestNodeData; import lombok.val; +import lombok.var; import org.junit.Assert; import org.junit.Test; @@ -31,11 +32,8 @@ public void testRandomNodeSelector(){ serviceNodes.add(new ServiceNode<>("localhost-1", 9000, TestNodeData.builder().shardId(1).build())); serviceNodes.add(new ServiceNode<>("localhost-2", 9001, TestNodeData.builder().shardId(2).build())); serviceNodes.add(new ServiceNode<>("localhost-3", 9002, TestNodeData.builder().shardId(3).build())); - ServiceNode select = roundRobinSelector.select(serviceNodes); - Assert.assertEquals("localhost-2", select.getHost()); - select = roundRobinSelector.select(serviceNodes); - Assert.assertEquals("localhost-3", select.getHost()); - select = roundRobinSelector.select(serviceNodes); - Assert.assertEquals("localhost-1", select.getHost()); + Assert.assertEquals("localhost-2", roundRobinSelector.select(serviceNodes).getHost()); + Assert.assertEquals("localhost-3", roundRobinSelector.select(serviceNodes).getHost()); + Assert.assertEquals("localhost-1", roundRobinSelector.select(serviceNodes).getHost()); } } diff --git a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/ShardedRangerHttpClientTest.java b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/ShardedRangerHttpClientTest.java index 31341c08..3e1d6499 100644 --- a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/ShardedRangerHttpClientTest.java +++ b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/ShardedRangerHttpClientTest.java @@ -20,6 +20,7 @@ import com.flipkart.ranger.core.utils.RangerTestUtils; import com.flipkart.ranger.core.utils.TestUtils; import lombok.val; +import lombok.var; import org.junit.Assert; import org.junit.Test; @@ -37,17 +38,10 @@ public void testShardedHttpHubClient(){ client.start(); TestUtils.sleepForSeconds(6); - val service = RangerTestUtils.getService("test-n", "test-s"); - ServiceNode node = client.getNode(service).orElse(null); - Assert.assertNotNull(node); - - node = client.getNode(service, nodeData -> nodeData.getShardId() == 1).orElse(null); - Assert.assertNotNull(node); - - node = client.getNode(service, nodeData -> nodeData.getShardId() == 2).orElse(null); - Assert.assertNull(node); - + Assert.assertNotNull(client.getNode(service).orElse(null)); + Assert.assertNotNull(client.getNode(service, nodeData -> nodeData.getShardId() == 1).orElse(null)); + Assert.assertNull(client.getNode(service, nodeData -> nodeData.getShardId() == 2).orElse(null)); client.stop(); } } diff --git a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/SimpleRangerHttpClientTest.java b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/SimpleRangerHttpClientTest.java index 1486a18b..5fd9848c 100644 --- a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/SimpleRangerHttpClientTest.java +++ b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/SimpleRangerHttpClientTest.java @@ -35,18 +35,10 @@ public void testSimpleHttpRangerClient(){ .nodeRefreshIntervalMs(5000) .build(); client.start(); - TestUtils.sleepForSeconds(6); - - ServiceNode node = client.getNode().orElse(null); - Assert.assertNotNull(node); - - node = client.getNode(nodeData -> nodeData.getShardId() == 1).orElse(null); - Assert.assertNotNull(node); - - node = client.getNode(nodeData -> nodeData.getShardId() == 2).orElse(null); - Assert.assertNull(node); - + Assert.assertNotNull(client.getNode().orElse(null)); + Assert.assertNotNull(client.getNode(nodeData -> nodeData.getShardId() == 1).orElse(null)); + Assert.assertNull(client.getNode(nodeData -> nodeData.getShardId() == 2).orElse(null)); client.stop(); } } diff --git a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/UnshardedRangerHttpClientTest.java b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/UnshardedRangerHttpClientTest.java index fdc14a18..c89324e9 100644 --- a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/UnshardedRangerHttpClientTest.java +++ b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/UnshardedRangerHttpClientTest.java @@ -39,15 +39,9 @@ public void testUnshardedRangerHubClient(){ TestUtils.sleepForSeconds(6); val service = RangerTestUtils.getService("test-n", "test-s"); - ServiceNode node = client.getNode(service).orElse(null); - Assert.assertNotNull(node); - - node = client.getNode(service, nodeData -> nodeData.getShardId() == 1).orElse(null); - Assert.assertNotNull(node); - - node = client.getNode(service, nodeData -> nodeData.getShardId() == 2).orElse(null); - Assert.assertNull(node); - + Assert.assertNotNull(client.getNode(service).orElse(null)); + Assert.assertNotNull(client.getNode(service, nodeData -> nodeData.getShardId() == 1).orElse(null)); + Assert.assertNull(client.getNode(service, nodeData -> nodeData.getShardId() == 2).orElse(null)); client.stop(); } } diff --git a/ranger-server-bundle/src/test/java/com/flipkart/ranger/zk/server/bundle/RangerServerBundleTest.java b/ranger-server-bundle/src/test/java/com/flipkart/ranger/zk/server/bundle/RangerServerBundleTest.java index e6974ee2..424a6cb9 100644 --- a/ranger-server-bundle/src/test/java/com/flipkart/ranger/zk/server/bundle/RangerServerBundleTest.java +++ b/ranger-server-bundle/src/test/java/com/flipkart/ranger/zk/server/bundle/RangerServerBundleTest.java @@ -33,6 +33,7 @@ import io.dropwizard.setup.Bootstrap; import io.dropwizard.setup.Environment; import lombok.val; +import lombok.var; import org.eclipse.jetty.util.component.LifeCycle; import org.junit.After; import org.junit.Assert; @@ -106,7 +107,7 @@ public void setup() throws Exception { rangerServerBundle.initialize(bootstrap); rangerServerBundle.run(configuration, environment); - for (LifeCycle lifeCycle : lifecycleEnvironment.getManagedObjects()){ + for (val lifeCycle : lifecycleEnvironment.getManagedObjects()){ lifeCycle.start(); } } @@ -116,20 +117,14 @@ public void setup() throws Exception { public void testRangerBundle(){ TestUtils.sleepForSeconds(3); val hub = rangerServerBundle.getHubs().get(0); - Optional> node = hub.getNode(service); - Assert.assertTrue(node.isPresent()); - Assert.assertTrue(node.get().getHost().equalsIgnoreCase("localhost")); - Assert.assertEquals(9200, node.get().getPort()); - Assert.assertEquals(1, node.get().getNodeData().getShardId()); - - node = hub.getNode(RangerTestUtils.getService("test", "test")); - Assert.assertFalse(node.isPresent()); - - node = hub.getNode(service, nodeData -> nodeData.getShardId() == 2); - Assert.assertFalse(node.isPresent()); - - node = hub.getNode(RangerTestUtils.getService("test", "test"), nodeData -> nodeData.getShardId() == 1); - Assert.assertFalse(node.isPresent()); + var node = hub.getNode(service).orElse(null); + Assert.assertNotNull(node); + Assert.assertTrue(node.getHost().equalsIgnoreCase("localhost")); + Assert.assertEquals(9200, node.getPort()); + Assert.assertEquals(1, node.getNodeData().getShardId()); + Assert.assertNull(hub.getNode(RangerTestUtils.getService("test", "test")).orElse(null)); + Assert.assertNull(hub.getNode(service, nodeData -> nodeData.getShardId() == 2).orElse(null)); + Assert.assertNull(hub.getNode(RangerTestUtils.getService("test", "test"), nodeData -> nodeData.getShardId() == 1).orElse(null)); } @After diff --git a/ranger-server-common/src/test/java/com/flipkart/ranger/common/server/ShardInfoTest.java b/ranger-server-common/src/test/java/com/flipkart/ranger/common/server/ShardInfoTest.java index 98020fc9..15c7e24e 100644 --- a/ranger-server-common/src/test/java/com/flipkart/ranger/common/server/ShardInfoTest.java +++ b/ranger-server-common/src/test/java/com/flipkart/ranger/common/server/ShardInfoTest.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import lombok.SneakyThrows; +import lombok.val; import org.junit.Assert; import org.junit.Test; @@ -16,7 +17,7 @@ public class ShardInfoTest { private static final ObjectMapper mapper = new ObjectMapper(); private String getResource(String path) { - final InputStream data = ShardInfoTest.class.getClassLoader().getResourceAsStream(path); + val data = ShardInfoTest.class.getClassLoader().getResourceAsStream(path); return new BufferedReader( new InputStreamReader(data)) .lines() @@ -25,18 +26,16 @@ private String getResource(String path) { @SneakyThrows private T getResource(String path, Class klass) { - final String data = getResource(path); + val data = getResource(path); return mapper.readValue(data, klass); } @Test public void testShardInfo(){ - ShardInfo shardInfo1 = getResource("fixtures/env1.json", ShardInfo.class); - ShardInfo shardInfo2 = getResource("fixtures/env2.json", ShardInfo.class); - + val shardInfo1 = getResource("fixtures/env1.json", ShardInfo.class); + val shardInfo2 = getResource("fixtures/env2.json", ShardInfo.class); Assert.assertNotNull(shardInfo1); Assert.assertNotNull(shardInfo2); - Assert.assertNotEquals(shardInfo1, shardInfo2); Arrays.asList(shardInfo1, shardInfo2).forEach(shardInfo -> Assert.assertEquals("e", shardInfo.getEnvironment())); Assert.assertEquals("r", shardInfo1.getRegion()); diff --git a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/ShardedZKRangerClientTest.java b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/ShardedZKRangerClientTest.java index ac090323..d7b2287b 100644 --- a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/ShardedZKRangerClientTest.java +++ b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/ShardedZKRangerClientTest.java @@ -42,14 +42,8 @@ public void testShardedHub(){ zkHubClient.start(); TestUtils.sleepForSeconds(6); - - val service = RangerTestUtils.getService("test-n", "s1"); - Optional> node = zkHubClient.getNode(RangerTestUtils.getService("test-n", "s1")); - Assert.assertTrue(node.isPresent()); - - node = zkHubClient.getNode(service, nodeData -> nodeData.getShardId() == 1); - Assert.assertTrue(node.isPresent()); - + Assert.assertNotNull(zkHubClient.getNode(RangerTestUtils.getService("test-n", "s1")).orElse(null)); + Assert.assertNotNull(zkHubClient.getNode(RangerTestUtils.getService("test-n", "s1"), nodeData -> nodeData.getShardId() == 1).orElse(null)); zkHubClient.stop(); } } diff --git a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/SimpleRangerZKClientTest.java b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/SimpleRangerZKClientTest.java index 1be52d0b..8ecd4c1b 100644 --- a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/SimpleRangerZKClientTest.java +++ b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/SimpleRangerZKClientTest.java @@ -36,14 +36,8 @@ public void testBaseClient(){ .mapper(getObjectMapper()) .build(); client.start(); - - Optional> node = client.getNode(); - Assert.assertTrue(node.isPresent()); - - node = client.getNode(c -> c.getShardId() == 1); - Assert.assertTrue(node.isPresent()); - - node = client.getNode(c -> c.getShardId() == 2); - Assert.assertFalse(node.isPresent()); + Assert.assertNotNull( client.getNode().orElse(null)); + Assert.assertNotNull(client.getNode(c -> c.getShardId() == 1).orElse(null)); + Assert.assertNull(client.getNode(c -> c.getShardId() == 2).orElse(null)); } } diff --git a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/UnshardedZKRangerClientTest.java b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/UnshardedZKRangerClientTest.java index abeee7a0..92c7c740 100644 --- a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/UnshardedZKRangerClientTest.java +++ b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/UnshardedZKRangerClientTest.java @@ -39,17 +39,11 @@ public void testShardedHub(){ .mapper(getObjectMapper()) .deserializer(this::read) .build(); - zkHubClient.start(); TestUtils.sleepForSeconds(6); - val service = RangerTestUtils.getService("test-n", "s1"); - Optional> node = zkHubClient.getNode(service); - Assert.assertTrue(node.isPresent()); - - node = zkHubClient.getNode(service, nodeData -> nodeData.getShardId() == 1); - Assert.assertTrue(node.isPresent()); - + Assert.assertNotNull(zkHubClient.getNode(service).orElse(null)); + Assert.assertNotNull(zkHubClient.getNode(service, nodeData -> nodeData.getShardId() == 1).orElse(null)); zkHubClient.stop(); } } diff --git a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinder/ZkSimpleShardedServiceFinderBuilder.java b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinder/ZkSimpleShardedServiceFinderBuilder.java index 8c15ddd2..ed0da0da 100644 --- a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinder/ZkSimpleShardedServiceFinderBuilder.java +++ b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinder/ZkSimpleShardedServiceFinderBuilder.java @@ -24,6 +24,7 @@ import com.flipkart.ranger.zookeeper.servicefinder.signals.ZkWatcherRegistryUpdateSignal; import com.google.common.base.Preconditions; import lombok.extern.slf4j.Slf4j; +import lombok.val; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.retry.ExponentialBackoffRetry; @@ -51,7 +52,7 @@ public ZkSimpleShardedServiceFinderBuilder withConnectionString(final String @Override public SimpleShardedServiceFinder build() { - boolean curatorProvided = curatorFramework != null; + val curatorProvided = curatorFramework != null; if (!curatorProvided) { Preconditions.checkNotNull(connectionString); curatorFramework = CuratorFrameworkFactory.builder() diff --git a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinder/ZkSimpleUnshardedServiceFinderBuilder.java b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinder/ZkSimpleUnshardedServiceFinderBuilder.java index e0e92226..db5c6c41 100644 --- a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinder/ZkSimpleUnshardedServiceFinderBuilder.java +++ b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/servicefinder/ZkSimpleUnshardedServiceFinderBuilder.java @@ -24,6 +24,7 @@ import com.flipkart.ranger.zookeeper.servicefinder.signals.ZkWatcherRegistryUpdateSignal; import com.google.common.base.Preconditions; import lombok.extern.slf4j.Slf4j; +import lombok.val; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.retry.ExponentialBackoffRetry; @@ -52,7 +53,7 @@ public ZkSimpleUnshardedServiceFinderBuilder withConnectionString(final Strin @Override public SimpleUnshardedServiceFinder build() { - boolean curatorProvided = curatorFramework != null; + val curatorProvided = curatorFramework != null; if (!curatorProvided) { Preconditions.checkNotNull(connectionString); curatorFramework = CuratorFrameworkFactory.builder() diff --git a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/serviceprovider/ZkNodeDataSink.java b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/serviceprovider/ZkNodeDataSink.java index 83ad3ae5..53ac96d2 100644 --- a/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/serviceprovider/ZkNodeDataSink.java +++ b/ranger-zookeeper/src/main/java/com/flipkart/ranger/zookeeper/serviceprovider/ZkNodeDataSink.java @@ -49,7 +49,7 @@ public void updateState(S serializer, ServiceNode serviceNode) { return; } Preconditions.checkNotNull(serializer, "Serializer has not been set for node data"); - final String path = PathBuilder.instancePath(service, serviceNode); + val path = PathBuilder.instancePath(service, serviceNode); try { if (null == curatorFramework.checkExists().forPath(path)) { log.info("No node exists for path: {}. Will create now.", path); diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/ServiceProviderIntegrationTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/ServiceProviderIntegrationTest.java index 5e978957..5ec6bdae 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/ServiceProviderIntegrationTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/ServiceProviderIntegrationTest.java @@ -29,6 +29,7 @@ import com.flipkart.ranger.zookeeper.ServiceFinderBuilders; import com.flipkart.ranger.zookeeper.ServiceProviderBuilders; import lombok.val; +import lombok.var; import org.apache.curator.test.TestingCluster; import org.junit.After; import org.junit.Assert; @@ -104,11 +105,11 @@ public void stopTestCluster() throws Exception { public void testBasicDiscovery() throws Exception { /* clean slate */ - boolean delete = file.delete(); + var delete = file.delete(); delete = anotherFile.delete(); /* with file existing, 3 nodes should be healthy */ - boolean filecreate = file.createNewFile(); + var filecreate = file.createNewFile(); System.out.println("created file"); TestUtils.sleepForSeconds(8); List> all = serviceFinder.getAll(null); diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/monitor/RollingWindowHealthQueueTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/monitor/RollingWindowHealthQueueTest.java index 34b58db5..7fb256db 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/monitor/RollingWindowHealthQueueTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/monitor/RollingWindowHealthQueueTest.java @@ -17,14 +17,15 @@ import com.flipkart.ranger.core.healthcheck.HealthcheckStatus; import com.flipkart.ranger.core.healthservice.monitor.RollingWindowHealthQueue; +import lombok.val; import org.junit.Assert; import org.junit.Test; public class RollingWindowHealthQueueTest { @Test - public void testCheckInRollingWindow1() throws Exception { - RollingWindowHealthQueue rollingWindowHealthQueue = new RollingWindowHealthQueue(5, 3); + public void testCheckInRollingWindow1() { + val rollingWindowHealthQueue = new RollingWindowHealthQueue(5, 3); Assert.assertTrue(rollingWindowHealthQueue.checkInRollingWindow(HealthcheckStatus.healthy)); Assert.assertTrue(rollingWindowHealthQueue.checkInRollingWindow(HealthcheckStatus.healthy)); Assert.assertTrue(rollingWindowHealthQueue.checkInRollingWindow(HealthcheckStatus.healthy)); @@ -33,9 +34,9 @@ public void testCheckInRollingWindow1() throws Exception { } @Test - public void testCheckInRollingWindowEdge() throws Exception { + public void testCheckInRollingWindowEdge() { try { - RollingWindowHealthQueue rollingWindowHealthQueue = new RollingWindowHealthQueue(1, 3); + val rollingWindowHealthQueue = new RollingWindowHealthQueue(1, 3); Assert.assertTrue(rollingWindowHealthQueue.checkInRollingWindow(HealthcheckStatus.unhealthy)); } catch (Exception u) { Assert.assertTrue(u instanceof UnsupportedOperationException); @@ -43,8 +44,8 @@ public void testCheckInRollingWindowEdge() throws Exception { } @Test - public void testCheckInRollingWindowEdge2() throws Exception { - RollingWindowHealthQueue rollingWindowHealthQueue = new RollingWindowHealthQueue(3, 3); + public void testCheckInRollingWindowEdge2() { + val rollingWindowHealthQueue = new RollingWindowHealthQueue(3, 3); Assert.assertTrue(rollingWindowHealthQueue.checkInRollingWindow(HealthcheckStatus.unhealthy)); Assert.assertTrue(rollingWindowHealthQueue.checkInRollingWindow(HealthcheckStatus.unhealthy)); Assert.assertFalse(rollingWindowHealthQueue.checkInRollingWindow(HealthcheckStatus.unhealthy)); @@ -55,8 +56,8 @@ public void testCheckInRollingWindowEdge2() throws Exception { } @Test - public void testCheckInRollingWindowEdge3() throws Exception { - RollingWindowHealthQueue rollingWindowHealthQueue = new RollingWindowHealthQueue(5, 1); + public void testCheckInRollingWindowEdge3() { + val rollingWindowHealthQueue = new RollingWindowHealthQueue(5, 1); Assert.assertFalse(rollingWindowHealthQueue.checkInRollingWindow(HealthcheckStatus.unhealthy)); Assert.assertFalse(rollingWindowHealthQueue.checkInRollingWindow(HealthcheckStatus.unhealthy)); Assert.assertFalse(rollingWindowHealthQueue.checkInRollingWindow(HealthcheckStatus.unhealthy)); @@ -70,8 +71,8 @@ public void testCheckInRollingWindowEdge3() throws Exception { @Test - public void testCheckInRollingWindow2() throws Exception { - RollingWindowHealthQueue rollingWindowHealthQueue = new RollingWindowHealthQueue(5, 3); + public void testCheckInRollingWindow2() { + val rollingWindowHealthQueue = new RollingWindowHealthQueue(5, 3); Assert.assertTrue(rollingWindowHealthQueue.checkInRollingWindow(HealthcheckStatus.healthy)); Assert.assertTrue(rollingWindowHealthQueue.checkInRollingWindow(HealthcheckStatus.healthy)); Assert.assertTrue(rollingWindowHealthQueue.checkInRollingWindow(HealthcheckStatus.healthy)); @@ -83,8 +84,8 @@ public void testCheckInRollingWindow2() throws Exception { @Test - public void testCheckInRollingWindow3() throws Exception { - RollingWindowHealthQueue rollingWindowHealthQueue = new RollingWindowHealthQueue(5, 3); + public void testCheckInRollingWindow3() { + val rollingWindowHealthQueue = new RollingWindowHealthQueue(5, 3); Assert.assertTrue(rollingWindowHealthQueue.checkInRollingWindow(HealthcheckStatus.unhealthy)); Assert.assertTrue(rollingWindowHealthQueue.checkInRollingWindow(HealthcheckStatus.healthy)); Assert.assertTrue(rollingWindowHealthQueue.checkInRollingWindow(HealthcheckStatus.healthy)); @@ -96,8 +97,8 @@ public void testCheckInRollingWindow3() throws Exception { } @Test - public void testCheckInRollingWindow4() throws Exception { - RollingWindowHealthQueue rollingWindowHealthQueue = new RollingWindowHealthQueue(5, 3); + public void testCheckInRollingWindow4() { + val rollingWindowHealthQueue = new RollingWindowHealthQueue(5, 3); Assert.assertTrue(rollingWindowHealthQueue.checkInRollingWindow(HealthcheckStatus.unhealthy)); Assert.assertTrue(rollingWindowHealthQueue.checkInRollingWindow(HealthcheckStatus.healthy)); Assert.assertTrue(rollingWindowHealthQueue.checkInRollingWindow(HealthcheckStatus.healthy)); diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/monitor/sample/DiskSpaceMonitorTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/monitor/sample/DiskSpaceMonitorTest.java index be3c5e07..14a6c096 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/monitor/sample/DiskSpaceMonitorTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/monitor/sample/DiskSpaceMonitorTest.java @@ -27,7 +27,7 @@ public class DiskSpaceMonitorTest { - IsolatedHealthMonitor diskSpaceMonitor = new DiskSpaceMonitor("/", 1000, new TimeEntity(2, TimeUnit.SECONDS)); + final IsolatedHealthMonitor diskSpaceMonitor = new DiskSpaceMonitor("/", 1000, new TimeEntity(2, TimeUnit.SECONDS)); @Test public void testGetCount() throws Exception { diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/monitor/sample/PingCheckMonitorTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/monitor/sample/PingCheckMonitorTest.java index d0e0a9cc..0dd36915 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/monitor/sample/PingCheckMonitorTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/monitor/sample/PingCheckMonitorTest.java @@ -18,6 +18,7 @@ import com.flipkart.ranger.core.healthcheck.HealthcheckStatus; import com.flipkart.ranger.core.healthservice.TimeEntity; import com.flipkart.ranger.core.healthservice.monitor.sample.PingCheckMonitor; +import lombok.val; import org.apache.http.client.methods.HttpGet; import org.junit.Assert; import org.junit.Test; @@ -28,25 +29,23 @@ public class PingCheckMonitorTest { @Test public void testMonitor() throws Exception { - final HttpGet httpRequest = new HttpGet("/"); - PingCheckMonitor pingCheckMonitor = new PingCheckMonitor(new TimeEntity(2, TimeUnit.SECONDS), httpRequest, 5000, 5, 3, "google.com", 80); + val httpRequest = new HttpGet("/"); + val pingCheckMonitor = new PingCheckMonitor(new TimeEntity(2, TimeUnit.SECONDS), httpRequest, 5000, 5, 3, "google.com", 80); Assert.assertEquals(HealthcheckStatus.healthy, pingCheckMonitor.monitor()); Assert.assertEquals(HealthcheckStatus.healthy, pingCheckMonitor.monitor()); Assert.assertEquals(HealthcheckStatus.healthy, pingCheckMonitor.monitor()); Assert.assertEquals(HealthcheckStatus.healthy, pingCheckMonitor.monitor()); Assert.assertEquals(HealthcheckStatus.healthy, pingCheckMonitor.monitor()); Assert.assertEquals(HealthcheckStatus.healthy, pingCheckMonitor.monitor()); - } @Test public void testMonitor2() throws Exception { - final HttpGet httpRequest = new HttpGet("/help"); - PingCheckMonitor pingCheckMonitor = new PingCheckMonitor(new TimeEntity(2, TimeUnit.SECONDS), httpRequest, 5000, 5, 3, "google.com", 80); + val httpRequest = new HttpGet("/help"); + val pingCheckMonitor = new PingCheckMonitor(new TimeEntity(2, TimeUnit.SECONDS), httpRequest, 5000, 5, 3, "google.com", 80); Assert.assertEquals(HealthcheckStatus.healthy, pingCheckMonitor.monitor()); Assert.assertEquals(HealthcheckStatus.healthy, pingCheckMonitor.monitor()); Assert.assertEquals(HealthcheckStatus.unhealthy, pingCheckMonitor.monitor()); Assert.assertEquals(HealthcheckStatus.unhealthy, pingCheckMonitor.monitor()); - } } \ No newline at end of file diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/monitor/sample/RotationStatusMonitorTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/monitor/sample/RotationStatusMonitorTest.java index 63c4264a..d3454088 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/monitor/sample/RotationStatusMonitorTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/monitor/sample/RotationStatusMonitorTest.java @@ -17,6 +17,7 @@ import com.flipkart.ranger.core.healthcheck.HealthcheckStatus; import com.flipkart.ranger.core.healthservice.monitor.sample.RotationStatusMonitor; +import lombok.val; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -42,8 +43,8 @@ public void tearDown() throws Exception { @Test public void testMonitor() throws Exception { deleteRotationFile(); - RotationStatusMonitor rotationStatusMonitor = new RotationStatusMonitor("/tmp/rotationFile.html"); - final HealthcheckStatus monitorResult = rotationStatusMonitor.monitor(); + val rotationStatusMonitor = new RotationStatusMonitor("/tmp/rotationFile.html"); + val monitorResult = rotationStatusMonitor.monitor(); Assert.assertEquals(HealthcheckStatus.unhealthy, monitorResult); } @@ -51,8 +52,8 @@ public void testMonitor() throws Exception { public void testMonitor2() throws Exception { deleteRotationFile(); if (file.createNewFile()) { - RotationStatusMonitor rotationStatusMonitor = new RotationStatusMonitor(filePath); - final HealthcheckStatus monitorResult = rotationStatusMonitor.monitor(); + val rotationStatusMonitor = new RotationStatusMonitor(filePath); + val monitorResult = rotationStatusMonitor.monitor(); Assert.assertEquals(HealthcheckStatus.healthy, monitorResult); } else { System.out.println("Unable to create file = " + filePath); diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderExtCuratorTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderExtCuratorTest.java index 7b3ad669..ea92e3b8 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderExtCuratorTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderExtCuratorTest.java @@ -29,6 +29,7 @@ import com.google.common.collect.Lists; import lombok.extern.slf4j.Slf4j; import lombok.val; +import lombok.var; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.retry.ExponentialBackoffRetry; @@ -40,6 +41,7 @@ import java.io.IOException; import java.util.List; +import java.util.stream.LongStream; @Slf4j public class ServiceProviderExtCuratorTest { @@ -101,13 +103,11 @@ public void testBasicDiscovery() { Assert.assertNotNull(node); Assert.assertEquals(1, node.getNodeData().getShardId()); } - long startTime = System.currentTimeMillis(); - for(long i = 0; i <1000000; i++) - { - val node = serviceFinder.get(RangerTestUtils.getCriteria(2)).orElse(null); + val startTime = System.currentTimeMillis(); + LongStream.range(0, 1000000).mapToObj(i -> serviceFinder.get(RangerTestUtils.getCriteria(2)).orElse(null)).forEach(node -> { Assert.assertNotNull(node); Assert.assertEquals(2, node.getNodeData().getShardId()); - } + }); log.info("PERF::RandomSelector::Took (ms):" + (System.currentTimeMillis() - startTime)); { val node = serviceFinder.get(RangerTestUtils.getCriteria(99)).orElse(null); diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderTest.java index 6b62e8f6..53c3a58c 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderTest.java @@ -31,6 +31,7 @@ import com.google.common.collect.Lists; import lombok.extern.slf4j.Slf4j; import lombok.val; +import lombok.var; import org.apache.curator.test.TestingCluster; import org.junit.After; import org.junit.Assert; @@ -94,7 +95,7 @@ public void testBasicDiscovery() { Assert.assertNotNull(node); Assert.assertEquals(1, node.getNodeData().getShardId()); } - long startTime = System.currentTimeMillis(); + val startTime = System.currentTimeMillis(); LongStream.range(0, 1000000).mapToObj(i -> serviceFinder.get(RangerTestUtils.getCriteria(2)).orElse(null)).forEach(node -> { Assert.assertNotNull(node); Assert.assertEquals(2, node.getNodeData().getShardId()); diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/SimpleServiceProviderTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/SimpleServiceProviderTest.java index fe1a5ac7..6e68fe8a 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/SimpleServiceProviderTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/SimpleServiceProviderTest.java @@ -94,8 +94,8 @@ public void testBasicDiscovery() { Assert.assertNotNull(node); System.out.println(node.getHost()); } - Multiset frequency = HashMultiset.create(); - long startTime = System.currentTimeMillis(); + val frequency = HashMultiset.create(); + val startTime = System.currentTimeMillis(); LongStream.range(0, 1000000).mapToObj(i -> serviceFinder.get(null).orElse(null)).forEach(node -> { Assert.assertNotNull(node); frequency.add(node.getHost()); diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/serviceprovider/BaseServiceProviderBuilderTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/serviceprovider/BaseServiceProviderBuilderTest.java index 50bce5a5..bed6bd63 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/serviceprovider/BaseServiceProviderBuilderTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/serviceprovider/BaseServiceProviderBuilderTest.java @@ -53,8 +53,8 @@ public void stopTestCluster() throws Exception { @Test public void testbuilder() { - final String host = "localhost"; - final int port = 9000; + val host = "localhost"; + val port = 9000; Exception exception = null; try { val serviceProvider = ServiceProviderBuilders.unshardedServiceProviderBuilder() From 234c38802f822d07f4423f55aabfd259ba2d7bf5 Mon Sep 17 00:00:00 2001 From: Koushik R Date: Mon, 27 Dec 2021 14:58:42 +0530 Subject: [PATCH 45/69] Removed the explicit constructor in ServiceNode --- .../ranger/client/AbstractRangerClient.java | 1 - .../ranger/client/AbstractRangerHubClient.java | 1 - .../client/AbstractRangerHubClientTest.java | 4 ---- .../stubs/TestSimpleUnshardedServiceFinder.java | 6 +++++- .../ranger/core/finder/ServiceFinder.java | 1 - .../serviceregistry/ListBasedServiceRegistry.java | 1 - .../core/healthcheck/HealthcheckResult.java | 1 - .../healthservice/ServiceHealthAggregator.java | 1 - .../monitor/sample/PingCheckMonitor.java | 1 - .../flipkart/ranger/core/model/ServiceNode.java | 15 ++++++--------- .../BaseServiceProviderBuilder.java | 3 +-- .../RoundRobinServiceNodeSelectorTest.java | 7 +++---- .../ranger/core/utils/RegistryTestUtils.java | 12 ++++++------ .../client/http/BaseRangerHttpClientTest.java | 2 +- .../client/http/ShardedRangerHttpClientTest.java | 2 -- .../client/http/SimpleRangerHttpClientTest.java | 1 - .../http/UnshardedRangerHttpClientTest.java | 1 - .../HttpShardedServiceFinderBuilderTest.java | 2 +- .../HttpShardedServiceProviderBuilderTest.java | 2 +- .../zk/server/bundle/RangerServerBundleTest.java | 2 -- .../ranger/common/server/ShardInfoTest.java | 1 - .../client/zk/ShardedZKRangerClientTest.java | 3 --- .../client/zk/SimpleRangerZKClientTest.java | 3 --- .../client/zk/UnshardedZKRangerClientTest.java | 3 --- .../model/ServiceProviderExtCuratorTest.java | 1 - .../zookeeper/model/ServiceProviderTest.java | 1 - .../model/SimpleServiceProviderTest.java | 1 - 27 files changed, 24 insertions(+), 55 deletions(-) diff --git a/ranger-client/src/main/java/com/flipkart/ranger/client/AbstractRangerClient.java b/ranger-client/src/main/java/com/flipkart/ranger/client/AbstractRangerClient.java index 5cc792c3..16845096 100644 --- a/ranger-client/src/main/java/com/flipkart/ranger/client/AbstractRangerClient.java +++ b/ranger-client/src/main/java/com/flipkart/ranger/client/AbstractRangerClient.java @@ -19,7 +19,6 @@ import com.flipkart.ranger.core.finder.ServiceFinder; import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.model.ServiceRegistry; -import lombok.val; import java.util.List; import java.util.Optional; diff --git a/ranger-client/src/main/java/com/flipkart/ranger/client/AbstractRangerHubClient.java b/ranger-client/src/main/java/com/flipkart/ranger/client/AbstractRangerHubClient.java index 0330685a..fdffd781 100644 --- a/ranger-client/src/main/java/com/flipkart/ranger/client/AbstractRangerHubClient.java +++ b/ranger-client/src/main/java/com/flipkart/ranger/client/AbstractRangerHubClient.java @@ -27,7 +27,6 @@ import com.google.common.base.Preconditions; import lombok.Getter; import lombok.extern.slf4j.Slf4j; -import lombok.val; import java.util.Collection; import java.util.Collections; diff --git a/ranger-client/src/test/java/com/flipkart/ranger/client/AbstractRangerHubClientTest.java b/ranger-client/src/test/java/com/flipkart/ranger/client/AbstractRangerHubClientTest.java index f359a8d9..c1091316 100644 --- a/ranger-client/src/test/java/com/flipkart/ranger/client/AbstractRangerHubClientTest.java +++ b/ranger-client/src/test/java/com/flipkart/ranger/client/AbstractRangerHubClientTest.java @@ -17,8 +17,6 @@ import com.flipkart.ranger.client.utils.RangerHubTestUtils; import com.flipkart.ranger.core.model.Service; -import com.flipkart.ranger.core.model.ServiceNode; -import com.flipkart.ranger.core.units.TestNodeData; import com.flipkart.ranger.core.utils.RangerTestUtils; import com.flipkart.ranger.core.utils.TestUtils; import lombok.extern.slf4j.Slf4j; @@ -27,8 +25,6 @@ import org.junit.Assert; import org.junit.Test; -import java.util.Optional; - @Slf4j public class AbstractRangerHubClientTest { diff --git a/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestSimpleUnshardedServiceFinder.java b/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestSimpleUnshardedServiceFinder.java index 7d1d77a5..cbf0064a 100644 --- a/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestSimpleUnshardedServiceFinder.java +++ b/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/TestSimpleUnshardedServiceFinder.java @@ -46,7 +46,11 @@ static class TestDataSource implements NodeDataSource> refresh(Deserializer deserializer) { return Collections.singletonList( - new ServiceNode<>("localhost", 9200, TestNodeData.builder().shardId(1).build()) + ServiceNode.builder() + .host("localhost") + .port(9200) + .nodeData(TestNodeData.builder().shardId(1).build()) + .build() ); } diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/ServiceFinder.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/ServiceFinder.java index 553f741b..7ded56ce 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/ServiceFinder.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/ServiceFinder.java @@ -25,7 +25,6 @@ import lombok.extern.slf4j.Slf4j; import lombok.var; -import javax.swing.text.html.Option; import java.util.Collections; import java.util.List; import java.util.Optional; diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/serviceregistry/ListBasedServiceRegistry.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/serviceregistry/ListBasedServiceRegistry.java index 5a02014b..66e2123f 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/serviceregistry/ListBasedServiceRegistry.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/serviceregistry/ListBasedServiceRegistry.java @@ -21,7 +21,6 @@ import com.google.common.collect.ImmutableList; import lombok.val; -import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.concurrent.atomic.AtomicReference; diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/healthcheck/HealthcheckResult.java b/ranger-core/src/main/java/com/flipkart/ranger/core/healthcheck/HealthcheckResult.java index 65d0cfe4..6d7c2697 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/healthcheck/HealthcheckResult.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/healthcheck/HealthcheckResult.java @@ -16,7 +16,6 @@ package com.flipkart.ranger.core.healthcheck; import lombok.Builder; -import lombok.Data; import lombok.Value; /** diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/ServiceHealthAggregator.java b/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/ServiceHealthAggregator.java index 30eae894..ad9a838e 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/ServiceHealthAggregator.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/ServiceHealthAggregator.java @@ -27,7 +27,6 @@ import java.util.Date; import java.util.List; import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/sample/PingCheckMonitor.java b/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/sample/PingCheckMonitor.java index db754bc3..7b879961 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/sample/PingCheckMonitor.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/sample/PingCheckMonitor.java @@ -24,7 +24,6 @@ import org.apache.http.HttpHost; import org.apache.http.HttpRequest; import org.apache.http.HttpStatus; -import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.conn.routing.HttpRoute; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/model/ServiceNode.java b/ranger-core/src/main/java/com/flipkart/ranger/core/model/ServiceNode.java index 383c6a36..e6ae8871 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/model/ServiceNode.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/model/ServiceNode.java @@ -17,9 +17,15 @@ package com.flipkart.ranger.core.model; import com.flipkart.ranger.core.healthcheck.HealthcheckStatus; +import lombok.AllArgsConstructor; +import lombok.Builder; import lombok.Data; +import lombok.NoArgsConstructor; @Data +@NoArgsConstructor +@AllArgsConstructor +@Builder public class ServiceNode { private String host; private int port; @@ -27,15 +33,6 @@ public class ServiceNode { private HealthcheckStatus healthcheckStatus = HealthcheckStatus.healthy; private long lastUpdatedTimeStamp = Long.MIN_VALUE; - public ServiceNode() { - } - - public ServiceNode(String host, int port, T nodeData) { - this.host = host; - this.port = port; - this.nodeData = nodeData; - } - public String representation() { return String.format("%s:%d", host, port); } diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/serviceprovider/BaseServiceProviderBuilder.java b/ranger-core/src/main/java/com/flipkart/ranger/core/serviceprovider/BaseServiceProviderBuilder.java index 0025063e..48a2ed6a 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/serviceprovider/BaseServiceProviderBuilder.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/serviceprovider/BaseServiceProviderBuilder.java @@ -197,8 +197,7 @@ protected final ServiceProvider buildProvider() { .add(healthcheckUpdateSignalGenerator) .addAll(additionalRefreshSignals) .build(); - val serviceProvider = new ServiceProvider<>(service, - new ServiceNode<>(hostname, port, nodeData), + val serviceProvider = new ServiceProvider<>(service, ServiceNode.builder().host(hostname).port(port).nodeData(nodeData).build(), serializer, usableNodeDataSource, signalGenerators); diff --git a/ranger-core/src/test/java/com/flipkart/ranger/core/finder/nodeselector/RoundRobinServiceNodeSelectorTest.java b/ranger-core/src/test/java/com/flipkart/ranger/core/finder/nodeselector/RoundRobinServiceNodeSelectorTest.java index 15960ae3..273ff997 100644 --- a/ranger-core/src/test/java/com/flipkart/ranger/core/finder/nodeselector/RoundRobinServiceNodeSelectorTest.java +++ b/ranger-core/src/test/java/com/flipkart/ranger/core/finder/nodeselector/RoundRobinServiceNodeSelectorTest.java @@ -18,7 +18,6 @@ import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.units.TestNodeData; import lombok.val; -import lombok.var; import org.junit.Assert; import org.junit.Test; @@ -29,9 +28,9 @@ public class RoundRobinServiceNodeSelectorTest { public void testRandomNodeSelector(){ val roundRobinSelector = new RoundRobinServiceNodeSelector(); val serviceNodes = new ArrayList>(); - serviceNodes.add(new ServiceNode<>("localhost-1", 9000, TestNodeData.builder().shardId(1).build())); - serviceNodes.add(new ServiceNode<>("localhost-2", 9001, TestNodeData.builder().shardId(2).build())); - serviceNodes.add(new ServiceNode<>("localhost-3", 9002, TestNodeData.builder().shardId(3).build())); + serviceNodes.add(ServiceNode.builder().host("localhost-1").port(9000).nodeData(TestNodeData.builder().shardId(1).build()).build()); + serviceNodes.add(ServiceNode.builder().host("localhost-2").port(9001).nodeData(TestNodeData.builder().shardId(2).build()).build()); + serviceNodes.add(ServiceNode.builder().host("localhost-3").port(9002).nodeData(TestNodeData.builder().shardId(3).build()).build()); Assert.assertEquals("localhost-2", roundRobinSelector.select(serviceNodes).getHost()); Assert.assertEquals("localhost-3", roundRobinSelector.select(serviceNodes).getHost()); Assert.assertEquals("localhost-1", roundRobinSelector.select(serviceNodes).getHost()); diff --git a/ranger-core/src/test/java/com/flipkart/ranger/core/utils/RegistryTestUtils.java b/ranger-core/src/test/java/com/flipkart/ranger/core/utils/RegistryTestUtils.java index 759a946c..4f1620e7 100644 --- a/ranger-core/src/test/java/com/flipkart/ranger/core/utils/RegistryTestUtils.java +++ b/ranger-core/src/test/java/com/flipkart/ranger/core/utils/RegistryTestUtils.java @@ -30,9 +30,9 @@ public static MapBasedServiceRegistry getServiceRegistry(){ val service = RangerTestUtils.getService(); val serviceRegistry = new MapBasedServiceRegistry(service); val serviceNodes = ImmutableList.of( - new ServiceNode<>("localhost-1", 9000, TestNodeData.builder().shardId(1).build()), - new ServiceNode<>("localhost-2", 9001, TestNodeData.builder().shardId(2).build()), - new ServiceNode<>("localhost-3", 9002, TestNodeData.builder().shardId(3).build()) + ServiceNode.builder().host("localhost-1").port(9000).nodeData(TestNodeData.builder().shardId(1).build()).build(), + ServiceNode.builder().host("localhost-2").port(9001).nodeData(TestNodeData.builder().shardId(2).build()).build(), + ServiceNode.builder().host("localhost-3").port(9002).nodeData(TestNodeData.builder().shardId(3).build()).build() ); serviceRegistry.updateNodes(serviceNodes); return serviceRegistry; @@ -42,9 +42,9 @@ public static ListBasedServiceRegistry getUnshardedRegistry(){ val service = RangerTestUtils.getService(); val serviceRegistry = new ListBasedServiceRegistry(service); val serviceNodes = ImmutableList.of( - new ServiceNode<>("localhost-1", 9000, TestNodeData.builder().shardId(1).build()), - new ServiceNode<>("localhost-2", 9001, TestNodeData.builder().shardId(2).build()), - new ServiceNode<>("localhost-3", 9002, TestNodeData.builder().shardId(3).build()) + ServiceNode.builder().host("localhost-1").port(9000).nodeData(TestNodeData.builder().shardId(1).build()).build(), + ServiceNode.builder().host("localhost-2").port(9001).nodeData(TestNodeData.builder().shardId(2).build()).build(), + ServiceNode.builder().host("localhost-3").port(9002).nodeData(TestNodeData.builder().shardId(3).build()).build() ); serviceRegistry.updateNodes(serviceNodes); return serviceRegistry; diff --git a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/BaseRangerHttpClientTest.java b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/BaseRangerHttpClientTest.java index 3ed8d871..673f60e5 100644 --- a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/BaseRangerHttpClientTest.java +++ b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/BaseRangerHttpClientTest.java @@ -52,7 +52,7 @@ public abstract class BaseRangerHttpClientTest { @Before public void startTestCluster() throws Exception { val testNode = TestNodeData.builder().shardId(1).build(); - val node = new ServiceNode<>("127.0.0.1", 80, testNode); + val node = ServiceNode.builder().host("127.0.0.1").port(80).nodeData(testNode).build(); node.setHealthcheckStatus(HealthcheckStatus.healthy); node.setLastUpdatedTimeStamp(System.currentTimeMillis()); val payload = objectMapper.writeValueAsBytes( diff --git a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/ShardedRangerHttpClientTest.java b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/ShardedRangerHttpClientTest.java index 3e1d6499..62ca8c84 100644 --- a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/ShardedRangerHttpClientTest.java +++ b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/ShardedRangerHttpClientTest.java @@ -15,12 +15,10 @@ */ package com.flipkart.ranger.client.http; -import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.units.TestNodeData; import com.flipkart.ranger.core.utils.RangerTestUtils; import com.flipkart.ranger.core.utils.TestUtils; import lombok.val; -import lombok.var; import org.junit.Assert; import org.junit.Test; diff --git a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/SimpleRangerHttpClientTest.java b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/SimpleRangerHttpClientTest.java index 5fd9848c..37a4da2a 100644 --- a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/SimpleRangerHttpClientTest.java +++ b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/SimpleRangerHttpClientTest.java @@ -15,7 +15,6 @@ */ package com.flipkart.ranger.client.http; -import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.units.TestNodeData; import com.flipkart.ranger.core.utils.TestUtils; import lombok.val; diff --git a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/UnshardedRangerHttpClientTest.java b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/UnshardedRangerHttpClientTest.java index c89324e9..283dd268 100644 --- a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/UnshardedRangerHttpClientTest.java +++ b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/UnshardedRangerHttpClientTest.java @@ -15,7 +15,6 @@ */ package com.flipkart.ranger.client.http; -import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.units.TestNodeData; import com.flipkart.ranger.core.utils.RangerTestUtils; import com.flipkart.ranger.core.utils.TestUtils; diff --git a/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java b/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java index 4030e1ef..2b8ee723 100644 --- a/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java +++ b/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java @@ -58,7 +58,7 @@ public NodeData(@JsonProperty("name") String name) { @Test public void testFinder() throws Exception { val testNode = new NodeData("testNode"); - val node = new ServiceNode<>("127.0.0.1", 80, testNode); + val node = ServiceNode.builder().host("127.0.0.1").port(80).nodeData(testNode).build(); node.setHealthcheckStatus(HealthcheckStatus.healthy); node.setLastUpdatedTimeStamp(System.currentTimeMillis()); val payload = MAPPER.writeValueAsBytes( diff --git a/ranger-http/src/test/java/com/flipkart/ranger/http/serviceprovider/HttpShardedServiceProviderBuilderTest.java b/ranger-http/src/test/java/com/flipkart/ranger/http/serviceprovider/HttpShardedServiceProviderBuilderTest.java index a581a51a..dd5e3b0f 100644 --- a/ranger-http/src/test/java/com/flipkart/ranger/http/serviceprovider/HttpShardedServiceProviderBuilderTest.java +++ b/ranger-http/src/test/java/com/flipkart/ranger/http/serviceprovider/HttpShardedServiceProviderBuilderTest.java @@ -52,7 +52,7 @@ public TestNodeData(@JsonProperty("farmId") String farmId) { @Test public void testProvider() throws Exception { val farmNodeData = TestNodeData.builder().farmId("farm1").build(); - val testNode = new ServiceNode<>("localhost-1", 80, farmNodeData); + val testNode = ServiceNode.builder().host("127.0.0.1").port(80).nodeData(farmNodeData).build(); val response = MAPPER.writeValueAsBytes( GenericResponse.builder() .code(RangerResponseCode.SUCCESS) diff --git a/ranger-server-bundle/src/test/java/com/flipkart/ranger/zk/server/bundle/RangerServerBundleTest.java b/ranger-server-bundle/src/test/java/com/flipkart/ranger/zk/server/bundle/RangerServerBundleTest.java index 424a6cb9..b8e9fff0 100644 --- a/ranger-server-bundle/src/test/java/com/flipkart/ranger/zk/server/bundle/RangerServerBundleTest.java +++ b/ranger-server-bundle/src/test/java/com/flipkart/ranger/zk/server/bundle/RangerServerBundleTest.java @@ -20,7 +20,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.flipkart.ranger.client.RangerHubClient; import com.flipkart.ranger.client.utils.RangerHubTestUtils; -import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.units.TestNodeData; import com.flipkart.ranger.core.utils.RangerTestUtils; import com.flipkart.ranger.core.utils.TestUtils; @@ -42,7 +41,6 @@ import java.util.Collections; import java.util.List; -import java.util.Optional; import static com.flipkart.ranger.client.utils.RangerHubTestUtils.service; import static org.mockito.ArgumentMatchers.any; diff --git a/ranger-server-common/src/test/java/com/flipkart/ranger/common/server/ShardInfoTest.java b/ranger-server-common/src/test/java/com/flipkart/ranger/common/server/ShardInfoTest.java index 15c7e24e..33aa7de2 100644 --- a/ranger-server-common/src/test/java/com/flipkart/ranger/common/server/ShardInfoTest.java +++ b/ranger-server-common/src/test/java/com/flipkart/ranger/common/server/ShardInfoTest.java @@ -7,7 +7,6 @@ import org.junit.Test; import java.io.BufferedReader; -import java.io.InputStream; import java.io.InputStreamReader; import java.util.Arrays; import java.util.stream.Collectors; diff --git a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/ShardedZKRangerClientTest.java b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/ShardedZKRangerClientTest.java index d7b2287b..579bf905 100644 --- a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/ShardedZKRangerClientTest.java +++ b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/ShardedZKRangerClientTest.java @@ -15,7 +15,6 @@ */ package com.flipkart.ranger.client.zk; -import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.units.TestNodeData; import com.flipkart.ranger.core.utils.RangerTestUtils; import com.flipkart.ranger.core.utils.TestUtils; @@ -24,8 +23,6 @@ import org.junit.Assert; import org.junit.Test; -import java.util.Optional; - @Slf4j public class ShardedZKRangerClientTest extends BaseRangerZKClientTest { diff --git a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/SimpleRangerZKClientTest.java b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/SimpleRangerZKClientTest.java index 8ecd4c1b..5110fb6d 100644 --- a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/SimpleRangerZKClientTest.java +++ b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/SimpleRangerZKClientTest.java @@ -15,14 +15,11 @@ */ package com.flipkart.ranger.client.zk; -import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.units.TestNodeData; import lombok.val; import org.junit.Assert; import org.junit.Test; -import java.util.Optional; - public class SimpleRangerZKClientTest extends BaseRangerZKClientTest { @Test diff --git a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/UnshardedZKRangerClientTest.java b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/UnshardedZKRangerClientTest.java index 92c7c740..4f3a048f 100644 --- a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/UnshardedZKRangerClientTest.java +++ b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/UnshardedZKRangerClientTest.java @@ -15,7 +15,6 @@ */ package com.flipkart.ranger.client.zk; -import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.units.TestNodeData; import com.flipkart.ranger.core.utils.RangerTestUtils; import com.flipkart.ranger.core.utils.TestUtils; @@ -24,8 +23,6 @@ import org.junit.Assert; import org.junit.Test; -import java.util.Optional; - @Slf4j public class UnshardedZKRangerClientTest extends BaseRangerZKClientTest { diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderExtCuratorTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderExtCuratorTest.java index ea92e3b8..97e3fe59 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderExtCuratorTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderExtCuratorTest.java @@ -29,7 +29,6 @@ import com.google.common.collect.Lists; import lombok.extern.slf4j.Slf4j; import lombok.val; -import lombok.var; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.retry.ExponentialBackoffRetry; diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderTest.java index 53c3a58c..1d1e15e4 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderTest.java @@ -31,7 +31,6 @@ import com.google.common.collect.Lists; import lombok.extern.slf4j.Slf4j; import lombok.val; -import lombok.var; import org.apache.curator.test.TestingCluster; import org.junit.After; import org.junit.Assert; diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/SimpleServiceProviderTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/SimpleServiceProviderTest.java index 6e68fe8a..df41f08d 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/SimpleServiceProviderTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/SimpleServiceProviderTest.java @@ -24,7 +24,6 @@ import com.flipkart.ranger.zookeeper.ServiceFinderBuilders; import com.flipkart.ranger.zookeeper.ServiceProviderBuilders; import com.google.common.collect.HashMultiset; -import com.google.common.collect.Multiset; import lombok.val; import org.apache.curator.test.TestingCluster; import org.junit.After; From ab0967e72d3f234910a4ffdd128c73c52ebae308 Mon Sep 17 00:00:00 2001 From: Koushik R Date: Mon, 27 Dec 2021 15:50:56 +0530 Subject: [PATCH 46/69] Added a missing utility class annotation on ResourceHelper --- .../src/test/java/com/flipkart/ranger/http/ResourceHelper.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ranger-http/src/test/java/com/flipkart/ranger/http/ResourceHelper.java b/ranger-http/src/test/java/com/flipkart/ranger/http/ResourceHelper.java index 678761bf..9fd7340c 100644 --- a/ranger-http/src/test/java/com/flipkart/ranger/http/ResourceHelper.java +++ b/ranger-http/src/test/java/com/flipkart/ranger/http/ResourceHelper.java @@ -17,12 +17,14 @@ import com.fasterxml.jackson.databind.ObjectMapper; import lombok.SneakyThrows; +import lombok.experimental.UtilityClass; import lombok.val; import java.io.BufferedReader; import java.io.InputStreamReader; import java.util.stream.Collectors; +@UtilityClass public class ResourceHelper { private static final ObjectMapper objectMapper = new ObjectMapper(); @@ -42,5 +44,4 @@ public static T getResource(String path, Class klass) { if(null == data) return null; return objectMapper.readValue(data, klass); } - } From a4b0ac3621e55b7510c45f7945c56f7a02eecd13 Mon Sep 17 00:00:00 2001 From: Koushik R Date: Mon, 27 Dec 2021 15:53:38 +0530 Subject: [PATCH 47/69] Removed the unnecessary comment --- .../flipkart/ranger/core/healthcheck/HealthcheckResult.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/healthcheck/HealthcheckResult.java b/ranger-core/src/main/java/com/flipkart/ranger/core/healthcheck/HealthcheckResult.java index 6d7c2697..8f8c2b0b 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/healthcheck/HealthcheckResult.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/healthcheck/HealthcheckResult.java @@ -18,9 +18,6 @@ import lombok.Builder; import lombok.Value; -/** - * - */ @Value @Builder public class HealthcheckResult { From e4567991d4020cc9c01304bf60322de7be6086b4 Mon Sep 17 00:00:00 2001 From: Koushik R Date: Mon, 27 Dec 2021 16:03:40 +0530 Subject: [PATCH 48/69] Added the missing val in ServiceNoPoviderTest --- .../flipkart/ranger/zookeeper/model/ServiceNoProviderTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceNoProviderTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceNoProviderTest.java index 7e71c6f9..513dc249 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceNoProviderTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceNoProviderTest.java @@ -53,7 +53,7 @@ public void stopTestCluster() throws Exception { @Test public void testBasicDiscovery() { - SimpleShardedServiceFinder serviceFinder = ServiceFinderBuilders.shardedFinderBuilder() + val serviceFinder = ServiceFinderBuilders.shardedFinderBuilder() .withConnectionString(testingCluster.getConnectString()) .withNamespace("test") .withServiceName("test-service") From c11da3a726eeb59972021000fabe3be4b2d0df97 Mon Sep 17 00:00:00 2001 From: Koushik R Date: Fri, 31 Dec 2021 15:36:16 +0530 Subject: [PATCH 49/69] Fixed await until usages, using an upper bound. Brought down the testcase exec time from 3 minutes to 40 seconds --- .../client/AbstractRangerHubClientTest.java | 9 ++++- .../ranger/client/CriteriaUtilsTest.java | 7 +--- .../ranger/client/stubs/RangerTestHub.java | 2 ++ .../client/utils/RangerHubTestUtils.java | 1 + .../ranger/core/finder/ServiceFinder.java | 4 +++ .../core/finderhub/ServiceFinderHub.java | 5 +++ .../ServiceHealthAggregator.java | 18 +++------- .../monitor/IsolatedHealthMonitor.java | 15 +++----- .../flipkart/ranger/core/utils/TestUtils.java | 12 ++++++- .../http/ShardedRangerHttpClientTest.java | 2 +- .../http/SimpleRangerHttpClientTest.java | 2 +- .../http/UnshardedRangerHttpClientTest.java | 4 +-- .../http/server/bundle/HttpServerBundle.java | 26 -------------- .../HttpShardedServiceFinderBuilderTest.java | 2 +- .../zk/server/bundle/RangerServerBundle.java | 36 +++++++++++++------ .../server/bundle/model/LifecycleSignal.java | 23 ------------ .../server/bundle/RangerServerBundleTest.java | 31 +++------------- .../client/zk/ShardedZKRangerClientTest.java | 2 +- .../zk/UnshardedZKRangerClientTest.java | 2 +- .../zk/server/bundle/ZKServerBundle.java | 16 ++------- .../zk/server/lifecycle/CuratorLifecycle.java | 13 ++++--- .../ServiceHealthAggregatorTest.java | 17 ++++----- .../ServiceProviderIntegrationTest.java | 9 ++--- .../monitor/sample/PingCheckMonitorTest.java | 4 +-- .../model/ServiceNoProviderTest.java | 1 - .../model/ServiceProviderHealthcheckTest.java | 6 +++- .../zookeeper/servicehub/ServiceHubTest.java | 2 +- 27 files changed, 108 insertions(+), 163 deletions(-) delete mode 100644 ranger-server-bundle/src/main/java/com/flipkart/ranger/zk/server/bundle/model/LifecycleSignal.java diff --git a/ranger-client/src/test/java/com/flipkart/ranger/client/AbstractRangerHubClientTest.java b/ranger-client/src/test/java/com/flipkart/ranger/client/AbstractRangerHubClientTest.java index c1091316..1528dfa9 100644 --- a/ranger-client/src/test/java/com/flipkart/ranger/client/AbstractRangerHubClientTest.java +++ b/ranger-client/src/test/java/com/flipkart/ranger/client/AbstractRangerHubClientTest.java @@ -16,6 +16,7 @@ package com.flipkart.ranger.client; import com.flipkart.ranger.client.utils.RangerHubTestUtils; +import com.flipkart.ranger.core.finder.ServiceFinder; import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.core.utils.RangerTestUtils; import com.flipkart.ranger.core.utils.TestUtils; @@ -34,7 +35,13 @@ public class AbstractRangerHubClientTest { public void testAbstractHubClient() { val testAbstractHub = RangerHubTestUtils.getTestHub(); testAbstractHub.start(); - TestUtils.sleepForSeconds(3); + TestUtils.sleepUntil(() -> { + val finders = testAbstractHub.getHub().getFinders().get(); + if(!finders.isEmpty()){ + return finders.values().stream().allMatch(ServiceFinder::isStarted); + } + return false; + }); var node = testAbstractHub.getNode(service).orElse(null); Assert.assertNotNull(node); Assert.assertTrue(node.getHost().equalsIgnoreCase("localhost")); diff --git a/ranger-client/src/test/java/com/flipkart/ranger/client/CriteriaUtilsTest.java b/ranger-client/src/test/java/com/flipkart/ranger/client/CriteriaUtilsTest.java index 755e8112..dc8d7f59 100644 --- a/ranger-client/src/test/java/com/flipkart/ranger/client/CriteriaUtilsTest.java +++ b/ranger-client/src/test/java/com/flipkart/ranger/client/CriteriaUtilsTest.java @@ -27,12 +27,7 @@ public class CriteriaUtilsTest { private Predicate getCriteria(int shardId){ - return new Predicate() { - @Override - public boolean test(TestNodeData testNodeData) { - return testNodeData.getShardId() == shardId; - } - }; + return testNodeData -> testNodeData.getShardId() == shardId; } @Test diff --git a/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/RangerTestHub.java b/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/RangerTestHub.java index 058dc5b9..6e3a8500 100644 --- a/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/RangerTestHub.java +++ b/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/RangerTestHub.java @@ -23,9 +23,11 @@ import com.flipkart.ranger.core.units.TestNodeData; import com.google.common.collect.Sets; import lombok.Builder; +import lombok.Getter; import java.util.function.Predicate; +@Getter public class RangerTestHub extends AbstractRangerHubClient, TestDeserializer> { diff --git a/ranger-client/src/test/java/com/flipkart/ranger/client/utils/RangerHubTestUtils.java b/ranger-client/src/test/java/com/flipkart/ranger/client/utils/RangerHubTestUtils.java index a48c1999..180d95b1 100644 --- a/ranger-client/src/test/java/com/flipkart/ranger/client/utils/RangerHubTestUtils.java +++ b/ranger-client/src/test/java/com/flipkart/ranger/client/utils/RangerHubTestUtils.java @@ -38,4 +38,5 @@ public static RangerTestHub getTestHub(){ .deserilizer(new TestDeserializer<>()) .build(); } + } diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/ServiceFinder.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/ServiceFinder.java index 7ded56ce..51d74314 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/ServiceFinder.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/ServiceFinder.java @@ -40,6 +40,8 @@ public abstract class ServiceFinder startSignal = new ExternalTriggeredSignal<>(() -> null, Collections.emptyList()); @Getter private final ExternalTriggeredSignal stopSignal = new ExternalTriggeredSignal<>(() -> null, Collections.emptyList()); + @Getter + private boolean started = false; protected ServiceFinder( ServiceRegistryType serviceRegistry, @@ -61,9 +63,11 @@ public List> getAll(Predicate criteria) { public void start() { startSignal.trigger(); + started = true; } public void stop() { stopSignal.trigger(); + started = false; } } diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/ServiceFinderHub.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/ServiceFinderHub.java index b8399446..541b2605 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/ServiceFinderHub.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/ServiceFinderHub.java @@ -42,6 +42,7 @@ */ @Slf4j public class ServiceFinderHub> { + @Getter private final AtomicReference>> finders = new AtomicReference<>(new HashMap<>()); private final Lock updateLock = new ReentrantLock(); private final Condition updateCond = updateLock.newCondition(); @@ -63,6 +64,8 @@ public class ServiceFinderHub> { private final AtomicBoolean alreadyUpdating = new AtomicBoolean(false); private Future monitorFuture = null; + @Getter + private boolean started = false; public ServiceFinderHub( ServiceDataSource serviceDataSource, @@ -84,11 +87,13 @@ public void start() { refreshSignals.forEach(signal -> signal.registerConsumer(x -> updateAvailable())); startSignal.trigger(); updateAvailable(); + started = true; log.info("Service finder hub started"); } public void stop() { stopSignal.trigger(); + started = false; if (null != monitorFuture) { try { monitorFuture.cancel(true); diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/ServiceHealthAggregator.java b/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/ServiceHealthAggregator.java index ad9a838e..d833e006 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/ServiceHealthAggregator.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/ServiceHealthAggregator.java @@ -21,6 +21,7 @@ import com.flipkart.ranger.core.healthservice.monitor.IsolatedHealthMonitor; import com.flipkart.ranger.core.healthservice.monitor.Monitor; import com.google.common.collect.Lists; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import lombok.val; @@ -52,6 +53,7 @@ public class ServiceHealthAggregator implements HealthService private List isolatedHealthMonitorList; /* If aggregator is running or not */ + @Getter private AtomicBoolean running; public ServiceHealthAggregator() { @@ -137,10 +139,6 @@ public synchronized void stop() { */ @Override public HealthcheckStatus getServiceHealth() { - if (!running.get()) { - /*throw new IllegalStateException("Cannot get HealthStatus, when Aggregator isnt running. " + - "Please start the aggregator before trying to get health");*/ - } healthcheckStatus.set(HealthcheckStatus.healthy); val currentTime = new Date(); /* check health status of isolated monitors */ @@ -160,13 +158,11 @@ public HealthcheckStatus check() { return getServiceHealth(); } - private boolean isIsolatedMonitorHealthy( IsolatedHealthMonitor isolatedHealthMonitor, Date currentTime) { + private boolean isIsolatedMonitorHealthy(IsolatedHealthMonitor isolatedHealthMonitor, Date currentTime) { if (HealthcheckStatus.unhealthy == isolatedHealthMonitor.getHealthStatus()) { return true; } - val hasValidUpdateTime - = null != isolatedHealthMonitor.getLastStatusUpdateTime() - && hasValidUpdatedTime(isolatedHealthMonitor, currentTime); + val hasValidUpdateTime = isolatedHealthMonitor.hasValidUpdatedTime(currentTime); /* check if the monitor and its last updated time is stale, if so, mark status as unhealthy */ if (!hasValidUpdateTime) { log.error("Monitor: {} is stuck and its status is stale. Marking service as unhealthy", @@ -176,12 +172,6 @@ private boolean isIsolatedMonitorHealthy( IsolatedHealthMonitor isolatedHealthMo return false; } - private boolean hasValidUpdatedTime(IsolatedHealthMonitor isolatedHealthMonitor, Date currentTime) { - val timeDifferenceMillis = currentTime.getTime() - isolatedHealthMonitor.getLastStatusUpdateTime() - .getTime(); - return timeDifferenceMillis <= isolatedHealthMonitor.getStalenessAllowedInMillis(); - } - private void processMonitors() { /* check status of all inline monitors in the same thread */ inlineHealthMonitorList.stream().filter(healthMonitor -> !healthMonitor.isDisabled()).map(Monitor::monitor).filter(monitorStatus -> HealthcheckStatus.unhealthy == monitorStatus).forEach(monitorStatus -> healthcheckStatus.set(HealthcheckStatus.unhealthy)); diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/IsolatedHealthMonitor.java b/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/IsolatedHealthMonitor.java index d3bf3bf6..d255153f 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/IsolatedHealthMonitor.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/healthservice/monitor/IsolatedHealthMonitor.java @@ -110,23 +110,16 @@ public T getHealthStatus() { return healthStatus.get(); } - public Date getLastStatusUpdateTime() { - return null != lastStatusUpdateTime ? new Date(lastStatusUpdateTime.getTime()) : null; - } - public String getName() { return name; } - /** - * @return default of 60 seconds if not set - */ - public long getStalenessAllowedInMillis() { - return stalenessAllowedInMillis; - } - @Override public boolean isDisabled() { return disabled.get(); } + + public boolean hasValidUpdatedTime(Date currentTime){ + return null != lastStatusUpdateTime && (currentTime.getTime() - lastStatusUpdateTime.getTime() <= stalenessAllowedInMillis); + } } diff --git a/ranger-core/src/test/java/com/flipkart/ranger/core/utils/TestUtils.java b/ranger-core/src/test/java/com/flipkart/ranger/core/utils/TestUtils.java index cd6072ec..ff9004c9 100644 --- a/ranger-core/src/test/java/com/flipkart/ranger/core/utils/TestUtils.java +++ b/ranger-core/src/test/java/com/flipkart/ranger/core/utils/TestUtils.java @@ -17,6 +17,7 @@ import lombok.experimental.UtilityClass; +import java.util.concurrent.Callable; import java.util.concurrent.TimeUnit; import static org.awaitility.Awaitility.await; @@ -27,7 +28,16 @@ @UtilityClass public class TestUtils { - public static void sleepForSeconds(int numSeconds) { + /* + If we know the upper bound condition, please use the until with the upper bound. + Only for cases, where you have to wait till the refreshInterval periods, don't want to introduce + refreshed and other boolean flags throughout the code. + */ + public static void sleepUntil(int numSeconds) { await().pollDelay(numSeconds, TimeUnit.SECONDS).until(() -> true); } + + public static void sleepUntil(Callable conditionEvaluator) { + await().until(conditionEvaluator); + } } diff --git a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/ShardedRangerHttpClientTest.java b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/ShardedRangerHttpClientTest.java index 62ca8c84..d31463d7 100644 --- a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/ShardedRangerHttpClientTest.java +++ b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/ShardedRangerHttpClientTest.java @@ -35,7 +35,7 @@ public void testShardedHttpHubClient(){ .build(); client.start(); - TestUtils.sleepForSeconds(6); + TestUtils.sleepUntil(() -> client.getHub().isStarted()); val service = RangerTestUtils.getService("test-n", "test-s"); Assert.assertNotNull(client.getNode(service).orElse(null)); Assert.assertNotNull(client.getNode(service, nodeData -> nodeData.getShardId() == 1).orElse(null)); diff --git a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/SimpleRangerHttpClientTest.java b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/SimpleRangerHttpClientTest.java index 37a4da2a..36b696d5 100644 --- a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/SimpleRangerHttpClientTest.java +++ b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/SimpleRangerHttpClientTest.java @@ -34,7 +34,7 @@ public void testSimpleHttpRangerClient(){ .nodeRefreshIntervalMs(5000) .build(); client.start(); - TestUtils.sleepForSeconds(6); + TestUtils.sleepUntil(() -> client.getServiceFinder().isStarted()); Assert.assertNotNull(client.getNode().orElse(null)); Assert.assertNotNull(client.getNode(nodeData -> nodeData.getShardId() == 1).orElse(null)); Assert.assertNull(client.getNode(nodeData -> nodeData.getShardId() == 2).orElse(null)); diff --git a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/UnshardedRangerHttpClientTest.java b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/UnshardedRangerHttpClientTest.java index 283dd268..6f725eb5 100644 --- a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/UnshardedRangerHttpClientTest.java +++ b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/UnshardedRangerHttpClientTest.java @@ -34,9 +34,7 @@ public void testUnshardedRangerHubClient(){ .nodeRefreshIntervalMs(5000) .build(); client.start(); - - TestUtils.sleepForSeconds(6); - + TestUtils.sleepUntil(() -> client.getHub().isStarted()); val service = RangerTestUtils.getService("test-n", "test-s"); Assert.assertNotNull(client.getNode(service).orElse(null)); Assert.assertNotNull(client.getNode(service, nodeData -> nodeData.getShardId() == 1).orElse(null)); diff --git a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/bundle/HttpServerBundle.java b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/bundle/HttpServerBundle.java index 9492ddac..7e7c195a 100644 --- a/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/bundle/HttpServerBundle.java +++ b/ranger-http-server/src/main/java/com/flipkart/ranger/http/server/bundle/HttpServerBundle.java @@ -24,8 +24,6 @@ import com.flipkart.ranger.http.server.AppConfiguration; import com.flipkart.ranger.http.server.healthcheck.RangerHttpHealthCheck; import com.flipkart.ranger.zk.server.bundle.RangerServerBundle; -import com.flipkart.ranger.zk.server.bundle.model.LifecycleSignal; -import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -33,7 +31,6 @@ import javax.inject.Singleton; import java.io.IOException; -import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -42,24 +39,6 @@ @NoArgsConstructor public class HttpServerBundle extends RangerServerBundle { - @Override - protected void verifyPreconditions(AppConfiguration configuration) { - val rangerConfiguration = configuration.getRangerConfiguration(); - Preconditions.checkNotNull(rangerConfiguration, - "ranger configuration can't be null"); - Preconditions.checkNotNull(rangerConfiguration.getNamespace(), - "Namespace can't be null"); - Preconditions.checkArgument(null != rangerConfiguration.getHttpClientConfigs() && !rangerConfiguration.getHttpClientConfigs().isEmpty(), - "Http client config can't be null"); - } - - @Override - protected void preBundle(AppConfiguration configuration) { - /* - Noop - */ - } - @Override protected List> withHubs(AppConfiguration configuration) { val rangerConfiguration = configuration.getRangerConfiguration(); @@ -85,11 +64,6 @@ protected boolean withInitialRotationStatus(AppConfiguration configuration) { return configuration.isInitialRotationStatus(); } - @Override - protected List withLifecycleSignals(AppConfiguration configuration) { - return Collections.emptyList(); - } - @Override protected List withHealthChecks(AppConfiguration configuration) { return ImmutableList.of(new RangerHttpHealthCheck()); diff --git a/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java b/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java index 2b8ee723..ecacf4d3 100644 --- a/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java +++ b/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java @@ -93,7 +93,7 @@ public void testFinder() throws Exception { .withShardSelector((criteria, registry) -> registry.nodeList()) .build(); finder.start(); - TestUtils.sleepForSeconds(3); + TestUtils.sleepUntil(finder::isStarted); Assert.assertTrue(finder.get(nodeData -> true).isPresent()); } diff --git a/ranger-server-bundle/src/main/java/com/flipkart/ranger/zk/server/bundle/RangerServerBundle.java b/ranger-server-bundle/src/main/java/com/flipkart/ranger/zk/server/bundle/RangerServerBundle.java index b3567f88..23ee0135 100644 --- a/ranger-server-bundle/src/main/java/com/flipkart/ranger/zk/server/bundle/RangerServerBundle.java +++ b/ranger-server-bundle/src/main/java/com/flipkart/ranger/zk/server/bundle/RangerServerBundle.java @@ -18,11 +18,12 @@ import com.codahale.metrics.health.HealthCheck; import com.fasterxml.jackson.databind.ObjectMapper; import com.flipkart.ranger.client.RangerHubClient; -import com.flipkart.ranger.zk.server.bundle.model.LifecycleSignal; +import com.flipkart.ranger.core.signals.Signal; import com.flipkart.ranger.zk.server.bundle.resources.RangerResource; import com.flipkart.ranger.zk.server.bundle.rotation.BirTask; import com.flipkart.ranger.zk.server.bundle.rotation.OorTask; import com.flipkart.ranger.zk.server.bundle.rotation.RotationStatus; +import com.google.common.collect.ImmutableList; import io.dropwizard.Configuration; import io.dropwizard.ConfiguredBundle; import io.dropwizard.lifecycle.Managed; @@ -39,7 +40,7 @@ public abstract class RangerServerBundle< T, U extends Configuration> implements ConfiguredBundle{ - /** + /* Why are we taking a list of hubs, instead of one? To be able to aggregate from different dataSources if need be instead of just fetching from a single dataSource. @@ -52,15 +53,29 @@ public abstract class RangerServerBundle< @Getter private ObjectMapper mapper; - protected abstract void verifyPreconditions(U configuration); - - protected abstract void preBundle(U configuration); + protected void preBundle(U configuration){ + /* + Noop here! Let the subclasses override if there are any. Not a mandatory check everywhere. Http doesn't need it, only dataStore builders will need them. + Need not be abstract! + */ + } - protected abstract List> withHubs(U configuration); + /* + Letting the subclasses override should they have to again. The default is set to false always. Need not be abstract, not a mandatory parameter. + */ + protected boolean withInitialRotationStatus(U configuration){ + return false; + } - protected abstract boolean withInitialRotationStatus(U configuration); + /* + Not a mandatory parameter (for example, needed for zk, not for http! + Letting the subclasses override should they have to, need not be abstract. Avoids boilerplate code everywhere, the default impl! + */ + protected List> withLifecycleSignals(U configuration){ + return ImmutableList.of(); + } - protected abstract List withLifecycleSignals(U configuration); + protected abstract List> withHubs(U configuration); protected abstract List withHealthChecks(U configuration); @@ -73,7 +88,6 @@ public void initialize(Bootstrap bootstrap) { @Override public void run(U configuration, Environment environment) { - verifyPreconditions(configuration); preBundle(configuration); mapper = environment.getObjectMapper(); @@ -92,7 +106,7 @@ public void run(U configuration, Environment environment) { @Override public void start() { log.info("Starting the server manager"); - lifecycleSignals.forEach(LifecycleSignal::start); + lifecycleSignals.forEach(Signal::start); hubs.forEach(RangerHubClient::start); log.info("Started the server manager"); } @@ -101,7 +115,7 @@ public void start() { public void stop() { log.info("Stopping the server manager"); hubs.forEach(RangerHubClient::stop); - lifecycleSignals.forEach(LifecycleSignal::stop); + lifecycleSignals.forEach(Signal::stop); log.info("Stopped the server manager"); } }); diff --git a/ranger-server-bundle/src/main/java/com/flipkart/ranger/zk/server/bundle/model/LifecycleSignal.java b/ranger-server-bundle/src/main/java/com/flipkart/ranger/zk/server/bundle/model/LifecycleSignal.java deleted file mode 100644 index f96b591d..00000000 --- a/ranger-server-bundle/src/main/java/com/flipkart/ranger/zk/server/bundle/model/LifecycleSignal.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright 2015 Flipkart Internet Pvt. Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.flipkart.ranger.zk.server.bundle.model; - -public interface LifecycleSignal { - - void start(); - - void stop(); -} diff --git a/ranger-server-bundle/src/test/java/com/flipkart/ranger/zk/server/bundle/RangerServerBundleTest.java b/ranger-server-bundle/src/test/java/com/flipkart/ranger/zk/server/bundle/RangerServerBundleTest.java index b8e9fff0..9ebf5fff 100644 --- a/ranger-server-bundle/src/test/java/com/flipkart/ranger/zk/server/bundle/RangerServerBundleTest.java +++ b/ranger-server-bundle/src/test/java/com/flipkart/ranger/zk/server/bundle/RangerServerBundleTest.java @@ -19,11 +19,11 @@ import com.codahale.metrics.health.HealthCheck; import com.fasterxml.jackson.databind.ObjectMapper; import com.flipkart.ranger.client.RangerHubClient; +import com.flipkart.ranger.client.stubs.RangerTestHub; import com.flipkart.ranger.client.utils.RangerHubTestUtils; import com.flipkart.ranger.core.units.TestNodeData; import com.flipkart.ranger.core.utils.RangerTestUtils; import com.flipkart.ranger.core.utils.TestUtils; -import com.flipkart.ranger.zk.server.bundle.model.LifecycleSignal; import io.dropwizard.Configuration; import io.dropwizard.jersey.DropwizardResourceConfig; import io.dropwizard.jersey.setup.JerseyEnvironment; @@ -58,35 +58,11 @@ public class RangerServerBundleTest { private final RangerServerBundle rangerServerBundle = new RangerServerBundle() { - @Override - protected void verifyPreconditions(Configuration configuration) { - /* - Nothing to do - */ - } - - @Override - protected void preBundle(Configuration configuration) { - /* - Noop - */ - } - @Override protected List> withHubs(Configuration configuration) { return Collections.singletonList(RangerHubTestUtils.getTestHub()); } - @Override - protected boolean withInitialRotationStatus(Configuration configuration) { - return false; - } - - @Override - protected List withLifecycleSignals(Configuration configuration) { - return Collections.emptyList(); - } - @Override protected List withHealthChecks(Configuration configuration) { return Collections.emptyList(); @@ -113,8 +89,9 @@ public void setup() throws Exception { @Test public void testRangerBundle(){ - TestUtils.sleepForSeconds(3); - val hub = rangerServerBundle.getHubs().get(0); + var hub = rangerServerBundle.getHubs().get(0); + Assert.assertTrue(hub instanceof RangerTestHub); + TestUtils.sleepUntil(() -> ((RangerTestHub) hub).getHub().isStarted()); var node = hub.getNode(service).orElse(null); Assert.assertNotNull(node); Assert.assertTrue(node.getHost().equalsIgnoreCase("localhost")); diff --git a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/ShardedZKRangerClientTest.java b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/ShardedZKRangerClientTest.java index 579bf905..0b828cdc 100644 --- a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/ShardedZKRangerClientTest.java +++ b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/ShardedZKRangerClientTest.java @@ -38,7 +38,7 @@ public void testShardedHub(){ .build(); zkHubClient.start(); - TestUtils.sleepForSeconds(6); + TestUtils.sleepUntil(() -> zkHubClient.getHub().isStarted()); Assert.assertNotNull(zkHubClient.getNode(RangerTestUtils.getService("test-n", "s1")).orElse(null)); Assert.assertNotNull(zkHubClient.getNode(RangerTestUtils.getService("test-n", "s1"), nodeData -> nodeData.getShardId() == 1).orElse(null)); zkHubClient.stop(); diff --git a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/UnshardedZKRangerClientTest.java b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/UnshardedZKRangerClientTest.java index 4f3a048f..28cb171b 100644 --- a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/UnshardedZKRangerClientTest.java +++ b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/UnshardedZKRangerClientTest.java @@ -37,7 +37,7 @@ public void testShardedHub(){ .deserializer(this::read) .build(); zkHubClient.start(); - TestUtils.sleepForSeconds(6); + TestUtils.sleepUntil(() -> zkHubClient.getHub().isStarted()); val service = RangerTestUtils.getService("test-n", "s1"); Assert.assertNotNull(zkHubClient.getNode(service).orElse(null)); Assert.assertNotNull(zkHubClient.getNode(service, nodeData -> nodeData.getShardId() == 1).orElse(null)); diff --git a/ranger-zk-server/src/main/java/com/flipkart/ranger/zk/server/bundle/ZKServerBundle.java b/ranger-zk-server/src/main/java/com/flipkart/ranger/zk/server/bundle/ZKServerBundle.java index 0f32680c..0b5eb99e 100644 --- a/ranger-zk-server/src/main/java/com/flipkart/ranger/zk/server/bundle/ZKServerBundle.java +++ b/ranger-zk-server/src/main/java/com/flipkart/ranger/zk/server/bundle/ZKServerBundle.java @@ -22,11 +22,10 @@ import com.flipkart.ranger.client.zk.UnshardedRangerZKHubClient; import com.flipkart.ranger.common.server.ShardInfo; import com.flipkart.ranger.core.model.ServiceNode; +import com.flipkart.ranger.core.signals.Signal; import com.flipkart.ranger.zk.server.AppConfiguration; -import com.flipkart.ranger.zk.server.bundle.model.LifecycleSignal; import com.flipkart.ranger.zk.server.healthcheck.RangerHealthCheck; import com.flipkart.ranger.zk.server.lifecycle.CuratorLifecycle; -import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -46,17 +45,6 @@ public class ZKServerBundle extends RangerServerBundle withLifecycleSignals(AppConfiguration configuration) { + protected List> withLifecycleSignals(AppConfiguration configuration) { return ImmutableList.of( new CuratorLifecycle(curatorFramework) ); diff --git a/ranger-zk-server/src/main/java/com/flipkart/ranger/zk/server/lifecycle/CuratorLifecycle.java b/ranger-zk-server/src/main/java/com/flipkart/ranger/zk/server/lifecycle/CuratorLifecycle.java index 3b25dbc5..23f90295 100644 --- a/ranger-zk-server/src/main/java/com/flipkart/ranger/zk/server/lifecycle/CuratorLifecycle.java +++ b/ranger-zk-server/src/main/java/com/flipkart/ranger/zk/server/lifecycle/CuratorLifecycle.java @@ -15,20 +15,25 @@ */ package com.flipkart.ranger.zk.server.lifecycle; -import com.flipkart.ranger.zk.server.bundle.model.LifecycleSignal; -import lombok.AllArgsConstructor; +import com.flipkart.ranger.common.server.ShardInfo; +import com.flipkart.ranger.core.signals.Signal; import lombok.extern.slf4j.Slf4j; import org.apache.curator.framework.CuratorFramework; import javax.inject.Singleton; +import java.util.Collections; @Slf4j @Singleton -@AllArgsConstructor -public class CuratorLifecycle implements LifecycleSignal { +public class CuratorLifecycle extends Signal { private final CuratorFramework curatorFramework; + public CuratorLifecycle(CuratorFramework curatorFramework) { + super(() -> null, Collections.emptyList()); + this.curatorFramework = curatorFramework; + } + @Override public void start() { log.info("Starting the curator"); diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/ServiceHealthAggregatorTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/ServiceHealthAggregatorTest.java index ac6c7f35..29e55177 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/ServiceHealthAggregatorTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/ServiceHealthAggregatorTest.java @@ -26,13 +26,15 @@ import org.junit.Before; import org.junit.Test; +import java.util.Date; + public class ServiceHealthAggregatorTest { ServiceHealthAggregator serviceHealthAggregator = new ServiceHealthAggregator(); TestMonitor testMonitor; @Before - public void setUp() throws Exception { + public void setUp() { testMonitor = new TestMonitor("TestHealthMonitor", TimeEntity.everySecond(), 1000); serviceHealthAggregator.addIsolatedMonitor(testMonitor); serviceHealthAggregator.addInlineMonitor(new Monitor() { @@ -48,29 +50,28 @@ public boolean isDisabled() { }); serviceHealthAggregator.start(); - TestUtils.sleepForSeconds(1); + TestUtils.sleepUntil(() -> serviceHealthAggregator.getRunning().get()); } @After - public void tearDown() throws Exception { + public void tearDown() { serviceHealthAggregator.stop(); } @Test - public void testStaleRun() throws Exception { + public void testStaleRun() { testMonitor.run(); testMonitor.setThreadSleep(2000); - TestUtils.sleepForSeconds(4); + TestUtils.sleepUntil(() -> !testMonitor.hasValidUpdatedTime(new Date())); /* in the TestMonitor, thread was sleeping for 2 seconds, */ /* so its state is supposed to be stale (>1 second) and service has to be unhealthy */ Assert.assertEquals(HealthcheckStatus.unhealthy, serviceHealthAggregator.getServiceHealth()); - - testMonitor.setThreadSleep(10); - TestUtils.sleepForSeconds(4); + testMonitor.setThreadSleep(5); + TestUtils.sleepUntil(() -> testMonitor.hasValidUpdatedTime(new Date())); /* in the TestMonitor, thread is sleeping only for 10 milliseconds, */ /* so its state is supposed to be NOT stale (>1 second) and service has to be healthy */ diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/ServiceProviderIntegrationTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/ServiceProviderIntegrationTest.java index 5ec6bdae..51ef1e7a 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/ServiceProviderIntegrationTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/ServiceProviderIntegrationTest.java @@ -88,6 +88,7 @@ public void startTestCluster() throws Exception { } return null; }) + .withNodeRefreshIntervalMs(1000) .build(); serviceFinder.start(); } @@ -98,7 +99,7 @@ public void stopTestCluster() throws Exception { testingCluster.close(); } serviceFinder.stop(); - TestUtils.sleepForSeconds(1); + TestUtils.sleepUntil(() -> !serviceFinder.isStarted()); } @Test @@ -111,7 +112,7 @@ public void testBasicDiscovery() throws Exception { /* with file existing, 3 nodes should be healthy */ var filecreate = file.createNewFile(); System.out.println("created file"); - TestUtils.sleepForSeconds(8); + TestUtils.sleepUntil(2); List> all = serviceFinder.getAll(null); System.out.println("all = " + all); Assert.assertEquals(3, all.size()); @@ -119,7 +120,7 @@ public void testBasicDiscovery() throws Exception { /* with file deleted, all 3 nodes should be unhealthy */ delete = file.delete(); System.out.println("deleted file"); - TestUtils.sleepForSeconds(8); + TestUtils.sleepUntil(2); all = serviceFinder.getAll(null); System.out.println("all = " + all); Assert.assertEquals(0, all.size()); @@ -127,7 +128,7 @@ public void testBasicDiscovery() throws Exception { /* with anotherFile created, the 4th node should become healthy and discoverable */ filecreate = anotherFile.createNewFile(); System.out.println("created anotherFile"); - TestUtils.sleepForSeconds(6); + TestUtils.sleepUntil(2); all = serviceFinder.getAll(null); System.out.println("all = " + all); Assert.assertEquals(1, all.size()); diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/monitor/sample/PingCheckMonitorTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/monitor/sample/PingCheckMonitorTest.java index 0dd36915..c79e68ba 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/monitor/sample/PingCheckMonitorTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/monitor/sample/PingCheckMonitorTest.java @@ -28,7 +28,7 @@ public class PingCheckMonitorTest { @Test - public void testMonitor() throws Exception { + public void testMonitor() { val httpRequest = new HttpGet("/"); val pingCheckMonitor = new PingCheckMonitor(new TimeEntity(2, TimeUnit.SECONDS), httpRequest, 5000, 5, 3, "google.com", 80); Assert.assertEquals(HealthcheckStatus.healthy, pingCheckMonitor.monitor()); @@ -40,7 +40,7 @@ public void testMonitor() throws Exception { } @Test - public void testMonitor2() throws Exception { + public void testMonitor2() { val httpRequest = new HttpGet("/help"); val pingCheckMonitor = new PingCheckMonitor(new TimeEntity(2, TimeUnit.SECONDS), httpRequest, 5000, 5, 3, "google.com", 80); Assert.assertEquals(HealthcheckStatus.healthy, pingCheckMonitor.monitor()); diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceNoProviderTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceNoProviderTest.java index 513dc249..29806c11 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceNoProviderTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceNoProviderTest.java @@ -17,7 +17,6 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import com.flipkart.ranger.core.finder.SimpleShardedServiceFinder; import com.flipkart.ranger.core.finder.nodeselector.RoundRobinServiceNodeSelector; import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.units.TestNodeData; diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderHealthcheckTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderHealthcheckTest.java index 611c0dab..18d29ffb 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderHealthcheckTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderHealthcheckTest.java @@ -28,6 +28,7 @@ import com.flipkart.ranger.zookeeper.ServiceFinderBuilders; import com.flipkart.ranger.zookeeper.ServiceProviderBuilders; import com.google.common.collect.Maps; +import lombok.Getter; import lombok.val; import org.apache.curator.test.TestingCluster; import org.junit.After; @@ -85,7 +86,7 @@ public void testBasicDiscovery() { Assert.assertEquals("localhost-1", node.getHost()); TestServiceProvider testServiceProvider = serviceProviders.get(node.getHost()); testServiceProvider.oor(); - TestUtils.sleepForSeconds(6); + TestUtils.sleepUntil(2); //Sleep till the increment refresh healthCheck interval (> 1sec), no upper bound condition. Assert.assertFalse(serviceFinder.get(RangerTestUtils.getCriteria(1)).isPresent()); serviceFinder.stop(); } @@ -111,6 +112,8 @@ private static final class TestServiceProvider { private final String host; private final int port; private final int shardId; + @Getter + private boolean started = false; public TestServiceProvider(ObjectMapper objectMapper, String connectionString, @@ -152,6 +155,7 @@ public void start() throws Exception { .withHealthUpdateIntervalMs(1000) .build(); serviceProvider.start(); + started = true; } } diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/servicehub/ServiceHubTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/servicehub/ServiceHubTest.java index afa6c4b1..176e318b 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/servicehub/ServiceHubTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/servicehub/ServiceHubTest.java @@ -118,7 +118,7 @@ public void testHub() { refreshProviderSignal.trigger(); refreshHubSignal.trigger(); - TestUtils.sleepForSeconds(3); + TestUtils.sleepUntil(hub::isStarted); val node = hub.finder(RangerTestUtils.getService(NAMESPACE, "s1")) .flatMap(finder -> finder.get(nodeData -> nodeData.getShardId() == 1)).orElse(null); Assert.assertNotNull(node); From 8771dca19184febbcb640d3b3b70bba42f2f0bcf Mon Sep 17 00:00:00 2001 From: Koushik R Date: Fri, 31 Dec 2021 15:38:30 +0530 Subject: [PATCH 50/69] Simplified another awaitility usage --- .../ranger/client/AbstractRangerHubClientTest.java | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/ranger-client/src/test/java/com/flipkart/ranger/client/AbstractRangerHubClientTest.java b/ranger-client/src/test/java/com/flipkart/ranger/client/AbstractRangerHubClientTest.java index 1528dfa9..01f5060a 100644 --- a/ranger-client/src/test/java/com/flipkart/ranger/client/AbstractRangerHubClientTest.java +++ b/ranger-client/src/test/java/com/flipkart/ranger/client/AbstractRangerHubClientTest.java @@ -35,13 +35,7 @@ public class AbstractRangerHubClientTest { public void testAbstractHubClient() { val testAbstractHub = RangerHubTestUtils.getTestHub(); testAbstractHub.start(); - TestUtils.sleepUntil(() -> { - val finders = testAbstractHub.getHub().getFinders().get(); - if(!finders.isEmpty()){ - return finders.values().stream().allMatch(ServiceFinder::isStarted); - } - return false; - }); + TestUtils.sleepUntil(() -> testAbstractHub.getHub().isStarted()); var node = testAbstractHub.getNode(service).orElse(null); Assert.assertNotNull(node); Assert.assertTrue(node.getHost().equalsIgnoreCase("localhost")); From f9ab00ff10d49e109b22dcdc599af656cc668ede Mon Sep 17 00:00:00 2001 From: Koushik R Date: Fri, 31 Dec 2021 15:52:26 +0530 Subject: [PATCH 51/69] Simplified exception handling in HttpNodeDataSink> Removed the unnecessary error in GenericResponse --- .../flipkart/ranger/client/AbstractRangerHubClient.java | 2 +- .../ranger/http/response/model/GenericResponse.java | 1 - .../ranger/http/model/ServiceDataSourceResponse.java | 1 - .../flipkart/ranger/http/model/ServiceNodesResponse.java | 1 - .../ranger/http/model/ServiceRegistrationResponse.java | 1 - .../ranger/http/servicefinder/HttpNodeDataSource.java | 2 +- .../http/servicefinderhub/HttpServiceDataSource.java | 5 ++--- .../ranger/http/serviceprovider/HttpNodeDataSink.java | 8 ++++---- 8 files changed, 8 insertions(+), 13 deletions(-) diff --git a/ranger-client/src/main/java/com/flipkart/ranger/client/AbstractRangerHubClient.java b/ranger-client/src/main/java/com/flipkart/ranger/client/AbstractRangerHubClient.java index fdffd781..9d5f16f6 100644 --- a/ranger-client/src/main/java/com/flipkart/ranger/client/AbstractRangerHubClient.java +++ b/ranger-client/src/main/java/com/flipkart/ranger/client/AbstractRangerHubClient.java @@ -116,7 +116,7 @@ public Collection getRegisteredServices() { try{ return this.getHub().getServiceDataSource().services(); }catch (Exception e){ - log.warn("Call to a hub failed with exception, {}", e.getMessage()); + log.error("Call to the hub failed with exception, {}", e.getMessage()); return Collections.emptySet(); } } diff --git a/ranger-http-model/src/main/java/com/flipkart/ranger/http/response/model/GenericResponse.java b/ranger-http-model/src/main/java/com/flipkart/ranger/http/response/model/GenericResponse.java index 1b279b9d..4b3c543d 100644 --- a/ranger-http-model/src/main/java/com/flipkart/ranger/http/response/model/GenericResponse.java +++ b/ranger-http-model/src/main/java/com/flipkart/ranger/http/response/model/GenericResponse.java @@ -27,7 +27,6 @@ @JsonIgnoreProperties(ignoreUnknown = true) public class GenericResponse { RangerResponseCode code; - String error; T data; @JsonIgnore diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceDataSourceResponse.java b/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceDataSourceResponse.java index 835be5eb..95d30dc5 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceDataSourceResponse.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceDataSourceResponse.java @@ -32,7 +32,6 @@ @JsonIgnoreProperties(ignoreUnknown = true) public class ServiceDataSourceResponse { RangerResponseCode code; - String error; Set data; @JsonIgnore diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceNodesResponse.java b/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceNodesResponse.java index 72742dd2..8886a885 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceNodesResponse.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceNodesResponse.java @@ -34,7 +34,6 @@ @JsonIgnoreProperties(ignoreUnknown = true) public class ServiceNodesResponse { RangerResponseCode code; - String error; List> data; @JsonIgnore diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceRegistrationResponse.java b/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceRegistrationResponse.java index cdcc4949..7a650645 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceRegistrationResponse.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceRegistrationResponse.java @@ -29,7 +29,6 @@ @JsonIgnoreProperties(ignoreUnknown = true) public class ServiceRegistrationResponse { RangerResponseCode code; - String error; @JsonIgnore public boolean isSuccess(){ diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpNodeDataSource.java b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpNodeDataSource.java index 5d5c1b22..6f40997f 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpNodeDataSource.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpNodeDataSource.java @@ -85,7 +85,7 @@ public List> refresh(D deserializer) { serviceNodesResponse.getData(), healthcheckZombieCheckThresholdTime(service)); } else{ - log.warn("Http call to {} returned a failure response with error {}", httpUrl, serviceNodesResponse.getError()); + log.warn("Http call to {} returned a failure response with code {}", httpUrl, serviceNodesResponse.getCode()); } } } diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSource.java b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSource.java index ad68d84d..538df411 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSource.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSource.java @@ -18,7 +18,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.flipkart.ranger.core.finderhub.ServiceDataSource; import com.flipkart.ranger.core.model.Service; -import com.flipkart.ranger.core.util.Exceptions; import com.flipkart.ranger.http.common.HttpNodeDataStoreConnector; import com.flipkart.ranger.http.config.HttpClientConfig; import com.flipkart.ranger.http.model.ServiceDataSourceResponse; @@ -71,7 +70,7 @@ public Collection services() { if(serviceDataSourceResponse.isSuccess()){ return serviceDataSourceResponse.getData(); }else{ - log.warn("Http call to {} returned a failure response with error {}", httpUrl, serviceDataSourceResponse.getError()); + log.warn("Http call to {} returned a failure response with code {}", httpUrl, serviceDataSourceResponse.getCode()); } } } @@ -81,7 +80,7 @@ public Collection services() { } } catch (IOException e) { - Exceptions.illegalState("Error fetching data from server: " + httpUrl, e); + log.info("Error parsing the response from server for : {} with exception {}", httpUrl, e); } log.error("No data returned from server: " + httpUrl); diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/serviceprovider/HttpNodeDataSink.java b/ranger-http/src/main/java/com/flipkart/ranger/http/serviceprovider/HttpNodeDataSink.java index 0d876a87..8245ca5d 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/serviceprovider/HttpNodeDataSink.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/serviceprovider/HttpNodeDataSink.java @@ -63,9 +63,9 @@ public void updateState(S serializer, ServiceNode serviceNode) { .encodedPath(url) .build(); val requestBody = RequestBody.create(serializer.serialize(serviceNode)); - val serviceRegistrationResponse = registerService(httpUrl, requestBody); - if(!serviceRegistrationResponse.isPresent() || !serviceRegistrationResponse.get().isSuccess()){ - log.warn("Http call to {} returned a failure response", httpUrl); + val serviceRegistrationResponse = registerService(httpUrl, requestBody).orElse(null); + if(null == serviceRegistrationResponse || !serviceRegistrationResponse.isSuccess()){ + log.warn("Http call to {} returned a failure response {}", httpUrl, serviceRegistrationResponse); Exceptions.illegalState("Error updating state on the server for node data: " + httpUrl); } } @@ -92,7 +92,7 @@ private Optional registerService(HttpUrl httpUrl, R } } catch (IOException e) { - Exceptions.illegalState("Error updating state on the server: " + httpUrl, e); + log.error("Error updating state on the server with httpUrl {} with exception {} ", httpUrl, e); } return Optional.empty(); } From 6e2027a49a2f0c8364572c7ef08bbaee34b7f1bf Mon Sep 17 00:00:00 2001 From: Koushik R Date: Fri, 31 Dec 2021 16:02:12 +0530 Subject: [PATCH 52/69] Trimmed down the success method in httpResponses --- .../ranger/http/model/ServiceDataSourceResponse.java | 4 ++-- .../com/flipkart/ranger/http/model/ServiceNodesResponse.java | 4 ++-- .../ranger/http/model/ServiceRegistrationResponse.java | 2 +- .../ranger/http/servicefinder/HttpNodeDataSource.java | 4 ++-- .../ranger/http/servicefinderhub/HttpServiceDataSource.java | 4 ++-- .../ranger/http/serviceprovider/HttpNodeDataSink.java | 2 +- .../ranger/http/model/ServiceRegistrationResponseTest.java | 5 ++--- 7 files changed, 12 insertions(+), 13 deletions(-) diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceDataSourceResponse.java b/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceDataSourceResponse.java index 95d30dc5..708db3bd 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceDataSourceResponse.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceDataSourceResponse.java @@ -35,7 +35,7 @@ public class ServiceDataSourceResponse { Set data; @JsonIgnore - public boolean isSuccess(){ - return code == RangerResponseCode.SUCCESS; + public boolean valid(){ + return code == RangerResponseCode.SUCCESS && null != data; } } diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceNodesResponse.java b/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceNodesResponse.java index 8886a885..bd606477 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceNodesResponse.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceNodesResponse.java @@ -37,7 +37,7 @@ public class ServiceNodesResponse { List> data; @JsonIgnore - public boolean isSuccess(){ - return code == RangerResponseCode.SUCCESS; + public boolean valid(){ + return code == RangerResponseCode.SUCCESS && null != data; } } diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceRegistrationResponse.java b/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceRegistrationResponse.java index 7a650645..8cc8ece2 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceRegistrationResponse.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceRegistrationResponse.java @@ -31,7 +31,7 @@ public class ServiceRegistrationResponse { RangerResponseCode code; @JsonIgnore - public boolean isSuccess(){ + public boolean valid(){ return code == RangerResponseCode.SUCCESS; } } diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpNodeDataSource.java b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpNodeDataSource.java index 6f40997f..8c55a8b6 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpNodeDataSource.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinder/HttpNodeDataSource.java @@ -79,13 +79,13 @@ public List> refresh(D deserializer) { } else { val bytes = body.bytes(); val serviceNodesResponse = deserializer.deserialize(bytes); - if(serviceNodesResponse.isSuccess()){ + if(serviceNodesResponse.valid()){ return FinderUtils.filterValidNodes( service, serviceNodesResponse.getData(), healthcheckZombieCheckThresholdTime(service)); } else{ - log.warn("Http call to {} returned a failure response with code {}", httpUrl, serviceNodesResponse.getCode()); + log.warn("Http call to {} returned a failure response with response {}", httpUrl, serviceNodesResponse); } } } diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSource.java b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSource.java index 538df411..f0a4b2dc 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSource.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSource.java @@ -67,10 +67,10 @@ public Collection services() { else { val bytes = body.bytes(); val serviceDataSourceResponse = mapper.readValue(bytes, ServiceDataSourceResponse.class); - if(serviceDataSourceResponse.isSuccess()){ + if(serviceDataSourceResponse.valid()){ return serviceDataSourceResponse.getData(); }else{ - log.warn("Http call to {} returned a failure response with code {}", httpUrl, serviceDataSourceResponse.getCode()); + log.warn("Http call to {} returned a failure response with data {}", httpUrl, serviceDataSourceResponse); } } } diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/serviceprovider/HttpNodeDataSink.java b/ranger-http/src/main/java/com/flipkart/ranger/http/serviceprovider/HttpNodeDataSink.java index 8245ca5d..435d1d4b 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/serviceprovider/HttpNodeDataSink.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/serviceprovider/HttpNodeDataSink.java @@ -64,7 +64,7 @@ public void updateState(S serializer, ServiceNode serviceNode) { .build(); val requestBody = RequestBody.create(serializer.serialize(serviceNode)); val serviceRegistrationResponse = registerService(httpUrl, requestBody).orElse(null); - if(null == serviceRegistrationResponse || !serviceRegistrationResponse.isSuccess()){ + if(null == serviceRegistrationResponse || !serviceRegistrationResponse.valid()){ log.warn("Http call to {} returned a failure response {}", httpUrl, serviceRegistrationResponse); Exceptions.illegalState("Error updating state on the server for node data: " + httpUrl); } diff --git a/ranger-http/src/test/java/com/flipkart/ranger/http/model/ServiceRegistrationResponseTest.java b/ranger-http/src/test/java/com/flipkart/ranger/http/model/ServiceRegistrationResponseTest.java index 4381de30..85121cb2 100644 --- a/ranger-http/src/test/java/com/flipkart/ranger/http/model/ServiceRegistrationResponseTest.java +++ b/ranger-http/src/test/java/com/flipkart/ranger/http/model/ServiceRegistrationResponseTest.java @@ -16,7 +16,6 @@ package com.flipkart.ranger.http.model; import com.flipkart.ranger.http.ResourceHelper; -import com.flipkart.ranger.http.response.model.GenericResponse; import lombok.val; import org.junit.Assert; import org.junit.Test; @@ -25,8 +24,8 @@ public class ServiceRegistrationResponseTest { @Test public void testServiceRegistrationResponse(){ - val resource = ResourceHelper.getResource("fixtures/serviceResponse.json", GenericResponse.class); + val resource = ResourceHelper.getResource("fixtures/serviceResponse.json", ServiceRegistrationResponse.class); Assert.assertNotNull(resource); - Assert.assertTrue(resource.success()); + Assert.assertTrue(resource.valid()); } } From d9c87f7a5a8e84ca4b3b6131cdf4243e1f98bc2f Mon Sep 17 00:00:00 2001 From: Koushik R Date: Fri, 31 Dec 2021 16:28:05 +0530 Subject: [PATCH 53/69] Fixed serviceProvider tests --- .../ranger/http/model/ServiceRegistrationResponse.java | 6 ++++-- .../HttpShardedServiceProviderBuilderTest.java | 5 +++++ .../src/test/resources/fixtures/serviceResponse.json | 6 +++++- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceRegistrationResponse.java b/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceRegistrationResponse.java index 8cc8ece2..f9482bd9 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceRegistrationResponse.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceRegistrationResponse.java @@ -17,6 +17,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.http.response.model.RangerResponseCode; import lombok.Builder; import lombok.Value; @@ -27,11 +28,12 @@ @Jacksonized @Builder @JsonIgnoreProperties(ignoreUnknown = true) -public class ServiceRegistrationResponse { +public class ServiceRegistrationResponse { RangerResponseCode code; + ServiceNode data; @JsonIgnore public boolean valid(){ - return code == RangerResponseCode.SUCCESS; + return code == RangerResponseCode.SUCCESS && null != data; } } diff --git a/ranger-http/src/test/java/com/flipkart/ranger/http/serviceprovider/HttpShardedServiceProviderBuilderTest.java b/ranger-http/src/test/java/com/flipkart/ranger/http/serviceprovider/HttpShardedServiceProviderBuilderTest.java index dd5e3b0f..2baca948 100644 --- a/ranger-http/src/test/java/com/flipkart/ranger/http/serviceprovider/HttpShardedServiceProviderBuilderTest.java +++ b/ranger-http/src/test/java/com/flipkart/ranger/http/serviceprovider/HttpShardedServiceProviderBuilderTest.java @@ -56,6 +56,11 @@ public void testProvider() throws Exception { val response = MAPPER.writeValueAsBytes( GenericResponse.builder() .code(RangerResponseCode.SUCCESS) + .data(ServiceNode.builder() + .host("localhost") + .port(8080) + .build() + ) .build()); byte[] requestBytes = MAPPER.writeValueAsBytes(testNode); server.stubFor(post(urlEqualTo("/ranger/nodes/v1/add/testns/test")) diff --git a/ranger-http/src/test/resources/fixtures/serviceResponse.json b/ranger-http/src/test/resources/fixtures/serviceResponse.json index d167d8a8..232add1f 100644 --- a/ranger-http/src/test/resources/fixtures/serviceResponse.json +++ b/ranger-http/src/test/resources/fixtures/serviceResponse.json @@ -1,3 +1,7 @@ { - "code" : "SUCCESS" + "code" : "SUCCESS", + "data" : { + "host" : "localhost", + "port" : 8080 + } } \ No newline at end of file From 2a8f14ff522aa20fedb49a69e9d239f39fa7deae Mon Sep 17 00:00:00 2001 From: Koushik R Date: Fri, 31 Dec 2021 16:59:30 +0530 Subject: [PATCH 54/69] Removed the RangerResponseCode. It is adding no real value and keeping it for future, while not completely nailing down how a serviceProvider's errors may be manifested is not a good option, imo. In the case of a finderHub though, since there could be any of the hubs that may error out (and the errors are getting logged at the point whey they error out), one responseCode won't cut it. I don't think it is future provisioning as well, if anything this will hinder our ability to add proper error blocks in the future, when the need arises --- .../client/AbstractRangerHubClientTest.java | 1 - .../client/http/BaseRangerHttpClientTest.java | 3 --- .../http/response/model/GenericResponse.java | 5 ++--- .../response/model/RangerResponseCode.java | 22 ------------------- .../http/model/ServiceDataSourceResponse.java | 4 +--- .../http/model/ServiceNodesResponse.java | 5 ++--- .../model/ServiceRegistrationResponse.java | 4 +--- .../HttpShardedServiceFinderBuilderTest.java | 2 -- .../HttpServiceDataSourceTest.java | 2 -- ...HttpShardedServiceProviderBuilderTest.java | 2 -- .../fixtures/serviceNodesResponse.json | 1 - .../resources/fixtures/serviceResponse.json | 1 - .../bundle/resources/RangerResource.java | 4 ---- .../ServiceProviderIntegrationTest.java | 6 ++--- 14 files changed, 9 insertions(+), 53 deletions(-) delete mode 100644 ranger-http-model/src/main/java/com/flipkart/ranger/http/response/model/RangerResponseCode.java diff --git a/ranger-client/src/test/java/com/flipkart/ranger/client/AbstractRangerHubClientTest.java b/ranger-client/src/test/java/com/flipkart/ranger/client/AbstractRangerHubClientTest.java index 01f5060a..0b9fbe4f 100644 --- a/ranger-client/src/test/java/com/flipkart/ranger/client/AbstractRangerHubClientTest.java +++ b/ranger-client/src/test/java/com/flipkart/ranger/client/AbstractRangerHubClientTest.java @@ -16,7 +16,6 @@ package com.flipkart.ranger.client; import com.flipkart.ranger.client.utils.RangerHubTestUtils; -import com.flipkart.ranger.core.finder.ServiceFinder; import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.core.utils.RangerTestUtils; import com.flipkart.ranger.core.utils.TestUtils; diff --git a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/BaseRangerHttpClientTest.java b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/BaseRangerHttpClientTest.java index 673f60e5..409e9e5e 100644 --- a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/BaseRangerHttpClientTest.java +++ b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/BaseRangerHttpClientTest.java @@ -25,7 +25,6 @@ import com.flipkart.ranger.http.config.HttpClientConfig; import com.flipkart.ranger.http.model.ServiceDataSourceResponse; import com.flipkart.ranger.http.model.ServiceNodesResponse; -import com.flipkart.ranger.http.response.model.RangerResponseCode; import com.github.tomakehurst.wiremock.junit.WireMockRule; import com.google.common.collect.Lists; import com.google.common.collect.Sets; @@ -58,7 +57,6 @@ public void startTestCluster() throws Exception { val payload = objectMapper.writeValueAsBytes( ServiceNodesResponse.builder() .data(Lists.newArrayList(node)) - .code(RangerResponseCode.SUCCESS) .build()); server.stubFor(get(urlEqualTo("/ranger/nodes/v1/test-n/test-s")) .willReturn(aResponse() @@ -66,7 +64,6 @@ public void startTestCluster() throws Exception { .withStatus(200))); val responseObj = ServiceDataSourceResponse.builder() - .code(RangerResponseCode.SUCCESS) .data(Sets.newHashSet( RangerTestUtils.getService("test-n", "test-s") )) diff --git a/ranger-http-model/src/main/java/com/flipkart/ranger/http/response/model/GenericResponse.java b/ranger-http-model/src/main/java/com/flipkart/ranger/http/response/model/GenericResponse.java index 4b3c543d..320e5cf0 100644 --- a/ranger-http-model/src/main/java/com/flipkart/ranger/http/response/model/GenericResponse.java +++ b/ranger-http-model/src/main/java/com/flipkart/ranger/http/response/model/GenericResponse.java @@ -26,11 +26,10 @@ @Jacksonized @JsonIgnoreProperties(ignoreUnknown = true) public class GenericResponse { - RangerResponseCode code; T data; @JsonIgnore - public boolean success(){ - return code == RangerResponseCode.SUCCESS; + public boolean valid(){ + return null != data; } } diff --git a/ranger-http-model/src/main/java/com/flipkart/ranger/http/response/model/RangerResponseCode.java b/ranger-http-model/src/main/java/com/flipkart/ranger/http/response/model/RangerResponseCode.java deleted file mode 100644 index e528e5f6..00000000 --- a/ranger-http-model/src/main/java/com/flipkart/ranger/http/response/model/RangerResponseCode.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright 2015 Flipkart Internet Pvt. Ltd. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.flipkart.ranger.http.response.model; - -public enum RangerResponseCode { - - SUCCESS; - -} diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceDataSourceResponse.java b/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceDataSourceResponse.java index 708db3bd..dfc73760 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceDataSourceResponse.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceDataSourceResponse.java @@ -18,7 +18,6 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.flipkart.ranger.core.model.Service; -import com.flipkart.ranger.http.response.model.RangerResponseCode; import lombok.Builder; import lombok.Value; import lombok.extern.jackson.Jacksonized; @@ -31,11 +30,10 @@ @Builder @JsonIgnoreProperties(ignoreUnknown = true) public class ServiceDataSourceResponse { - RangerResponseCode code; Set data; @JsonIgnore public boolean valid(){ - return code == RangerResponseCode.SUCCESS && null != data; + return null != data; } } diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceNodesResponse.java b/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceNodesResponse.java index bd606477..369b36ca 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceNodesResponse.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceNodesResponse.java @@ -18,7 +18,6 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.flipkart.ranger.core.model.ServiceNode; -import com.flipkart.ranger.http.response.model.RangerResponseCode; import lombok.Builder; import lombok.Value; import lombok.extern.jackson.Jacksonized; @@ -33,11 +32,11 @@ @Builder @JsonIgnoreProperties(ignoreUnknown = true) public class ServiceNodesResponse { - RangerResponseCode code; + List> data; @JsonIgnore public boolean valid(){ - return code == RangerResponseCode.SUCCESS && null != data; + return null != data; } } diff --git a/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceRegistrationResponse.java b/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceRegistrationResponse.java index f9482bd9..0dc5f682 100644 --- a/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceRegistrationResponse.java +++ b/ranger-http/src/main/java/com/flipkart/ranger/http/model/ServiceRegistrationResponse.java @@ -18,7 +18,6 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.flipkart.ranger.core.model.ServiceNode; -import com.flipkart.ranger.http.response.model.RangerResponseCode; import lombok.Builder; import lombok.Value; import lombok.extern.jackson.Jacksonized; @@ -29,11 +28,10 @@ @Builder @JsonIgnoreProperties(ignoreUnknown = true) public class ServiceRegistrationResponse { - RangerResponseCode code; ServiceNode data; @JsonIgnore public boolean valid(){ - return code == RangerResponseCode.SUCCESS && null != data; + return null != data; } } diff --git a/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java b/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java index ecacf4d3..bae5a7a7 100644 --- a/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java +++ b/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java @@ -23,7 +23,6 @@ import com.flipkart.ranger.core.utils.TestUtils; import com.flipkart.ranger.http.config.HttpClientConfig; import com.flipkart.ranger.http.model.ServiceNodesResponse; -import com.flipkart.ranger.http.response.model.RangerResponseCode; import com.github.tomakehurst.wiremock.junit.WireMockRule; import lombok.Data; import lombok.val; @@ -64,7 +63,6 @@ public void testFinder() throws Exception { val payload = MAPPER.writeValueAsBytes( ServiceNodesResponse.builder() .data(Collections.singletonList(node)) - .code(RangerResponseCode.SUCCESS) .build()); server.stubFor(get(urlEqualTo("/ranger/nodes/v1/testns/test")) .willReturn(aResponse() diff --git a/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSourceTest.java b/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSourceTest.java index 56dfe711..fa7f4b88 100644 --- a/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSourceTest.java +++ b/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinderhub/HttpServiceDataSourceTest.java @@ -20,7 +20,6 @@ import com.flipkart.ranger.core.utils.RangerTestUtils; import com.flipkart.ranger.http.config.HttpClientConfig; import com.flipkart.ranger.http.model.ServiceDataSourceResponse; -import com.flipkart.ranger.http.response.model.RangerResponseCode; import com.github.tomakehurst.wiremock.junit.WireMockRule; import com.google.common.collect.Sets; import lombok.Builder; @@ -54,7 +53,6 @@ public TestNodeData(@JsonProperty("farmId") String farmId) { @Test public void testServiceDataSource() throws IOException { val responseObj = ServiceDataSourceResponse.builder() - .code(RangerResponseCode.SUCCESS) .data(Sets.newHashSet( RangerTestUtils.getService("test-n", "test-s"), RangerTestUtils.getService("test-n", "test-s1"), diff --git a/ranger-http/src/test/java/com/flipkart/ranger/http/serviceprovider/HttpShardedServiceProviderBuilderTest.java b/ranger-http/src/test/java/com/flipkart/ranger/http/serviceprovider/HttpShardedServiceProviderBuilderTest.java index 2baca948..aaa1cbf5 100644 --- a/ranger-http/src/test/java/com/flipkart/ranger/http/serviceprovider/HttpShardedServiceProviderBuilderTest.java +++ b/ranger-http/src/test/java/com/flipkart/ranger/http/serviceprovider/HttpShardedServiceProviderBuilderTest.java @@ -21,7 +21,6 @@ import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.http.config.HttpClientConfig; import com.flipkart.ranger.http.response.model.GenericResponse; -import com.flipkart.ranger.http.response.model.RangerResponseCode; import com.github.tomakehurst.wiremock.junit.WireMockRule; import lombok.Builder; import lombok.Data; @@ -55,7 +54,6 @@ public void testProvider() throws Exception { val testNode = ServiceNode.builder().host("127.0.0.1").port(80).nodeData(farmNodeData).build(); val response = MAPPER.writeValueAsBytes( GenericResponse.builder() - .code(RangerResponseCode.SUCCESS) .data(ServiceNode.builder() .host("localhost") .port(8080) diff --git a/ranger-http/src/test/resources/fixtures/serviceNodesResponse.json b/ranger-http/src/test/resources/fixtures/serviceNodesResponse.json index 4a725388..0fa72860 100644 --- a/ranger-http/src/test/resources/fixtures/serviceNodesResponse.json +++ b/ranger-http/src/test/resources/fixtures/serviceNodesResponse.json @@ -1,5 +1,4 @@ { - "code" : "SUCCESS", "data" : [ { "host" : "localhost-1", diff --git a/ranger-http/src/test/resources/fixtures/serviceResponse.json b/ranger-http/src/test/resources/fixtures/serviceResponse.json index 232add1f..c35da116 100644 --- a/ranger-http/src/test/resources/fixtures/serviceResponse.json +++ b/ranger-http/src/test/resources/fixtures/serviceResponse.json @@ -1,5 +1,4 @@ { - "code" : "SUCCESS", "data" : { "host" : "localhost", "port" : 8080 diff --git a/ranger-server-bundle/src/main/java/com/flipkart/ranger/zk/server/bundle/resources/RangerResource.java b/ranger-server-bundle/src/main/java/com/flipkart/ranger/zk/server/bundle/resources/RangerResource.java index c0f21373..689a9190 100644 --- a/ranger-server-bundle/src/main/java/com/flipkart/ranger/zk/server/bundle/resources/RangerResource.java +++ b/ranger-server-bundle/src/main/java/com/flipkart/ranger/zk/server/bundle/resources/RangerResource.java @@ -33,8 +33,6 @@ import java.util.Set; import java.util.stream.Collectors; -import static com.flipkart.ranger.http.response.model.RangerResponseCode.SUCCESS; - @Slf4j @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) @@ -53,7 +51,6 @@ public RangerResource(List> rangerHubs){ @Metered public GenericResponse> getServices() { return GenericResponse.>builder() - .code(SUCCESS) .data(rangerHubs.stream().map(RangerHubClient::getRegisteredServices) .flatMap(Collection::stream).collect(Collectors.toSet())) .build(); @@ -68,7 +65,6 @@ public GenericResponse>> getNodes( ){ val service = Service.builder().namespace(namespace).serviceName(serviceName).build(); return GenericResponse.>>builder() - .code(SUCCESS) .data(rangerHubs.stream().map(hub -> hub.getAllNodes(service)) .flatMap(List::stream).collect(Collectors.toList())) .build(); diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/ServiceProviderIntegrationTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/ServiceProviderIntegrationTest.java index 51ef1e7a..215be99b 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/ServiceProviderIntegrationTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/ServiceProviderIntegrationTest.java @@ -112,7 +112,7 @@ public void testBasicDiscovery() throws Exception { /* with file existing, 3 nodes should be healthy */ var filecreate = file.createNewFile(); System.out.println("created file"); - TestUtils.sleepUntil(2); + TestUtils.sleepUntil(3); List> all = serviceFinder.getAll(null); System.out.println("all = " + all); Assert.assertEquals(3, all.size()); @@ -120,7 +120,7 @@ public void testBasicDiscovery() throws Exception { /* with file deleted, all 3 nodes should be unhealthy */ delete = file.delete(); System.out.println("deleted file"); - TestUtils.sleepUntil(2); + TestUtils.sleepUntil(3); all = serviceFinder.getAll(null); System.out.println("all = " + all); Assert.assertEquals(0, all.size()); @@ -128,7 +128,7 @@ public void testBasicDiscovery() throws Exception { /* with anotherFile created, the 4th node should become healthy and discoverable */ filecreate = anotherFile.createNewFile(); System.out.println("created anotherFile"); - TestUtils.sleepUntil(2); + TestUtils.sleepUntil(3); all = serviceFinder.getAll(null); System.out.println("all = " + all); Assert.assertEquals(1, all.size()); From 601774440d020838dc6c7085d55cd2d67b0f93c1 Mon Sep 17 00:00:00 2001 From: Koushik R Date: Fri, 31 Dec 2021 17:22:44 +0530 Subject: [PATCH 55/69] Remvoed the bound condition in the hub, using the finder one all around --- .../flipkart/ranger/client/AbstractRangerHubClientTest.java | 3 ++- .../com/flipkart/ranger/core/finderhub/ServiceFinderHub.java | 4 ---- .../ranger/client/http/ShardedRangerHttpClientTest.java | 3 ++- .../ranger/client/http/UnshardedRangerHttpClientTest.java | 3 ++- .../ranger/zk/server/bundle/RangerServerBundleTest.java | 3 ++- .../flipkart/ranger/client/zk/ShardedZKRangerClientTest.java | 3 ++- .../ranger/client/zk/UnshardedZKRangerClientTest.java | 3 ++- .../flipkart/ranger/zookeeper/servicehub/ServiceHubTest.java | 3 ++- 8 files changed, 14 insertions(+), 11 deletions(-) diff --git a/ranger-client/src/test/java/com/flipkart/ranger/client/AbstractRangerHubClientTest.java b/ranger-client/src/test/java/com/flipkart/ranger/client/AbstractRangerHubClientTest.java index 0b9fbe4f..b35f5ea3 100644 --- a/ranger-client/src/test/java/com/flipkart/ranger/client/AbstractRangerHubClientTest.java +++ b/ranger-client/src/test/java/com/flipkart/ranger/client/AbstractRangerHubClientTest.java @@ -16,6 +16,7 @@ package com.flipkart.ranger.client; import com.flipkart.ranger.client.utils.RangerHubTestUtils; +import com.flipkart.ranger.core.finder.ServiceFinder; import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.core.utils.RangerTestUtils; import com.flipkart.ranger.core.utils.TestUtils; @@ -34,7 +35,7 @@ public class AbstractRangerHubClientTest { public void testAbstractHubClient() { val testAbstractHub = RangerHubTestUtils.getTestHub(); testAbstractHub.start(); - TestUtils.sleepUntil(() -> testAbstractHub.getHub().isStarted()); + TestUtils.sleepUntil(() -> testAbstractHub.getHub().getFinders().get().values().stream().allMatch(ServiceFinder::isStarted)); var node = testAbstractHub.getNode(service).orElse(null); Assert.assertNotNull(node); Assert.assertTrue(node.getHost().equalsIgnoreCase("localhost")); diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/ServiceFinderHub.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/ServiceFinderHub.java index 541b2605..da3808aa 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/ServiceFinderHub.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finderhub/ServiceFinderHub.java @@ -64,8 +64,6 @@ public class ServiceFinderHub> { private final AtomicBoolean alreadyUpdating = new AtomicBoolean(false); private Future monitorFuture = null; - @Getter - private boolean started = false; public ServiceFinderHub( ServiceDataSource serviceDataSource, @@ -87,13 +85,11 @@ public void start() { refreshSignals.forEach(signal -> signal.registerConsumer(x -> updateAvailable())); startSignal.trigger(); updateAvailable(); - started = true; log.info("Service finder hub started"); } public void stop() { stopSignal.trigger(); - started = false; if (null != monitorFuture) { try { monitorFuture.cancel(true); diff --git a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/ShardedRangerHttpClientTest.java b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/ShardedRangerHttpClientTest.java index d31463d7..14a083e3 100644 --- a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/ShardedRangerHttpClientTest.java +++ b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/ShardedRangerHttpClientTest.java @@ -15,6 +15,7 @@ */ package com.flipkart.ranger.client.http; +import com.flipkart.ranger.core.finder.ServiceFinder; import com.flipkart.ranger.core.units.TestNodeData; import com.flipkart.ranger.core.utils.RangerTestUtils; import com.flipkart.ranger.core.utils.TestUtils; @@ -35,7 +36,7 @@ public void testShardedHttpHubClient(){ .build(); client.start(); - TestUtils.sleepUntil(() -> client.getHub().isStarted()); + TestUtils.sleepUntil(() -> client.getHub().getFinders().get().values().stream().allMatch(ServiceFinder::isStarted)); val service = RangerTestUtils.getService("test-n", "test-s"); Assert.assertNotNull(client.getNode(service).orElse(null)); Assert.assertNotNull(client.getNode(service, nodeData -> nodeData.getShardId() == 1).orElse(null)); diff --git a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/UnshardedRangerHttpClientTest.java b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/UnshardedRangerHttpClientTest.java index 6f725eb5..21760d75 100644 --- a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/UnshardedRangerHttpClientTest.java +++ b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/UnshardedRangerHttpClientTest.java @@ -15,6 +15,7 @@ */ package com.flipkart.ranger.client.http; +import com.flipkart.ranger.core.finder.ServiceFinder; import com.flipkart.ranger.core.units.TestNodeData; import com.flipkart.ranger.core.utils.RangerTestUtils; import com.flipkart.ranger.core.utils.TestUtils; @@ -34,7 +35,7 @@ public void testUnshardedRangerHubClient(){ .nodeRefreshIntervalMs(5000) .build(); client.start(); - TestUtils.sleepUntil(() -> client.getHub().isStarted()); + TestUtils.sleepUntil(() -> client.getHub().getFinders().get().values().stream().allMatch(ServiceFinder::isStarted)); val service = RangerTestUtils.getService("test-n", "test-s"); Assert.assertNotNull(client.getNode(service).orElse(null)); Assert.assertNotNull(client.getNode(service, nodeData -> nodeData.getShardId() == 1).orElse(null)); diff --git a/ranger-server-bundle/src/test/java/com/flipkart/ranger/zk/server/bundle/RangerServerBundleTest.java b/ranger-server-bundle/src/test/java/com/flipkart/ranger/zk/server/bundle/RangerServerBundleTest.java index 9ebf5fff..cc7a8499 100644 --- a/ranger-server-bundle/src/test/java/com/flipkart/ranger/zk/server/bundle/RangerServerBundleTest.java +++ b/ranger-server-bundle/src/test/java/com/flipkart/ranger/zk/server/bundle/RangerServerBundleTest.java @@ -21,6 +21,7 @@ import com.flipkart.ranger.client.RangerHubClient; import com.flipkart.ranger.client.stubs.RangerTestHub; import com.flipkart.ranger.client.utils.RangerHubTestUtils; +import com.flipkart.ranger.core.finder.ServiceFinder; import com.flipkart.ranger.core.units.TestNodeData; import com.flipkart.ranger.core.utils.RangerTestUtils; import com.flipkart.ranger.core.utils.TestUtils; @@ -91,7 +92,7 @@ public void setup() throws Exception { public void testRangerBundle(){ var hub = rangerServerBundle.getHubs().get(0); Assert.assertTrue(hub instanceof RangerTestHub); - TestUtils.sleepUntil(() -> ((RangerTestHub) hub).getHub().isStarted()); + TestUtils.sleepUntil(() -> ((RangerTestHub) hub).getHub().getFinders().get().values().stream().allMatch(ServiceFinder::isStarted)); var node = hub.getNode(service).orElse(null); Assert.assertNotNull(node); Assert.assertTrue(node.getHost().equalsIgnoreCase("localhost")); diff --git a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/ShardedZKRangerClientTest.java b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/ShardedZKRangerClientTest.java index 0b828cdc..c9cd1ca0 100644 --- a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/ShardedZKRangerClientTest.java +++ b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/ShardedZKRangerClientTest.java @@ -15,6 +15,7 @@ */ package com.flipkart.ranger.client.zk; +import com.flipkart.ranger.core.finder.ServiceFinder; import com.flipkart.ranger.core.units.TestNodeData; import com.flipkart.ranger.core.utils.RangerTestUtils; import com.flipkart.ranger.core.utils.TestUtils; @@ -38,7 +39,7 @@ public void testShardedHub(){ .build(); zkHubClient.start(); - TestUtils.sleepUntil(() -> zkHubClient.getHub().isStarted()); + TestUtils.sleepUntil(() -> zkHubClient.getHub().getFinders().get().values().stream().allMatch(ServiceFinder::isStarted)); Assert.assertNotNull(zkHubClient.getNode(RangerTestUtils.getService("test-n", "s1")).orElse(null)); Assert.assertNotNull(zkHubClient.getNode(RangerTestUtils.getService("test-n", "s1"), nodeData -> nodeData.getShardId() == 1).orElse(null)); zkHubClient.stop(); diff --git a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/UnshardedZKRangerClientTest.java b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/UnshardedZKRangerClientTest.java index 28cb171b..a85ae756 100644 --- a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/UnshardedZKRangerClientTest.java +++ b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/UnshardedZKRangerClientTest.java @@ -15,6 +15,7 @@ */ package com.flipkart.ranger.client.zk; +import com.flipkart.ranger.core.finder.ServiceFinder; import com.flipkart.ranger.core.units.TestNodeData; import com.flipkart.ranger.core.utils.RangerTestUtils; import com.flipkart.ranger.core.utils.TestUtils; @@ -37,7 +38,7 @@ public void testShardedHub(){ .deserializer(this::read) .build(); zkHubClient.start(); - TestUtils.sleepUntil(() -> zkHubClient.getHub().isStarted()); + TestUtils.sleepUntil(() -> zkHubClient.getHub().getFinders().get().values().stream().allMatch(ServiceFinder::isStarted)); val service = RangerTestUtils.getService("test-n", "s1"); Assert.assertNotNull(zkHubClient.getNode(service).orElse(null)); Assert.assertNotNull(zkHubClient.getNode(service, nodeData -> nodeData.getShardId() == 1).orElse(null)); diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/servicehub/ServiceHubTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/servicehub/ServiceHubTest.java index 176e318b..b8b7cb47 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/servicehub/ServiceHubTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/servicehub/ServiceHubTest.java @@ -17,6 +17,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; +import com.flipkart.ranger.core.finder.ServiceFinder; import com.flipkart.ranger.core.finder.serviceregistry.MapBasedServiceRegistry; import com.flipkart.ranger.core.healthcheck.HealthcheckResult; import com.flipkart.ranger.core.healthcheck.HealthcheckStatus; @@ -118,7 +119,7 @@ public void testHub() { refreshProviderSignal.trigger(); refreshHubSignal.trigger(); - TestUtils.sleepUntil(hub::isStarted); + TestUtils.sleepUntil(() -> hub.getFinders().get().values().stream().allMatch(ServiceFinder::isStarted)); val node = hub.finder(RangerTestUtils.getService(NAMESPACE, "s1")) .flatMap(finder -> finder.get(nodeData -> nodeData.getShardId() == 1)).orElse(null); Assert.assertNotNull(node); From a15cbf8d3ef7194a6be0e261a0ad08049fa5228b Mon Sep 17 00:00:00 2001 From: Koushik R Date: Fri, 31 Dec 2021 17:41:26 +0530 Subject: [PATCH 56/69] Updated README --- README.md | 93 +++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 90 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 84f49451..a03914dd 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ Use the following maven dependency: com.flipkart.ranger ranger - 0.6.1 + 1.0-SNAPSHOT ``` ## How it works @@ -297,10 +297,97 @@ Stop the finder once you are done. (Generally this is when process ends) serviceFinder.stop() ``` +### Service Finder Hub +A service finder hub contains a collection of the above-mentioned service finders. A hub also makes creation of serviceFinders easy, for a service +that is dependent on multiple other services, don't have to now create multiple serviceFinders, instead create a hub, with the set of services and +the service finders get created automatically. + +Could either be an http hub or a ZK hub. (Can add any hub in the future) + +Hub clients for both ZK and http have been provided to initialize the same. A sample http hub client would look like the following. + +``` + RangerHubClient hubClient = UnshardedRangerZKHubClient.builder() + .namespace(rangerConfiguration.getNamespace()) + .connectionString(rangerConfiguration.getZookeeper()) + .curatorFramework(curatorFramework) + .disablePushUpdaters(rangerConfiguration.isDisablePushUpdaters()) + .mapper(getMapper()) + .refreshTimeMs(rangerConfiguration.getNodeRefreshTimeMs()) + .deserializer(data -> { + try { + return getMapper().readValue(data, new TypeReference>() { + }); + } catch (IOException e) { + log.warn("Error parsing node data with value {}", new String(data)); + } + return null; + }) + .services(Sets.newHashset("service1", "service2")) //Set of services here. + .build() + hubClient.start(); +``` + +Now you can find the service: + +``` +ServiceNode node = hubClient.get(new TestShardInfo(1)); +//Use host, port etc from the node +``` + +Stop the hub client once you are done. (Generally this is when process ends) + +``` +hubClient.stop() +``` + +### Ranger Server + +The earlier ranger's service finder construct operated on zookeeper as the datasource, the server has been introduced to support http data sources and to be +able to provide a serviceFinder interface atop multiple data sources. Eg: you could have one server running atop zk, one atop http - and can deploy another http +server fronting them both. Particularly useful when you have to aggregate amongst multiple service registries. A server bundle is provided to start a quick server (atop dropwizard) + +To use the http server bundle along with boostrap use. + ``` + bootstrap.add(new RangerServerBundle { + @Override + protected List> withHubs(AppConfiguration configuration) { + val rangerConfiguration = configuration.getRangerConfiguration(); + return rangerConfiguration.getHttpClientConfigs().stream().map(clientConfig -> UnshardedRangerHttpHubClient.builder() + .namespace(rangerConfiguration.getNamespace()) + .mapper(getMapper()) + .clientConfig(clientConfig) + .nodeRefreshIntervalMs(rangerConfiguration.getNodeRefreshTimeMs()) + .deserializer(data -> { + try { + return getMapper().readValue(data, new TypeReference>() { + }); + } catch (IOException e) { + log.warn("Error parsing node data with value {}", new String(data)); + } + return null; + }) + .build()).collect(Collectors.toList()); + } + + @Override + protected boolean withInitialRotationStatus(AppConfiguration configuration) { + return configuration.isInitialRotationStatus(); + } + + @Override + protected List withHealthChecks(AppConfiguration configuration) { + return ImmutableList.of(new RangerHttpHealthCheck()); + } + }); +``` + +It comes with a rangerResource that provides you with interfaces for getting the list of services across hubs and the nodes per service across hubs. + + Version ---- - -0.3.0 +1.0-SNAPSHOT Tech ----------- From f6ad95f0a8b33b14a86f80f163462c0d52025aee Mon Sep 17 00:00:00 2001 From: Koushik R Date: Fri, 31 Dec 2021 18:15:38 +0530 Subject: [PATCH 57/69] Changed the utils to remove the useless started flag in finder --- .../ranger/client/AbstractRangerHubClientTest.java | 2 +- .../flipkart/ranger/core/finder/ServiceFinder.java | 4 ---- .../com/flipkart/ranger/core/utils/TestUtils.java | 13 +++++++++++++ .../client/http/ShardedRangerHttpClientTest.java | 3 +-- .../client/http/SimpleRangerHttpClientTest.java | 2 +- .../client/http/UnshardedRangerHttpClientTest.java | 2 +- .../HttpShardedServiceFinderBuilderTest.java | 2 +- .../zk/server/bundle/RangerServerBundleTest.java | 2 +- .../ranger/client/zk/ShardedZKRangerClientTest.java | 3 +-- .../client/zk/UnshardedZKRangerClientTest.java | 2 +- .../ServiceProviderIntegrationTest.java | 7 +++---- .../ranger/zookeeper/servicehub/ServiceHubTest.java | 2 +- 12 files changed, 25 insertions(+), 19 deletions(-) diff --git a/ranger-client/src/test/java/com/flipkart/ranger/client/AbstractRangerHubClientTest.java b/ranger-client/src/test/java/com/flipkart/ranger/client/AbstractRangerHubClientTest.java index b35f5ea3..d78c2da0 100644 --- a/ranger-client/src/test/java/com/flipkart/ranger/client/AbstractRangerHubClientTest.java +++ b/ranger-client/src/test/java/com/flipkart/ranger/client/AbstractRangerHubClientTest.java @@ -35,7 +35,7 @@ public class AbstractRangerHubClientTest { public void testAbstractHubClient() { val testAbstractHub = RangerHubTestUtils.getTestHub(); testAbstractHub.start(); - TestUtils.sleepUntil(() -> testAbstractHub.getHub().getFinders().get().values().stream().allMatch(ServiceFinder::isStarted)); + TestUtils.sleepUntilHubIsActive(testAbstractHub.getHub()); var node = testAbstractHub.getNode(service).orElse(null); Assert.assertNotNull(node); Assert.assertTrue(node.getHost().equalsIgnoreCase("localhost")); diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/ServiceFinder.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/ServiceFinder.java index 51d74314..7ded56ce 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/ServiceFinder.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/ServiceFinder.java @@ -40,8 +40,6 @@ public abstract class ServiceFinder startSignal = new ExternalTriggeredSignal<>(() -> null, Collections.emptyList()); @Getter private final ExternalTriggeredSignal stopSignal = new ExternalTriggeredSignal<>(() -> null, Collections.emptyList()); - @Getter - private boolean started = false; protected ServiceFinder( ServiceRegistryType serviceRegistry, @@ -63,11 +61,9 @@ public List> getAll(Predicate criteria) { public void start() { startSignal.trigger(); - started = true; } public void stop() { stopSignal.trigger(); - started = false; } } diff --git a/ranger-core/src/test/java/com/flipkart/ranger/core/utils/TestUtils.java b/ranger-core/src/test/java/com/flipkart/ranger/core/utils/TestUtils.java index ff9004c9..777e15c2 100644 --- a/ranger-core/src/test/java/com/flipkart/ranger/core/utils/TestUtils.java +++ b/ranger-core/src/test/java/com/flipkart/ranger/core/utils/TestUtils.java @@ -15,10 +15,14 @@ */ package com.flipkart.ranger.core.utils; +import com.flipkart.ranger.core.finder.ServiceFinder; +import com.flipkart.ranger.core.finderhub.ServiceFinderHub; +import com.flipkart.ranger.core.model.ServiceRegistry; import lombok.experimental.UtilityClass; import java.util.concurrent.Callable; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import static org.awaitility.Awaitility.await; @@ -40,4 +44,13 @@ public static void sleepUntil(int numSeconds) { public static void sleepUntil(Callable conditionEvaluator) { await().until(conditionEvaluator); } + + public static > void sleepUntilFinderIsActive(ServiceFinder finder){ + await().untilAsserted(finder::start); + } + + public static > void sleepUntilHubIsActive(ServiceFinderHub hub){ + await().untilAsserted(() -> hub.getFinders().get().values().forEach(ServiceFinder::start)); + await().untilAsserted(() -> hub.getServiceDataSource().start()); + } } diff --git a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/ShardedRangerHttpClientTest.java b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/ShardedRangerHttpClientTest.java index 14a083e3..2b7a5e08 100644 --- a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/ShardedRangerHttpClientTest.java +++ b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/ShardedRangerHttpClientTest.java @@ -35,8 +35,7 @@ public void testShardedHttpHubClient(){ .nodeRefreshIntervalMs(5000) .build(); client.start(); - - TestUtils.sleepUntil(() -> client.getHub().getFinders().get().values().stream().allMatch(ServiceFinder::isStarted)); + TestUtils.sleepUntilHubIsActive(client.getHub()); val service = RangerTestUtils.getService("test-n", "test-s"); Assert.assertNotNull(client.getNode(service).orElse(null)); Assert.assertNotNull(client.getNode(service, nodeData -> nodeData.getShardId() == 1).orElse(null)); diff --git a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/SimpleRangerHttpClientTest.java b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/SimpleRangerHttpClientTest.java index 36b696d5..0fb4fa97 100644 --- a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/SimpleRangerHttpClientTest.java +++ b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/SimpleRangerHttpClientTest.java @@ -34,7 +34,7 @@ public void testSimpleHttpRangerClient(){ .nodeRefreshIntervalMs(5000) .build(); client.start(); - TestUtils.sleepUntil(() -> client.getServiceFinder().isStarted()); + TestUtils.sleepUntilFinderIsActive(client.getServiceFinder()); Assert.assertNotNull(client.getNode().orElse(null)); Assert.assertNotNull(client.getNode(nodeData -> nodeData.getShardId() == 1).orElse(null)); Assert.assertNull(client.getNode(nodeData -> nodeData.getShardId() == 2).orElse(null)); diff --git a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/UnshardedRangerHttpClientTest.java b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/UnshardedRangerHttpClientTest.java index 21760d75..bd8cf941 100644 --- a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/UnshardedRangerHttpClientTest.java +++ b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/UnshardedRangerHttpClientTest.java @@ -35,7 +35,7 @@ public void testUnshardedRangerHubClient(){ .nodeRefreshIntervalMs(5000) .build(); client.start(); - TestUtils.sleepUntil(() -> client.getHub().getFinders().get().values().stream().allMatch(ServiceFinder::isStarted)); + TestUtils.sleepUntilHubIsActive(client.getHub()); val service = RangerTestUtils.getService("test-n", "test-s"); Assert.assertNotNull(client.getNode(service).orElse(null)); Assert.assertNotNull(client.getNode(service, nodeData -> nodeData.getShardId() == 1).orElse(null)); diff --git a/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java b/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java index bae5a7a7..06e0a429 100644 --- a/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java +++ b/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java @@ -91,7 +91,7 @@ public void testFinder() throws Exception { .withShardSelector((criteria, registry) -> registry.nodeList()) .build(); finder.start(); - TestUtils.sleepUntil(finder::isStarted); + TestUtils.sleepUntilFinderIsActive(finder); Assert.assertTrue(finder.get(nodeData -> true).isPresent()); } diff --git a/ranger-server-bundle/src/test/java/com/flipkart/ranger/zk/server/bundle/RangerServerBundleTest.java b/ranger-server-bundle/src/test/java/com/flipkart/ranger/zk/server/bundle/RangerServerBundleTest.java index cc7a8499..b592b62b 100644 --- a/ranger-server-bundle/src/test/java/com/flipkart/ranger/zk/server/bundle/RangerServerBundleTest.java +++ b/ranger-server-bundle/src/test/java/com/flipkart/ranger/zk/server/bundle/RangerServerBundleTest.java @@ -92,7 +92,7 @@ public void setup() throws Exception { public void testRangerBundle(){ var hub = rangerServerBundle.getHubs().get(0); Assert.assertTrue(hub instanceof RangerTestHub); - TestUtils.sleepUntil(() -> ((RangerTestHub) hub).getHub().getFinders().get().values().stream().allMatch(ServiceFinder::isStarted)); + TestUtils.sleepUntilHubIsActive(((RangerTestHub) hub).getHub()); var node = hub.getNode(service).orElse(null); Assert.assertNotNull(node); Assert.assertTrue(node.getHost().equalsIgnoreCase("localhost")); diff --git a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/ShardedZKRangerClientTest.java b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/ShardedZKRangerClientTest.java index c9cd1ca0..ec28f8b1 100644 --- a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/ShardedZKRangerClientTest.java +++ b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/ShardedZKRangerClientTest.java @@ -37,9 +37,8 @@ public void testShardedHub(){ .mapper(getObjectMapper()) .deserializer(this::read) .build(); - zkHubClient.start(); - TestUtils.sleepUntil(() -> zkHubClient.getHub().getFinders().get().values().stream().allMatch(ServiceFinder::isStarted)); + TestUtils.sleepUntilHubIsActive(zkHubClient.getHub()); Assert.assertNotNull(zkHubClient.getNode(RangerTestUtils.getService("test-n", "s1")).orElse(null)); Assert.assertNotNull(zkHubClient.getNode(RangerTestUtils.getService("test-n", "s1"), nodeData -> nodeData.getShardId() == 1).orElse(null)); zkHubClient.stop(); diff --git a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/UnshardedZKRangerClientTest.java b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/UnshardedZKRangerClientTest.java index a85ae756..eee4308b 100644 --- a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/UnshardedZKRangerClientTest.java +++ b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/UnshardedZKRangerClientTest.java @@ -38,7 +38,7 @@ public void testShardedHub(){ .deserializer(this::read) .build(); zkHubClient.start(); - TestUtils.sleepUntil(() -> zkHubClient.getHub().getFinders().get().values().stream().allMatch(ServiceFinder::isStarted)); + TestUtils.sleepUntilHubIsActive(zkHubClient.getHub()); val service = RangerTestUtils.getService("test-n", "s1"); Assert.assertNotNull(zkHubClient.getNode(service).orElse(null)); Assert.assertNotNull(zkHubClient.getNode(service, nodeData -> nodeData.getShardId() == 1).orElse(null)); diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/ServiceProviderIntegrationTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/ServiceProviderIntegrationTest.java index 215be99b..7b05b0d4 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/ServiceProviderIntegrationTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/ServiceProviderIntegrationTest.java @@ -99,7 +99,6 @@ public void stopTestCluster() throws Exception { testingCluster.close(); } serviceFinder.stop(); - TestUtils.sleepUntil(() -> !serviceFinder.isStarted()); } @Test @@ -112,7 +111,7 @@ public void testBasicDiscovery() throws Exception { /* with file existing, 3 nodes should be healthy */ var filecreate = file.createNewFile(); System.out.println("created file"); - TestUtils.sleepUntil(3); + TestUtils.sleepUntil(5); List> all = serviceFinder.getAll(null); System.out.println("all = " + all); Assert.assertEquals(3, all.size()); @@ -120,7 +119,7 @@ public void testBasicDiscovery() throws Exception { /* with file deleted, all 3 nodes should be unhealthy */ delete = file.delete(); System.out.println("deleted file"); - TestUtils.sleepUntil(3); + TestUtils.sleepUntil(5); all = serviceFinder.getAll(null); System.out.println("all = " + all); Assert.assertEquals(0, all.size()); @@ -128,7 +127,7 @@ public void testBasicDiscovery() throws Exception { /* with anotherFile created, the 4th node should become healthy and discoverable */ filecreate = anotherFile.createNewFile(); System.out.println("created anotherFile"); - TestUtils.sleepUntil(3); + TestUtils.sleepUntil(5); all = serviceFinder.getAll(null); System.out.println("all = " + all); Assert.assertEquals(1, all.size()); diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/servicehub/ServiceHubTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/servicehub/ServiceHubTest.java index b8b7cb47..f4459a42 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/servicehub/ServiceHubTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/servicehub/ServiceHubTest.java @@ -118,8 +118,8 @@ public void testHub() { refreshProviderSignal.trigger(); refreshHubSignal.trigger(); + TestUtils.sleepUntilHubIsActive(hub); - TestUtils.sleepUntil(() -> hub.getFinders().get().values().stream().allMatch(ServiceFinder::isStarted)); val node = hub.finder(RangerTestUtils.getService(NAMESPACE, "s1")) .flatMap(finder -> finder.get(nodeData -> nodeData.getShardId() == 1)).orElse(null); Assert.assertNotNull(node); From 89022d236b1238fc2158ef0580fd334652d30f70 Mon Sep 17 00:00:00 2001 From: Koushik R Date: Tue, 4 Jan 2022 18:19:20 +0530 Subject: [PATCH 58/69] Fixed the typo in serializer spelling --- .../java/com/flipkart/ranger/client/stubs/RangerTestHub.java | 4 ++-- .../com/flipkart/ranger/client/utils/RangerHubTestUtils.java | 2 +- .../java/com/flipkart/ranger/core/finder/ServiceFinder.java | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/RangerTestHub.java b/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/RangerTestHub.java index 6e3a8500..db1c362b 100644 --- a/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/RangerTestHub.java +++ b/ranger-client/src/test/java/com/flipkart/ranger/client/stubs/RangerTestHub.java @@ -33,8 +33,8 @@ public class RangerTestHub extends AbstractRangerHubClient criteria, TestDeserializer deserilizer) { - super(namespace, mapper, nodeRefreshTimeMs, criteria, deserilizer, false); + Predicate criteria, TestDeserializer deserializer) { + super(namespace, mapper, nodeRefreshTimeMs, criteria, deserializer, false); } @Override diff --git a/ranger-client/src/test/java/com/flipkart/ranger/client/utils/RangerHubTestUtils.java b/ranger-client/src/test/java/com/flipkart/ranger/client/utils/RangerHubTestUtils.java index 180d95b1..94780785 100644 --- a/ranger-client/src/test/java/com/flipkart/ranger/client/utils/RangerHubTestUtils.java +++ b/ranger-client/src/test/java/com/flipkart/ranger/client/utils/RangerHubTestUtils.java @@ -35,7 +35,7 @@ public static RangerTestHub getTestHub(){ .mapper(mapper) .nodeRefreshTimeMs(1000) .criteria(new TestCriteria()) - .deserilizer(new TestDeserializer<>()) + .deserializer(new TestDeserializer<>()) .build(); } diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/ServiceFinder.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/ServiceFinder.java index 7ded56ce..e8829896 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/ServiceFinder.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/ServiceFinder.java @@ -23,6 +23,7 @@ import com.flipkart.ranger.core.signals.ExternalTriggeredSignal; import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import lombok.val; import lombok.var; import java.util.Collections; @@ -51,7 +52,7 @@ protected ServiceFinder( } public Optional> get(Predicate criteria) { - var nodes = shardSelector.nodes(criteria, serviceRegistry); + val nodes = shardSelector.nodes(criteria, serviceRegistry); return null == nodes || nodes.isEmpty() ? Optional.empty() : Optional.of(nodeSelector.select(nodes)); } From a1658e89f1755c79e343f6a1b9a12cd82ac09e33 Mon Sep 17 00:00:00 2001 From: Koushik R Date: Tue, 4 Jan 2022 20:44:20 +0530 Subject: [PATCH 59/69] Removed the unnecessary imports --- .../com/flipkart/ranger/client/AbstractRangerHubClientTest.java | 1 - .../main/java/com/flipkart/ranger/core/finder/ServiceFinder.java | 1 - .../src/test/java/com/flipkart/ranger/core/utils/TestUtils.java | 1 - .../flipkart/ranger/client/http/ShardedRangerHttpClientTest.java | 1 - .../ranger/client/http/UnshardedRangerHttpClientTest.java | 1 - .../flipkart/ranger/zk/server/bundle/RangerServerBundleTest.java | 1 - .../com/flipkart/ranger/client/zk/ShardedZKRangerClientTest.java | 1 - .../flipkart/ranger/client/zk/UnshardedZKRangerClientTest.java | 1 - .../com/flipkart/ranger/zookeeper/servicehub/ServiceHubTest.java | 1 - 9 files changed, 9 deletions(-) diff --git a/ranger-client/src/test/java/com/flipkart/ranger/client/AbstractRangerHubClientTest.java b/ranger-client/src/test/java/com/flipkart/ranger/client/AbstractRangerHubClientTest.java index d78c2da0..e7d61bb4 100644 --- a/ranger-client/src/test/java/com/flipkart/ranger/client/AbstractRangerHubClientTest.java +++ b/ranger-client/src/test/java/com/flipkart/ranger/client/AbstractRangerHubClientTest.java @@ -16,7 +16,6 @@ package com.flipkart.ranger.client; import com.flipkart.ranger.client.utils.RangerHubTestUtils; -import com.flipkart.ranger.core.finder.ServiceFinder; import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.core.utils.RangerTestUtils; import com.flipkart.ranger.core.utils.TestUtils; diff --git a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/ServiceFinder.java b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/ServiceFinder.java index e8829896..b3e95994 100644 --- a/ranger-core/src/main/java/com/flipkart/ranger/core/finder/ServiceFinder.java +++ b/ranger-core/src/main/java/com/flipkart/ranger/core/finder/ServiceFinder.java @@ -24,7 +24,6 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; import lombok.val; -import lombok.var; import java.util.Collections; import java.util.List; diff --git a/ranger-core/src/test/java/com/flipkart/ranger/core/utils/TestUtils.java b/ranger-core/src/test/java/com/flipkart/ranger/core/utils/TestUtils.java index 777e15c2..215a3a33 100644 --- a/ranger-core/src/test/java/com/flipkart/ranger/core/utils/TestUtils.java +++ b/ranger-core/src/test/java/com/flipkart/ranger/core/utils/TestUtils.java @@ -22,7 +22,6 @@ import java.util.concurrent.Callable; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; import static org.awaitility.Awaitility.await; diff --git a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/ShardedRangerHttpClientTest.java b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/ShardedRangerHttpClientTest.java index 2b7a5e08..03ffa0fa 100644 --- a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/ShardedRangerHttpClientTest.java +++ b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/ShardedRangerHttpClientTest.java @@ -15,7 +15,6 @@ */ package com.flipkart.ranger.client.http; -import com.flipkart.ranger.core.finder.ServiceFinder; import com.flipkart.ranger.core.units.TestNodeData; import com.flipkart.ranger.core.utils.RangerTestUtils; import com.flipkart.ranger.core.utils.TestUtils; diff --git a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/UnshardedRangerHttpClientTest.java b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/UnshardedRangerHttpClientTest.java index bd8cf941..28daf7c0 100644 --- a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/UnshardedRangerHttpClientTest.java +++ b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/UnshardedRangerHttpClientTest.java @@ -15,7 +15,6 @@ */ package com.flipkart.ranger.client.http; -import com.flipkart.ranger.core.finder.ServiceFinder; import com.flipkart.ranger.core.units.TestNodeData; import com.flipkart.ranger.core.utils.RangerTestUtils; import com.flipkart.ranger.core.utils.TestUtils; diff --git a/ranger-server-bundle/src/test/java/com/flipkart/ranger/zk/server/bundle/RangerServerBundleTest.java b/ranger-server-bundle/src/test/java/com/flipkart/ranger/zk/server/bundle/RangerServerBundleTest.java index b592b62b..9c5e7f48 100644 --- a/ranger-server-bundle/src/test/java/com/flipkart/ranger/zk/server/bundle/RangerServerBundleTest.java +++ b/ranger-server-bundle/src/test/java/com/flipkart/ranger/zk/server/bundle/RangerServerBundleTest.java @@ -21,7 +21,6 @@ import com.flipkart.ranger.client.RangerHubClient; import com.flipkart.ranger.client.stubs.RangerTestHub; import com.flipkart.ranger.client.utils.RangerHubTestUtils; -import com.flipkart.ranger.core.finder.ServiceFinder; import com.flipkart.ranger.core.units.TestNodeData; import com.flipkart.ranger.core.utils.RangerTestUtils; import com.flipkart.ranger.core.utils.TestUtils; diff --git a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/ShardedZKRangerClientTest.java b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/ShardedZKRangerClientTest.java index ec28f8b1..6124ee9f 100644 --- a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/ShardedZKRangerClientTest.java +++ b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/ShardedZKRangerClientTest.java @@ -15,7 +15,6 @@ */ package com.flipkart.ranger.client.zk; -import com.flipkart.ranger.core.finder.ServiceFinder; import com.flipkart.ranger.core.units.TestNodeData; import com.flipkart.ranger.core.utils.RangerTestUtils; import com.flipkart.ranger.core.utils.TestUtils; diff --git a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/UnshardedZKRangerClientTest.java b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/UnshardedZKRangerClientTest.java index eee4308b..d17ca8a4 100644 --- a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/UnshardedZKRangerClientTest.java +++ b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/UnshardedZKRangerClientTest.java @@ -15,7 +15,6 @@ */ package com.flipkart.ranger.client.zk; -import com.flipkart.ranger.core.finder.ServiceFinder; import com.flipkart.ranger.core.units.TestNodeData; import com.flipkart.ranger.core.utils.RangerTestUtils; import com.flipkart.ranger.core.utils.TestUtils; diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/servicehub/ServiceHubTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/servicehub/ServiceHubTest.java index f4459a42..aa5337d4 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/servicehub/ServiceHubTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/servicehub/ServiceHubTest.java @@ -17,7 +17,6 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import com.flipkart.ranger.core.finder.ServiceFinder; import com.flipkart.ranger.core.finder.serviceregistry.MapBasedServiceRegistry; import com.flipkart.ranger.core.healthcheck.HealthcheckResult; import com.flipkart.ranger.core.healthcheck.HealthcheckStatus; From 969db5e2755a3806103be15e5833a0c753d5d39a Mon Sep 17 00:00:00 2001 From: Koushik R Date: Tue, 4 Jan 2022 20:44:44 +0530 Subject: [PATCH 60/69] Changed the order of hub assertions --- .../src/test/java/com/flipkart/ranger/core/utils/TestUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ranger-core/src/test/java/com/flipkart/ranger/core/utils/TestUtils.java b/ranger-core/src/test/java/com/flipkart/ranger/core/utils/TestUtils.java index 215a3a33..42a19f75 100644 --- a/ranger-core/src/test/java/com/flipkart/ranger/core/utils/TestUtils.java +++ b/ranger-core/src/test/java/com/flipkart/ranger/core/utils/TestUtils.java @@ -49,7 +49,7 @@ public static > void sleepUntilFinderIsActive(Se } public static > void sleepUntilHubIsActive(ServiceFinderHub hub){ - await().untilAsserted(() -> hub.getFinders().get().values().forEach(ServiceFinder::start)); await().untilAsserted(() -> hub.getServiceDataSource().start()); + await().untilAsserted(() -> hub.getFinders().get().values().forEach(ServiceFinder::start)); } } From 18647f715fd5c6984591028c3182b5b940ed9177 Mon Sep 17 00:00:00 2001 From: Koushik R Date: Tue, 4 Jan 2022 20:58:14 +0530 Subject: [PATCH 61/69] Added awaitility with seconds and bounds --- .../com/flipkart/ranger/core/utils/TestUtils.java | 13 ++++++++----- .../healthservice/ServiceHealthAggregatorTest.java | 6 +++--- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/ranger-core/src/test/java/com/flipkart/ranger/core/utils/TestUtils.java b/ranger-core/src/test/java/com/flipkart/ranger/core/utils/TestUtils.java index 42a19f75..b7af4686 100644 --- a/ranger-core/src/test/java/com/flipkart/ranger/core/utils/TestUtils.java +++ b/ranger-core/src/test/java/com/flipkart/ranger/core/utils/TestUtils.java @@ -31,6 +31,9 @@ @UtilityClass public class TestUtils { + static final int HUB_ACTIVE_DURATION_SECONDS = 3; + static final int FINDER_ACTIVE_DURATION_SECONDS = 3; + /* If we know the upper bound condition, please use the until with the upper bound. Only for cases, where you have to wait till the refreshInterval periods, don't want to introduce @@ -40,16 +43,16 @@ public static void sleepUntil(int numSeconds) { await().pollDelay(numSeconds, TimeUnit.SECONDS).until(() -> true); } - public static void sleepUntil(Callable conditionEvaluator) { - await().until(conditionEvaluator); + public static void sleepUntil(int numSeconds, Callable conditionEvaluator) { + await().pollDelay(numSeconds, TimeUnit.SECONDS).until(conditionEvaluator); } public static > void sleepUntilFinderIsActive(ServiceFinder finder){ - await().untilAsserted(finder::start); + await().pollDelay(FINDER_ACTIVE_DURATION_SECONDS, TimeUnit.SECONDS).untilAsserted(finder::start); } public static > void sleepUntilHubIsActive(ServiceFinderHub hub){ - await().untilAsserted(() -> hub.getServiceDataSource().start()); - await().untilAsserted(() -> hub.getFinders().get().values().forEach(ServiceFinder::start)); + await().pollDelay(HUB_ACTIVE_DURATION_SECONDS, TimeUnit.SECONDS).untilAsserted(() -> hub.getServiceDataSource().start()); + await().pollDelay(HUB_ACTIVE_DURATION_SECONDS, TimeUnit.SECONDS).untilAsserted(() -> hub.getFinders().get().values().forEach(ServiceFinder::start)); } } diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/ServiceHealthAggregatorTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/ServiceHealthAggregatorTest.java index 29e55177..9c87c0d1 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/ServiceHealthAggregatorTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/ServiceHealthAggregatorTest.java @@ -50,7 +50,7 @@ public boolean isDisabled() { }); serviceHealthAggregator.start(); - TestUtils.sleepUntil(() -> serviceHealthAggregator.getRunning().get()); + TestUtils.sleepUntil(3, () -> serviceHealthAggregator.getRunning().get()); } @After @@ -64,14 +64,14 @@ public void testStaleRun() { testMonitor.run(); testMonitor.setThreadSleep(2000); - TestUtils.sleepUntil(() -> !testMonitor.hasValidUpdatedTime(new Date())); + TestUtils.sleepUntil(3, () -> !testMonitor.hasValidUpdatedTime(new Date())); /* in the TestMonitor, thread was sleeping for 2 seconds, */ /* so its state is supposed to be stale (>1 second) and service has to be unhealthy */ Assert.assertEquals(HealthcheckStatus.unhealthy, serviceHealthAggregator.getServiceHealth()); testMonitor.setThreadSleep(5); - TestUtils.sleepUntil(() -> testMonitor.hasValidUpdatedTime(new Date())); + TestUtils.sleepUntil(3, () -> testMonitor.hasValidUpdatedTime(new Date())); /* in the TestMonitor, thread is sleeping only for 10 milliseconds, */ /* so its state is supposed to be NOT stale (>1 second) and service has to be healthy */ From 5d9cfa9b04003bb8a911be66d5157da0a1c059f7 Mon Sep 17 00:00:00 2001 From: Koushik R Date: Tue, 4 Jan 2022 21:07:29 +0530 Subject: [PATCH 62/69] Introduced an atMost to on await --- .../test/java/com/flipkart/ranger/core/utils/TestUtils.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ranger-core/src/test/java/com/flipkart/ranger/core/utils/TestUtils.java b/ranger-core/src/test/java/com/flipkart/ranger/core/utils/TestUtils.java index b7af4686..402cf989 100644 --- a/ranger-core/src/test/java/com/flipkart/ranger/core/utils/TestUtils.java +++ b/ranger-core/src/test/java/com/flipkart/ranger/core/utils/TestUtils.java @@ -48,11 +48,11 @@ public static void sleepUntil(int numSeconds, Callable conditionEvaluat } public static > void sleepUntilFinderIsActive(ServiceFinder finder){ - await().pollDelay(FINDER_ACTIVE_DURATION_SECONDS, TimeUnit.SECONDS).untilAsserted(finder::start); + await().atMost(FINDER_ACTIVE_DURATION_SECONDS, TimeUnit.SECONDS).untilAsserted(finder::start); } public static > void sleepUntilHubIsActive(ServiceFinderHub hub){ - await().pollDelay(HUB_ACTIVE_DURATION_SECONDS, TimeUnit.SECONDS).untilAsserted(() -> hub.getServiceDataSource().start()); - await().pollDelay(HUB_ACTIVE_DURATION_SECONDS, TimeUnit.SECONDS).untilAsserted(() -> hub.getFinders().get().values().forEach(ServiceFinder::start)); + await().atMost(HUB_ACTIVE_DURATION_SECONDS, TimeUnit.SECONDS).untilAsserted(() -> hub.getServiceDataSource().start()); + await().atMost(HUB_ACTIVE_DURATION_SECONDS, TimeUnit.SECONDS).untilAsserted(() -> hub.getFinders().get().values().forEach(ServiceFinder::start)); } } From 4591194dc51ae71b7683bd0a9eccfd49c0fcd460 Mon Sep 17 00:00:00 2001 From: Koushik R Date: Tue, 4 Jan 2022 21:45:02 +0530 Subject: [PATCH 63/69] Awaitility intervals coded for finder and hub starts --- .../client/utils/RangerHubTestUtils.java | 1 + .../flipkart/ranger/core/utils/TestUtils.java | 19 +++++++++++++------ .../http/ShardedRangerHttpClientTest.java | 2 +- .../http/SimpleRangerHttpClientTest.java | 2 +- .../http/UnshardedRangerHttpClientTest.java | 2 +- .../HttpShardedServiceFinderBuilderTest.java | 1 + .../client/zk/ShardedZKRangerClientTest.java | 1 + .../zk/UnshardedZKRangerClientTest.java | 1 + 8 files changed, 20 insertions(+), 9 deletions(-) diff --git a/ranger-client/src/test/java/com/flipkart/ranger/client/utils/RangerHubTestUtils.java b/ranger-client/src/test/java/com/flipkart/ranger/client/utils/RangerHubTestUtils.java index 94780785..95396578 100644 --- a/ranger-client/src/test/java/com/flipkart/ranger/client/utils/RangerHubTestUtils.java +++ b/ranger-client/src/test/java/com/flipkart/ranger/client/utils/RangerHubTestUtils.java @@ -36,6 +36,7 @@ public static RangerTestHub getTestHub(){ .nodeRefreshTimeMs(1000) .criteria(new TestCriteria()) .deserializer(new TestDeserializer<>()) + .nodeRefreshTimeMs(1000) .build(); } diff --git a/ranger-core/src/test/java/com/flipkart/ranger/core/utils/TestUtils.java b/ranger-core/src/test/java/com/flipkart/ranger/core/utils/TestUtils.java index 402cf989..de3b4c29 100644 --- a/ranger-core/src/test/java/com/flipkart/ranger/core/utils/TestUtils.java +++ b/ranger-core/src/test/java/com/flipkart/ranger/core/utils/TestUtils.java @@ -19,7 +19,9 @@ import com.flipkart.ranger.core.finderhub.ServiceFinderHub; import com.flipkart.ranger.core.model.ServiceRegistry; import lombok.experimental.UtilityClass; +import org.awaitility.core.ThrowingRunnable; +import java.time.Duration; import java.util.concurrent.Callable; import java.util.concurrent.TimeUnit; @@ -31,9 +33,6 @@ @UtilityClass public class TestUtils { - static final int HUB_ACTIVE_DURATION_SECONDS = 3; - static final int FINDER_ACTIVE_DURATION_SECONDS = 3; - /* If we know the upper bound condition, please use the until with the upper bound. Only for cases, where you have to wait till the refreshInterval periods, don't want to introduce @@ -47,12 +46,20 @@ public static void sleepUntil(int numSeconds, Callable conditionEvaluat await().pollDelay(numSeconds, TimeUnit.SECONDS).until(conditionEvaluator); } + /* + Use this when you have to alter the numSeconds in any of the specific assertions. For finder and hub, the values are appropriately coded + keeping the start intervals in mind. + */ + public static void sleepUntil(int numSeconds, ThrowingRunnable assertion){ + await().pollDelay(Duration.ofSeconds(numSeconds)).untilAsserted(assertion); + } + public static > void sleepUntilFinderIsActive(ServiceFinder finder){ - await().atMost(FINDER_ACTIVE_DURATION_SECONDS, TimeUnit.SECONDS).untilAsserted(finder::start); + sleepUntil(3, () -> finder.getStartSignal().start()); } public static > void sleepUntilHubIsActive(ServiceFinderHub hub){ - await().atMost(HUB_ACTIVE_DURATION_SECONDS, TimeUnit.SECONDS).untilAsserted(() -> hub.getServiceDataSource().start()); - await().atMost(HUB_ACTIVE_DURATION_SECONDS, TimeUnit.SECONDS).untilAsserted(() -> hub.getFinders().get().values().forEach(ServiceFinder::start)); + sleepUntil(3, () -> hub.getServiceDataSource().start()); + sleepUntil(3, () -> hub.getFinders().get().values().forEach(finder -> finder.getStartSignal().start())); } } diff --git a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/ShardedRangerHttpClientTest.java b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/ShardedRangerHttpClientTest.java index 03ffa0fa..bf252099 100644 --- a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/ShardedRangerHttpClientTest.java +++ b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/ShardedRangerHttpClientTest.java @@ -31,7 +31,7 @@ public void testShardedHttpHubClient(){ .namespace("test-n") .deserializer(this::read) .mapper(getObjectMapper()) - .nodeRefreshIntervalMs(5000) + .nodeRefreshIntervalMs(1000) .build(); client.start(); TestUtils.sleepUntilHubIsActive(client.getHub()); diff --git a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/SimpleRangerHttpClientTest.java b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/SimpleRangerHttpClientTest.java index 0fb4fa97..af6a1cfe 100644 --- a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/SimpleRangerHttpClientTest.java +++ b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/SimpleRangerHttpClientTest.java @@ -31,7 +31,7 @@ public void testSimpleHttpRangerClient(){ .deserializer(this::read) .namespace("test-n") .serviceName("test-s") - .nodeRefreshIntervalMs(5000) + .nodeRefreshIntervalMs(1000) .build(); client.start(); TestUtils.sleepUntilFinderIsActive(client.getServiceFinder()); diff --git a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/UnshardedRangerHttpClientTest.java b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/UnshardedRangerHttpClientTest.java index 28daf7c0..a9eae555 100644 --- a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/UnshardedRangerHttpClientTest.java +++ b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/UnshardedRangerHttpClientTest.java @@ -31,7 +31,7 @@ public void testUnshardedRangerHubClient(){ .namespace("test-n") .deserializer(this::read) .mapper(getObjectMapper()) - .nodeRefreshIntervalMs(5000) + .nodeRefreshIntervalMs(1000) .build(); client.start(); TestUtils.sleepUntilHubIsActive(client.getHub()); diff --git a/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java b/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java index 06e0a429..2b1a7e39 100644 --- a/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java +++ b/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java @@ -89,6 +89,7 @@ public void testFinder() throws Exception { } }) .withShardSelector((criteria, registry) -> registry.nodeList()) + .withNodeRefreshIntervalMs(1000) .build(); finder.start(); TestUtils.sleepUntilFinderIsActive(finder); diff --git a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/ShardedZKRangerClientTest.java b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/ShardedZKRangerClientTest.java index 6124ee9f..ae3d6bd2 100644 --- a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/ShardedZKRangerClientTest.java +++ b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/ShardedZKRangerClientTest.java @@ -35,6 +35,7 @@ public void testShardedHub(){ .disablePushUpdaters(true) .mapper(getObjectMapper()) .deserializer(this::read) + .refreshTimeMs(1000) .build(); zkHubClient.start(); TestUtils.sleepUntilHubIsActive(zkHubClient.getHub()); diff --git a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/UnshardedZKRangerClientTest.java b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/UnshardedZKRangerClientTest.java index d17ca8a4..4d648dd8 100644 --- a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/UnshardedZKRangerClientTest.java +++ b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/UnshardedZKRangerClientTest.java @@ -35,6 +35,7 @@ public void testShardedHub(){ .disablePushUpdaters(true) .mapper(getObjectMapper()) .deserializer(this::read) + .refreshTimeMs(1000) .build(); zkHubClient.start(); TestUtils.sleepUntilHubIsActive(zkHubClient.getHub()); From 81bcfd79adfa84a49854756c5419281247dcc682 Mon Sep 17 00:00:00 2001 From: Koushik R Date: Tue, 4 Jan 2022 22:20:12 +0530 Subject: [PATCH 64/69] Merged all testUtils into a single utils class --- .../client/AbstractRangerHubClientTest.java | 3 +- .../client/utils/RangerHubTestUtils.java | 1 - .../ranger/core/utils/RangerTestUtils.java | 60 ++++++++++++++++- .../flipkart/ranger/core/utils/TestUtils.java | 65 ------------------- .../http/ShardedRangerHttpClientTest.java | 3 +- .../http/SimpleRangerHttpClientTest.java | 4 +- .../http/UnshardedRangerHttpClientTest.java | 3 +- .../HttpShardedServiceFinderBuilderTest.java | 4 +- .../server/bundle/RangerServerBundleTest.java | 3 +- .../client/zk/ShardedZKRangerClientTest.java | 3 +- .../zk/UnshardedZKRangerClientTest.java | 3 +- .../ServiceHealthAggregatorTest.java | 8 +-- .../ServiceProviderIntegrationTest.java | 8 +-- .../model/ServiceProviderHealthcheckTest.java | 3 +- .../zookeeper/servicehub/ServiceHubTest.java | 3 +- 15 files changed, 78 insertions(+), 96 deletions(-) delete mode 100644 ranger-core/src/test/java/com/flipkart/ranger/core/utils/TestUtils.java diff --git a/ranger-client/src/test/java/com/flipkart/ranger/client/AbstractRangerHubClientTest.java b/ranger-client/src/test/java/com/flipkart/ranger/client/AbstractRangerHubClientTest.java index e7d61bb4..99eaa177 100644 --- a/ranger-client/src/test/java/com/flipkart/ranger/client/AbstractRangerHubClientTest.java +++ b/ranger-client/src/test/java/com/flipkart/ranger/client/AbstractRangerHubClientTest.java @@ -18,7 +18,6 @@ import com.flipkart.ranger.client.utils.RangerHubTestUtils; import com.flipkart.ranger.core.model.Service; import com.flipkart.ranger.core.utils.RangerTestUtils; -import com.flipkart.ranger.core.utils.TestUtils; import lombok.extern.slf4j.Slf4j; import lombok.val; import lombok.var; @@ -34,7 +33,7 @@ public class AbstractRangerHubClientTest { public void testAbstractHubClient() { val testAbstractHub = RangerHubTestUtils.getTestHub(); testAbstractHub.start(); - TestUtils.sleepUntilHubIsActive(testAbstractHub.getHub()); + RangerTestUtils.sleepUntilHubIsActive(testAbstractHub.getHub()); var node = testAbstractHub.getNode(service).orElse(null); Assert.assertNotNull(node); Assert.assertTrue(node.getHost().equalsIgnoreCase("localhost")); diff --git a/ranger-client/src/test/java/com/flipkart/ranger/client/utils/RangerHubTestUtils.java b/ranger-client/src/test/java/com/flipkart/ranger/client/utils/RangerHubTestUtils.java index 95396578..94780785 100644 --- a/ranger-client/src/test/java/com/flipkart/ranger/client/utils/RangerHubTestUtils.java +++ b/ranger-client/src/test/java/com/flipkart/ranger/client/utils/RangerHubTestUtils.java @@ -36,7 +36,6 @@ public static RangerTestHub getTestHub(){ .nodeRefreshTimeMs(1000) .criteria(new TestCriteria()) .deserializer(new TestDeserializer<>()) - .nodeRefreshTimeMs(1000) .build(); } diff --git a/ranger-core/src/test/java/com/flipkart/ranger/core/utils/RangerTestUtils.java b/ranger-core/src/test/java/com/flipkart/ranger/core/utils/RangerTestUtils.java index bf6dcfd9..4ae82b69 100644 --- a/ranger-core/src/test/java/com/flipkart/ranger/core/utils/RangerTestUtils.java +++ b/ranger-core/src/test/java/com/flipkart/ranger/core/utils/RangerTestUtils.java @@ -15,17 +15,73 @@ */ package com.flipkart.ranger.core.utils; +import com.flipkart.ranger.core.finder.ServiceFinder; +import com.flipkart.ranger.core.finderhub.ServiceFinderHub; import com.flipkart.ranger.core.model.Service; +import com.flipkart.ranger.core.model.ServiceRegistry; import com.flipkart.ranger.core.units.TestNodeData; import lombok.experimental.UtilityClass; +import org.awaitility.core.ThrowingRunnable; +import java.time.Duration; +import java.util.concurrent.Callable; +import java.util.concurrent.TimeUnit; import java.util.function.Predicate; +import static org.awaitility.Awaitility.await; + +/** + * + */ @UtilityClass public class RangerTestUtils { - private static String TEST_NAMESPACE = "test"; - private static String TEST_SERVICE = "test-service"; + static String TEST_NAMESPACE = "test"; + static String TEST_SERVICE = "test-service"; + + /* + Node list is notEmpty only after the first updateRegistry has happened. So this should suffice for finder.start. + However, we have to use the sleepUntil with () -> true for when we need periodic wait (deterministic) + */ + public static > boolean validate(ServiceFinder finder){ + return finder.getServiceRegistry() != null && + finder.getServiceRegistry().nodeList() != null && + !finder.getServiceRegistry().nodeList().isEmpty(); + } + + /* + If we know the upper bound condition, please use the until with the upper bound. + Only for cases, where you have to wait till the refreshInterval periods, don't want to introduce + refreshed and other boolean flags throughout the code. + */ + public static void sleepUntil(int numSeconds) { + await().pollDelay(numSeconds, TimeUnit.SECONDS).until(() -> true); + } + + /* + Use this when you have to alter the numSeconds in any of the specific assertions. For finder and hub, the values are appropriately coded + keeping the start intervals in mind. + */ + public static void sleepUntil(int numSeconds, Callable conditionEvaluator) { + await().pollDelay(numSeconds, TimeUnit.SECONDS).until(conditionEvaluator); + } + + /* + Use this when you have to alter the numSeconds in any of the specific assertions. For finder and hub, the values are appropriately coded + keeping the start intervals in mind. + */ + public static void sleepUntil(int numSeconds, ThrowingRunnable assertion){ + await().pollDelay(Duration.ofSeconds(numSeconds)).untilAsserted(assertion); + } + + public static > void sleepUntilFinderIsActive(ServiceFinder finder){ + sleepUntil(3, () -> validate(finder)); + } + + public static > void sleepUntilHubIsActive(ServiceFinderHub hub){ + sleepUntil(3, () -> hub.getServiceDataSource().start()); + sleepUntil(3, () -> hub.getFinders().get().values().stream().allMatch(RangerTestUtils::validate)); + } public static Predicate getCriteria(final int shardId){ return nodeData -> nodeData.getShardId() == shardId; diff --git a/ranger-core/src/test/java/com/flipkart/ranger/core/utils/TestUtils.java b/ranger-core/src/test/java/com/flipkart/ranger/core/utils/TestUtils.java deleted file mode 100644 index de3b4c29..00000000 --- a/ranger-core/src/test/java/com/flipkart/ranger/core/utils/TestUtils.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright 2015 Flipkart Internet Pvt. Ltd. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.flipkart.ranger.core.utils; - -import com.flipkart.ranger.core.finder.ServiceFinder; -import com.flipkart.ranger.core.finderhub.ServiceFinderHub; -import com.flipkart.ranger.core.model.ServiceRegistry; -import lombok.experimental.UtilityClass; -import org.awaitility.core.ThrowingRunnable; - -import java.time.Duration; -import java.util.concurrent.Callable; -import java.util.concurrent.TimeUnit; - -import static org.awaitility.Awaitility.await; - -/** - * - */ -@UtilityClass -public class TestUtils { - - /* - If we know the upper bound condition, please use the until with the upper bound. - Only for cases, where you have to wait till the refreshInterval periods, don't want to introduce - refreshed and other boolean flags throughout the code. - */ - public static void sleepUntil(int numSeconds) { - await().pollDelay(numSeconds, TimeUnit.SECONDS).until(() -> true); - } - - public static void sleepUntil(int numSeconds, Callable conditionEvaluator) { - await().pollDelay(numSeconds, TimeUnit.SECONDS).until(conditionEvaluator); - } - - /* - Use this when you have to alter the numSeconds in any of the specific assertions. For finder and hub, the values are appropriately coded - keeping the start intervals in mind. - */ - public static void sleepUntil(int numSeconds, ThrowingRunnable assertion){ - await().pollDelay(Duration.ofSeconds(numSeconds)).untilAsserted(assertion); - } - - public static > void sleepUntilFinderIsActive(ServiceFinder finder){ - sleepUntil(3, () -> finder.getStartSignal().start()); - } - - public static > void sleepUntilHubIsActive(ServiceFinderHub hub){ - sleepUntil(3, () -> hub.getServiceDataSource().start()); - sleepUntil(3, () -> hub.getFinders().get().values().forEach(finder -> finder.getStartSignal().start())); - } -} diff --git a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/ShardedRangerHttpClientTest.java b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/ShardedRangerHttpClientTest.java index bf252099..2eb31241 100644 --- a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/ShardedRangerHttpClientTest.java +++ b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/ShardedRangerHttpClientTest.java @@ -17,7 +17,6 @@ import com.flipkart.ranger.core.units.TestNodeData; import com.flipkart.ranger.core.utils.RangerTestUtils; -import com.flipkart.ranger.core.utils.TestUtils; import lombok.val; import org.junit.Assert; import org.junit.Test; @@ -34,7 +33,7 @@ public void testShardedHttpHubClient(){ .nodeRefreshIntervalMs(1000) .build(); client.start(); - TestUtils.sleepUntilHubIsActive(client.getHub()); + RangerTestUtils.sleepUntilHubIsActive(client.getHub()); val service = RangerTestUtils.getService("test-n", "test-s"); Assert.assertNotNull(client.getNode(service).orElse(null)); Assert.assertNotNull(client.getNode(service, nodeData -> nodeData.getShardId() == 1).orElse(null)); diff --git a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/SimpleRangerHttpClientTest.java b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/SimpleRangerHttpClientTest.java index af6a1cfe..16c74798 100644 --- a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/SimpleRangerHttpClientTest.java +++ b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/SimpleRangerHttpClientTest.java @@ -16,7 +16,7 @@ package com.flipkart.ranger.client.http; import com.flipkart.ranger.core.units.TestNodeData; -import com.flipkart.ranger.core.utils.TestUtils; +import com.flipkart.ranger.core.utils.RangerTestUtils; import lombok.val; import org.junit.Assert; import org.junit.Test; @@ -34,7 +34,7 @@ public void testSimpleHttpRangerClient(){ .nodeRefreshIntervalMs(1000) .build(); client.start(); - TestUtils.sleepUntilFinderIsActive(client.getServiceFinder()); + RangerTestUtils.sleepUntilFinderIsActive(client.getServiceFinder()); Assert.assertNotNull(client.getNode().orElse(null)); Assert.assertNotNull(client.getNode(nodeData -> nodeData.getShardId() == 1).orElse(null)); Assert.assertNull(client.getNode(nodeData -> nodeData.getShardId() == 2).orElse(null)); diff --git a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/UnshardedRangerHttpClientTest.java b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/UnshardedRangerHttpClientTest.java index a9eae555..075940a8 100644 --- a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/UnshardedRangerHttpClientTest.java +++ b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/UnshardedRangerHttpClientTest.java @@ -17,7 +17,6 @@ import com.flipkart.ranger.core.units.TestNodeData; import com.flipkart.ranger.core.utils.RangerTestUtils; -import com.flipkart.ranger.core.utils.TestUtils; import lombok.val; import org.junit.Assert; import org.junit.Test; @@ -34,7 +33,7 @@ public void testUnshardedRangerHubClient(){ .nodeRefreshIntervalMs(1000) .build(); client.start(); - TestUtils.sleepUntilHubIsActive(client.getHub()); + RangerTestUtils.sleepUntilHubIsActive(client.getHub()); val service = RangerTestUtils.getService("test-n", "test-s"); Assert.assertNotNull(client.getNode(service).orElse(null)); Assert.assertNotNull(client.getNode(service, nodeData -> nodeData.getShardId() == 1).orElse(null)); diff --git a/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java b/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java index 2b1a7e39..5d8467a9 100644 --- a/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java +++ b/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java @@ -20,7 +20,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.flipkart.ranger.core.healthcheck.HealthcheckStatus; import com.flipkart.ranger.core.model.ServiceNode; -import com.flipkart.ranger.core.utils.TestUtils; +import com.flipkart.ranger.core.utils.RangerTestUtils; import com.flipkart.ranger.http.config.HttpClientConfig; import com.flipkart.ranger.http.model.ServiceNodesResponse; import com.github.tomakehurst.wiremock.junit.WireMockRule; @@ -92,7 +92,7 @@ public void testFinder() throws Exception { .withNodeRefreshIntervalMs(1000) .build(); finder.start(); - TestUtils.sleepUntilFinderIsActive(finder); + RangerTestUtils.sleepUntilFinderIsActive(finder); Assert.assertTrue(finder.get(nodeData -> true).isPresent()); } diff --git a/ranger-server-bundle/src/test/java/com/flipkart/ranger/zk/server/bundle/RangerServerBundleTest.java b/ranger-server-bundle/src/test/java/com/flipkart/ranger/zk/server/bundle/RangerServerBundleTest.java index 9c5e7f48..6b712951 100644 --- a/ranger-server-bundle/src/test/java/com/flipkart/ranger/zk/server/bundle/RangerServerBundleTest.java +++ b/ranger-server-bundle/src/test/java/com/flipkart/ranger/zk/server/bundle/RangerServerBundleTest.java @@ -23,7 +23,6 @@ import com.flipkart.ranger.client.utils.RangerHubTestUtils; import com.flipkart.ranger.core.units.TestNodeData; import com.flipkart.ranger.core.utils.RangerTestUtils; -import com.flipkart.ranger.core.utils.TestUtils; import io.dropwizard.Configuration; import io.dropwizard.jersey.DropwizardResourceConfig; import io.dropwizard.jersey.setup.JerseyEnvironment; @@ -91,7 +90,7 @@ public void setup() throws Exception { public void testRangerBundle(){ var hub = rangerServerBundle.getHubs().get(0); Assert.assertTrue(hub instanceof RangerTestHub); - TestUtils.sleepUntilHubIsActive(((RangerTestHub) hub).getHub()); + RangerTestUtils.sleepUntilHubIsActive(((RangerTestHub) hub).getHub()); var node = hub.getNode(service).orElse(null); Assert.assertNotNull(node); Assert.assertTrue(node.getHost().equalsIgnoreCase("localhost")); diff --git a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/ShardedZKRangerClientTest.java b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/ShardedZKRangerClientTest.java index ae3d6bd2..5a34963f 100644 --- a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/ShardedZKRangerClientTest.java +++ b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/ShardedZKRangerClientTest.java @@ -17,7 +17,6 @@ import com.flipkart.ranger.core.units.TestNodeData; import com.flipkart.ranger.core.utils.RangerTestUtils; -import com.flipkart.ranger.core.utils.TestUtils; import lombok.extern.slf4j.Slf4j; import lombok.val; import org.junit.Assert; @@ -38,7 +37,7 @@ public void testShardedHub(){ .refreshTimeMs(1000) .build(); zkHubClient.start(); - TestUtils.sleepUntilHubIsActive(zkHubClient.getHub()); + RangerTestUtils.sleepUntilHubIsActive(zkHubClient.getHub()); Assert.assertNotNull(zkHubClient.getNode(RangerTestUtils.getService("test-n", "s1")).orElse(null)); Assert.assertNotNull(zkHubClient.getNode(RangerTestUtils.getService("test-n", "s1"), nodeData -> nodeData.getShardId() == 1).orElse(null)); zkHubClient.stop(); diff --git a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/UnshardedZKRangerClientTest.java b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/UnshardedZKRangerClientTest.java index 4d648dd8..9db7c3e4 100644 --- a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/UnshardedZKRangerClientTest.java +++ b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/UnshardedZKRangerClientTest.java @@ -17,7 +17,6 @@ import com.flipkart.ranger.core.units.TestNodeData; import com.flipkart.ranger.core.utils.RangerTestUtils; -import com.flipkart.ranger.core.utils.TestUtils; import lombok.extern.slf4j.Slf4j; import lombok.val; import org.junit.Assert; @@ -38,7 +37,7 @@ public void testShardedHub(){ .refreshTimeMs(1000) .build(); zkHubClient.start(); - TestUtils.sleepUntilHubIsActive(zkHubClient.getHub()); + RangerTestUtils.sleepUntilHubIsActive(zkHubClient.getHub()); val service = RangerTestUtils.getService("test-n", "s1"); Assert.assertNotNull(zkHubClient.getNode(service).orElse(null)); Assert.assertNotNull(zkHubClient.getNode(service, nodeData -> nodeData.getShardId() == 1).orElse(null)); diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/ServiceHealthAggregatorTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/ServiceHealthAggregatorTest.java index 9c87c0d1..d0a149d6 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/ServiceHealthAggregatorTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/ServiceHealthAggregatorTest.java @@ -20,7 +20,7 @@ import com.flipkart.ranger.core.healthservice.TimeEntity; import com.flipkart.ranger.core.healthservice.monitor.IsolatedHealthMonitor; import com.flipkart.ranger.core.healthservice.monitor.Monitor; -import com.flipkart.ranger.core.utils.TestUtils; +import com.flipkart.ranger.core.utils.RangerTestUtils; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -50,7 +50,7 @@ public boolean isDisabled() { }); serviceHealthAggregator.start(); - TestUtils.sleepUntil(3, () -> serviceHealthAggregator.getRunning().get()); + RangerTestUtils.sleepUntil(3, () -> serviceHealthAggregator.getRunning().get()); } @After @@ -64,14 +64,14 @@ public void testStaleRun() { testMonitor.run(); testMonitor.setThreadSleep(2000); - TestUtils.sleepUntil(3, () -> !testMonitor.hasValidUpdatedTime(new Date())); + RangerTestUtils.sleepUntil(3, () -> !testMonitor.hasValidUpdatedTime(new Date())); /* in the TestMonitor, thread was sleeping for 2 seconds, */ /* so its state is supposed to be stale (>1 second) and service has to be unhealthy */ Assert.assertEquals(HealthcheckStatus.unhealthy, serviceHealthAggregator.getServiceHealth()); testMonitor.setThreadSleep(5); - TestUtils.sleepUntil(3, () -> testMonitor.hasValidUpdatedTime(new Date())); + RangerTestUtils.sleepUntil(3, () -> testMonitor.hasValidUpdatedTime(new Date())); /* in the TestMonitor, thread is sleeping only for 10 milliseconds, */ /* so its state is supposed to be NOT stale (>1 second) and service has to be healthy */ diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/ServiceProviderIntegrationTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/ServiceProviderIntegrationTest.java index 7b05b0d4..29e4f255 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/ServiceProviderIntegrationTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/ServiceProviderIntegrationTest.java @@ -25,7 +25,7 @@ import com.flipkart.ranger.core.healthservice.monitor.sample.RotationStatusMonitor; import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.util.Exceptions; -import com.flipkart.ranger.core.utils.TestUtils; +import com.flipkart.ranger.core.utils.RangerTestUtils; import com.flipkart.ranger.zookeeper.ServiceFinderBuilders; import com.flipkart.ranger.zookeeper.ServiceProviderBuilders; import lombok.val; @@ -111,7 +111,7 @@ public void testBasicDiscovery() throws Exception { /* with file existing, 3 nodes should be healthy */ var filecreate = file.createNewFile(); System.out.println("created file"); - TestUtils.sleepUntil(5); + RangerTestUtils.sleepUntil(5); List> all = serviceFinder.getAll(null); System.out.println("all = " + all); Assert.assertEquals(3, all.size()); @@ -119,7 +119,7 @@ public void testBasicDiscovery() throws Exception { /* with file deleted, all 3 nodes should be unhealthy */ delete = file.delete(); System.out.println("deleted file"); - TestUtils.sleepUntil(5); + RangerTestUtils.sleepUntil(5); all = serviceFinder.getAll(null); System.out.println("all = " + all); Assert.assertEquals(0, all.size()); @@ -127,7 +127,7 @@ public void testBasicDiscovery() throws Exception { /* with anotherFile created, the 4th node should become healthy and discoverable */ filecreate = anotherFile.createNewFile(); System.out.println("created anotherFile"); - TestUtils.sleepUntil(5); + RangerTestUtils.sleepUntil(5); all = serviceFinder.getAll(null); System.out.println("all = " + all); Assert.assertEquals(1, all.size()); diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderHealthcheckTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderHealthcheckTest.java index 18d29ffb..f521d739 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderHealthcheckTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/model/ServiceProviderHealthcheckTest.java @@ -24,7 +24,6 @@ import com.flipkart.ranger.core.model.ServiceNode; import com.flipkart.ranger.core.units.TestNodeData; import com.flipkart.ranger.core.utils.RangerTestUtils; -import com.flipkart.ranger.core.utils.TestUtils; import com.flipkart.ranger.zookeeper.ServiceFinderBuilders; import com.flipkart.ranger.zookeeper.ServiceProviderBuilders; import com.google.common.collect.Maps; @@ -86,7 +85,7 @@ public void testBasicDiscovery() { Assert.assertEquals("localhost-1", node.getHost()); TestServiceProvider testServiceProvider = serviceProviders.get(node.getHost()); testServiceProvider.oor(); - TestUtils.sleepUntil(2); //Sleep till the increment refresh healthCheck interval (> 1sec), no upper bound condition. + RangerTestUtils.sleepUntil(2); //Sleep till the increment refresh healthCheck interval (> 1sec), no upper bound condition. Assert.assertFalse(serviceFinder.get(RangerTestUtils.getCriteria(1)).isPresent()); serviceFinder.stop(); } diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/servicehub/ServiceHubTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/servicehub/ServiceHubTest.java index aa5337d4..877c3aa3 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/servicehub/ServiceHubTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/servicehub/ServiceHubTest.java @@ -25,7 +25,6 @@ import com.flipkart.ranger.core.units.TestNodeData; import com.flipkart.ranger.core.util.Exceptions; import com.flipkart.ranger.core.utils.RangerTestUtils; -import com.flipkart.ranger.core.utils.TestUtils; import com.flipkart.ranger.zookeeper.ServiceProviderBuilders; import com.flipkart.ranger.zookeeper.servicefinderhub.ZkServiceDataSource; import com.flipkart.ranger.zookeeper.servicefinderhub.ZkServiceFinderHubBuilder; @@ -117,7 +116,7 @@ public void testHub() { refreshProviderSignal.trigger(); refreshHubSignal.trigger(); - TestUtils.sleepUntilHubIsActive(hub); + RangerTestUtils.sleepUntilHubIsActive(hub); val node = hub.finder(RangerTestUtils.getService(NAMESPACE, "s1")) .flatMap(finder -> finder.get(nodeData -> nodeData.getShardId() == 1)).orElse(null); From ce1e2c4786e5893414a760a084b1c95588a05b37 Mon Sep 17 00:00:00 2001 From: Koushik R Date: Tue, 4 Jan 2022 22:33:55 +0530 Subject: [PATCH 65/69] Refactored the rangerTestUtils to simplify awaitility usages --- .../client/AbstractRangerHubClientTest.java | 2 +- .../ranger/core/utils/RangerTestUtils.java | 45 +++++++++---------- .../http/ShardedRangerHttpClientTest.java | 2 +- .../http/SimpleRangerHttpClientTest.java | 2 +- .../http/UnshardedRangerHttpClientTest.java | 2 +- .../HttpShardedServiceFinderBuilderTest.java | 2 +- .../server/bundle/RangerServerBundleTest.java | 2 +- .../client/zk/ShardedZKRangerClientTest.java | 2 +- .../zk/UnshardedZKRangerClientTest.java | 2 +- .../ServiceProviderIntegrationTest.java | 2 +- .../zookeeper/servicehub/ServiceHubTest.java | 2 +- 11 files changed, 31 insertions(+), 34 deletions(-) diff --git a/ranger-client/src/test/java/com/flipkart/ranger/client/AbstractRangerHubClientTest.java b/ranger-client/src/test/java/com/flipkart/ranger/client/AbstractRangerHubClientTest.java index 99eaa177..2530e7bd 100644 --- a/ranger-client/src/test/java/com/flipkart/ranger/client/AbstractRangerHubClientTest.java +++ b/ranger-client/src/test/java/com/flipkart/ranger/client/AbstractRangerHubClientTest.java @@ -33,7 +33,7 @@ public class AbstractRangerHubClientTest { public void testAbstractHubClient() { val testAbstractHub = RangerHubTestUtils.getTestHub(); testAbstractHub.start(); - RangerTestUtils.sleepUntilHubIsActive(testAbstractHub.getHub()); + RangerTestUtils.sleepUntilHubStarts(testAbstractHub.getHub()); var node = testAbstractHub.getNode(service).orElse(null); Assert.assertNotNull(node); Assert.assertTrue(node.getHost().equalsIgnoreCase("localhost")); diff --git a/ranger-core/src/test/java/com/flipkart/ranger/core/utils/RangerTestUtils.java b/ranger-core/src/test/java/com/flipkart/ranger/core/utils/RangerTestUtils.java index 4ae82b69..11ed9a95 100644 --- a/ranger-core/src/test/java/com/flipkart/ranger/core/utils/RangerTestUtils.java +++ b/ranger-core/src/test/java/com/flipkart/ranger/core/utils/RangerTestUtils.java @@ -39,6 +39,21 @@ public class RangerTestUtils { static String TEST_NAMESPACE = "test"; static String TEST_SERVICE = "test-service"; + public static Predicate getCriteria(final int shardId){ + return nodeData -> nodeData.getShardId() == shardId; + } + + public static Service getService(){ + return getService(TEST_NAMESPACE, TEST_SERVICE); + } + + public static Service getService(String namespace, String serviceName){ + return Service.builder() + .serviceName(serviceName) + .namespace(namespace) + .build(); + } + /* Node list is notEmpty only after the first updateRegistry has happened. So this should suffice for finder.start. However, we have to use the sleepUntil with () -> true for when we need periodic wait (deterministic) @@ -67,34 +82,16 @@ public static void sleepUntil(int numSeconds, Callable conditionEvaluat } /* - Use this when you have to alter the numSeconds in any of the specific assertions. For finder and hub, the values are appropriately coded - keeping the start intervals in mind. + Only applicable for initial node population using finder. Works when you intend to start the finder with nodes in 'em. */ - public static void sleepUntil(int numSeconds, ThrowingRunnable assertion){ - await().pollDelay(Duration.ofSeconds(numSeconds)).untilAsserted(assertion); - } - - public static > void sleepUntilFinderIsActive(ServiceFinder finder){ + public static > void sleepUntilFinderStarts(ServiceFinder finder){ sleepUntil(3, () -> validate(finder)); } - public static > void sleepUntilHubIsActive(ServiceFinderHub hub){ - sleepUntil(3, () -> hub.getServiceDataSource().start()); + /* + Only applicable for initial node population using hub of finders. Works when you intend to start the finder hub with nodes in 'em. + */ + public static > void sleepUntilHubStarts(ServiceFinderHub hub){ sleepUntil(3, () -> hub.getFinders().get().values().stream().allMatch(RangerTestUtils::validate)); } - - public static Predicate getCriteria(final int shardId){ - return nodeData -> nodeData.getShardId() == shardId; - } - - public static Service getService(){ - return getService(TEST_NAMESPACE, TEST_SERVICE); - } - - public static Service getService(String namespace, String serviceName){ - return Service.builder() - .serviceName(serviceName) - .namespace(namespace) - .build(); - } } diff --git a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/ShardedRangerHttpClientTest.java b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/ShardedRangerHttpClientTest.java index 2eb31241..e4d1147c 100644 --- a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/ShardedRangerHttpClientTest.java +++ b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/ShardedRangerHttpClientTest.java @@ -33,7 +33,7 @@ public void testShardedHttpHubClient(){ .nodeRefreshIntervalMs(1000) .build(); client.start(); - RangerTestUtils.sleepUntilHubIsActive(client.getHub()); + RangerTestUtils.sleepUntilHubStarts(client.getHub()); val service = RangerTestUtils.getService("test-n", "test-s"); Assert.assertNotNull(client.getNode(service).orElse(null)); Assert.assertNotNull(client.getNode(service, nodeData -> nodeData.getShardId() == 1).orElse(null)); diff --git a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/SimpleRangerHttpClientTest.java b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/SimpleRangerHttpClientTest.java index 16c74798..d17ffdfd 100644 --- a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/SimpleRangerHttpClientTest.java +++ b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/SimpleRangerHttpClientTest.java @@ -34,7 +34,7 @@ public void testSimpleHttpRangerClient(){ .nodeRefreshIntervalMs(1000) .build(); client.start(); - RangerTestUtils.sleepUntilFinderIsActive(client.getServiceFinder()); + RangerTestUtils.sleepUntilFinderStarts(client.getServiceFinder()); Assert.assertNotNull(client.getNode().orElse(null)); Assert.assertNotNull(client.getNode(nodeData -> nodeData.getShardId() == 1).orElse(null)); Assert.assertNull(client.getNode(nodeData -> nodeData.getShardId() == 2).orElse(null)); diff --git a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/UnshardedRangerHttpClientTest.java b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/UnshardedRangerHttpClientTest.java index 075940a8..8bbda15f 100644 --- a/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/UnshardedRangerHttpClientTest.java +++ b/ranger-http-client/src/test/java/com/flipkart/ranger/client/http/UnshardedRangerHttpClientTest.java @@ -33,7 +33,7 @@ public void testUnshardedRangerHubClient(){ .nodeRefreshIntervalMs(1000) .build(); client.start(); - RangerTestUtils.sleepUntilHubIsActive(client.getHub()); + RangerTestUtils.sleepUntilHubStarts(client.getHub()); val service = RangerTestUtils.getService("test-n", "test-s"); Assert.assertNotNull(client.getNode(service).orElse(null)); Assert.assertNotNull(client.getNode(service, nodeData -> nodeData.getShardId() == 1).orElse(null)); diff --git a/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java b/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java index 5d8467a9..38f3e571 100644 --- a/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java +++ b/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java @@ -92,7 +92,7 @@ public void testFinder() throws Exception { .withNodeRefreshIntervalMs(1000) .build(); finder.start(); - RangerTestUtils.sleepUntilFinderIsActive(finder); + RangerTestUtils.sleepUntilFinderStarts(finder); Assert.assertTrue(finder.get(nodeData -> true).isPresent()); } diff --git a/ranger-server-bundle/src/test/java/com/flipkart/ranger/zk/server/bundle/RangerServerBundleTest.java b/ranger-server-bundle/src/test/java/com/flipkart/ranger/zk/server/bundle/RangerServerBundleTest.java index 6b712951..62942ac8 100644 --- a/ranger-server-bundle/src/test/java/com/flipkart/ranger/zk/server/bundle/RangerServerBundleTest.java +++ b/ranger-server-bundle/src/test/java/com/flipkart/ranger/zk/server/bundle/RangerServerBundleTest.java @@ -90,7 +90,7 @@ public void setup() throws Exception { public void testRangerBundle(){ var hub = rangerServerBundle.getHubs().get(0); Assert.assertTrue(hub instanceof RangerTestHub); - RangerTestUtils.sleepUntilHubIsActive(((RangerTestHub) hub).getHub()); + RangerTestUtils.sleepUntilHubStarts(((RangerTestHub) hub).getHub()); var node = hub.getNode(service).orElse(null); Assert.assertNotNull(node); Assert.assertTrue(node.getHost().equalsIgnoreCase("localhost")); diff --git a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/ShardedZKRangerClientTest.java b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/ShardedZKRangerClientTest.java index 5a34963f..3f193419 100644 --- a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/ShardedZKRangerClientTest.java +++ b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/ShardedZKRangerClientTest.java @@ -37,7 +37,7 @@ public void testShardedHub(){ .refreshTimeMs(1000) .build(); zkHubClient.start(); - RangerTestUtils.sleepUntilHubIsActive(zkHubClient.getHub()); + RangerTestUtils.sleepUntilHubStarts(zkHubClient.getHub()); Assert.assertNotNull(zkHubClient.getNode(RangerTestUtils.getService("test-n", "s1")).orElse(null)); Assert.assertNotNull(zkHubClient.getNode(RangerTestUtils.getService("test-n", "s1"), nodeData -> nodeData.getShardId() == 1).orElse(null)); zkHubClient.stop(); diff --git a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/UnshardedZKRangerClientTest.java b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/UnshardedZKRangerClientTest.java index 9db7c3e4..7b37cd6d 100644 --- a/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/UnshardedZKRangerClientTest.java +++ b/ranger-zk-client/src/test/java/com/flipkart/ranger/client/zk/UnshardedZKRangerClientTest.java @@ -37,7 +37,7 @@ public void testShardedHub(){ .refreshTimeMs(1000) .build(); zkHubClient.start(); - RangerTestUtils.sleepUntilHubIsActive(zkHubClient.getHub()); + RangerTestUtils.sleepUntilHubStarts(zkHubClient.getHub()); val service = RangerTestUtils.getService("test-n", "s1"); Assert.assertNotNull(zkHubClient.getNode(service).orElse(null)); Assert.assertNotNull(zkHubClient.getNode(service, nodeData -> nodeData.getShardId() == 1).orElse(null)); diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/ServiceProviderIntegrationTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/ServiceProviderIntegrationTest.java index 29e4f255..69b5597e 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/ServiceProviderIntegrationTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/healthservice/ServiceProviderIntegrationTest.java @@ -137,7 +137,7 @@ public void testBasicDiscovery() throws Exception { delete = anotherFile.delete(); } - private void registerService(String host, int port, int shardId, File file) throws Exception { + private void registerService(String host, int port, int shardId, File file) { val serviceProvider = ServiceProviderBuilders.unshardedServiceProviderBuilder() .withConnectionString(testingCluster.getConnectString()) .withNamespace("test") diff --git a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/servicehub/ServiceHubTest.java b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/servicehub/ServiceHubTest.java index 877c3aa3..b30b34c2 100644 --- a/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/servicehub/ServiceHubTest.java +++ b/ranger-zookeeper/src/test/java/com/flipkart/ranger/zookeeper/servicehub/ServiceHubTest.java @@ -116,7 +116,7 @@ public void testHub() { refreshProviderSignal.trigger(); refreshHubSignal.trigger(); - RangerTestUtils.sleepUntilHubIsActive(hub); + RangerTestUtils.sleepUntilHubStarts(hub); val node = hub.finder(RangerTestUtils.getService(NAMESPACE, "s1")) .flatMap(finder -> finder.get(nodeData -> nodeData.getShardId() == 1)).orElse(null); From 600e3d9f1f0113f7bcf202639cae5e5811baa6de Mon Sep 17 00:00:00 2001 From: Koushik R Date: Tue, 4 Jan 2022 22:43:22 +0530 Subject: [PATCH 66/69] Removed the unnecessary imports --- .../flipkart/ranger/core/utils/RangerTestUtils.java | 10 +++------- .../flipkart/ranger/core/utils/RegistryTestUtils.java | 4 ++-- .../HttpShardedServiceFinderBuilderTest.java | 2 +- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/ranger-core/src/test/java/com/flipkart/ranger/core/utils/RangerTestUtils.java b/ranger-core/src/test/java/com/flipkart/ranger/core/utils/RangerTestUtils.java index 11ed9a95..ea881638 100644 --- a/ranger-core/src/test/java/com/flipkart/ranger/core/utils/RangerTestUtils.java +++ b/ranger-core/src/test/java/com/flipkart/ranger/core/utils/RangerTestUtils.java @@ -21,9 +21,7 @@ import com.flipkart.ranger.core.model.ServiceRegistry; import com.flipkart.ranger.core.units.TestNodeData; import lombok.experimental.UtilityClass; -import org.awaitility.core.ThrowingRunnable; -import java.time.Duration; import java.util.concurrent.Callable; import java.util.concurrent.TimeUnit; import java.util.function.Predicate; @@ -38,15 +36,13 @@ public class RangerTestUtils { static String TEST_NAMESPACE = "test"; static String TEST_SERVICE = "test-service"; + //Visible outside the class scope + public static Service service = Service.builder().namespace(TEST_NAMESPACE).serviceName(TEST_SERVICE).build(); public static Predicate getCriteria(final int shardId){ return nodeData -> nodeData.getShardId() == shardId; } - public static Service getService(){ - return getService(TEST_NAMESPACE, TEST_SERVICE); - } - public static Service getService(String namespace, String serviceName){ return Service.builder() .serviceName(serviceName) @@ -58,7 +54,7 @@ public static Service getService(String namespace, String serviceName){ Node list is notEmpty only after the first updateRegistry has happened. So this should suffice for finder.start. However, we have to use the sleepUntil with () -> true for when we need periodic wait (deterministic) */ - public static > boolean validate(ServiceFinder finder){ + private static > boolean validate(ServiceFinder finder){ return finder.getServiceRegistry() != null && finder.getServiceRegistry().nodeList() != null && !finder.getServiceRegistry().nodeList().isEmpty(); diff --git a/ranger-core/src/test/java/com/flipkart/ranger/core/utils/RegistryTestUtils.java b/ranger-core/src/test/java/com/flipkart/ranger/core/utils/RegistryTestUtils.java index 4f1620e7..781e27f7 100644 --- a/ranger-core/src/test/java/com/flipkart/ranger/core/utils/RegistryTestUtils.java +++ b/ranger-core/src/test/java/com/flipkart/ranger/core/utils/RegistryTestUtils.java @@ -27,7 +27,7 @@ public class RegistryTestUtils { public static MapBasedServiceRegistry getServiceRegistry(){ - val service = RangerTestUtils.getService(); + val service = RangerTestUtils.service; val serviceRegistry = new MapBasedServiceRegistry(service); val serviceNodes = ImmutableList.of( ServiceNode.builder().host("localhost-1").port(9000).nodeData(TestNodeData.builder().shardId(1).build()).build(), @@ -39,7 +39,7 @@ public static MapBasedServiceRegistry getServiceRegistry(){ } public static ListBasedServiceRegistry getUnshardedRegistry(){ - val service = RangerTestUtils.getService(); + val service = RangerTestUtils.service; val serviceRegistry = new ListBasedServiceRegistry(service); val serviceNodes = ImmutableList.of( ServiceNode.builder().host("localhost-1").port(9000).nodeData(TestNodeData.builder().shardId(1).build()).build(), diff --git a/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java b/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java index 38f3e571..26ca3492 100644 --- a/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java +++ b/ranger-http/src/test/java/com/flipkart/ranger/http/servicefinder/HttpShardedServiceFinderBuilderTest.java @@ -93,7 +93,7 @@ public void testFinder() throws Exception { .build(); finder.start(); RangerTestUtils.sleepUntilFinderStarts(finder); - Assert.assertTrue(finder.get(nodeData -> true).isPresent()); + Assert.assertNotNull(finder.get(nodeData -> true).orElse(null)); } } \ No newline at end of file From b325512f4bfe8cbea60f9f99a1c4f82b65939433 Mon Sep 17 00:00:00 2001 From: Koushik R Date: Tue, 4 Jan 2022 22:58:08 +0530 Subject: [PATCH 67/69] Removed an unnecessary assignment in RegistryTestUtils --- .../com/flipkart/ranger/core/utils/RegistryTestUtils.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/ranger-core/src/test/java/com/flipkart/ranger/core/utils/RegistryTestUtils.java b/ranger-core/src/test/java/com/flipkart/ranger/core/utils/RegistryTestUtils.java index 781e27f7..156c42bd 100644 --- a/ranger-core/src/test/java/com/flipkart/ranger/core/utils/RegistryTestUtils.java +++ b/ranger-core/src/test/java/com/flipkart/ranger/core/utils/RegistryTestUtils.java @@ -27,8 +27,7 @@ public class RegistryTestUtils { public static MapBasedServiceRegistry getServiceRegistry(){ - val service = RangerTestUtils.service; - val serviceRegistry = new MapBasedServiceRegistry(service); + val serviceRegistry = new MapBasedServiceRegistry(RangerTestUtils.service); val serviceNodes = ImmutableList.of( ServiceNode.builder().host("localhost-1").port(9000).nodeData(TestNodeData.builder().shardId(1).build()).build(), ServiceNode.builder().host("localhost-2").port(9001).nodeData(TestNodeData.builder().shardId(2).build()).build(), @@ -39,8 +38,7 @@ public static MapBasedServiceRegistry getServiceRegistry(){ } public static ListBasedServiceRegistry getUnshardedRegistry(){ - val service = RangerTestUtils.service; - val serviceRegistry = new ListBasedServiceRegistry(service); + val serviceRegistry = new ListBasedServiceRegistry(RangerTestUtils.service); val serviceNodes = ImmutableList.of( ServiceNode.builder().host("localhost-1").port(9000).nodeData(TestNodeData.builder().shardId(1).build()).build(), ServiceNode.builder().host("localhost-2").port(9001).nodeData(TestNodeData.builder().shardId(2).build()).build(), From 88eb770c2522682b0614fa9f8e799d5eaea4719b Mon Sep 17 00:00:00 2001 From: Koushik R Date: Tue, 4 Jan 2022 23:54:56 +0530 Subject: [PATCH 68/69] Simplified the awaitility bound condition further --- .../ranger/core/utils/RangerTestUtils.java | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/ranger-core/src/test/java/com/flipkart/ranger/core/utils/RangerTestUtils.java b/ranger-core/src/test/java/com/flipkart/ranger/core/utils/RangerTestUtils.java index ea881638..3c30c006 100644 --- a/ranger-core/src/test/java/com/flipkart/ranger/core/utils/RangerTestUtils.java +++ b/ranger-core/src/test/java/com/flipkart/ranger/core/utils/RangerTestUtils.java @@ -22,6 +22,7 @@ import com.flipkart.ranger.core.units.TestNodeData; import lombok.experimental.UtilityClass; +import java.time.Duration; import java.util.concurrent.Callable; import java.util.concurrent.TimeUnit; import java.util.function.Predicate; @@ -50,16 +51,6 @@ public static Service getService(String namespace, String serviceName){ .build(); } - /* - Node list is notEmpty only after the first updateRegistry has happened. So this should suffice for finder.start. - However, we have to use the sleepUntil with () -> true for when we need periodic wait (deterministic) - */ - private static > boolean validate(ServiceFinder finder){ - return finder.getServiceRegistry() != null && - finder.getServiceRegistry().nodeList() != null && - !finder.getServiceRegistry().nodeList().isEmpty(); - } - /* If we know the upper bound condition, please use the until with the upper bound. Only for cases, where you have to wait till the refreshInterval periods, don't want to introduce @@ -81,13 +72,14 @@ public static void sleepUntil(int numSeconds, Callable conditionEvaluat Only applicable for initial node population using finder. Works when you intend to start the finder with nodes in 'em. */ public static > void sleepUntilFinderStarts(ServiceFinder finder){ - sleepUntil(3, () -> validate(finder)); + await().pollDelay(Duration.ofSeconds(3)).until(() -> !finder.getServiceRegistry().nodeList().isEmpty()); } /* Only applicable for initial node population using hub of finders. Works when you intend to start the finder hub with nodes in 'em. */ public static > void sleepUntilHubStarts(ServiceFinderHub hub){ - sleepUntil(3, () -> hub.getFinders().get().values().stream().allMatch(RangerTestUtils::validate)); + await().pollDelay(Duration.ofSeconds(3)).until(() -> hub.getServiceDataSource() != null && + hub.getFinders().get().values().stream().noneMatch(finder -> finder.getServiceRegistry().nodeList().isEmpty())); } } From 4afb6acd124447dc6dbfa87275fc585f1de389cb Mon Sep 17 00:00:00 2001 From: Koushik R Date: Tue, 4 Jan 2022 23:58:09 +0530 Subject: [PATCH 69/69] Using a consistent Duration.of everywhere --- .../java/com/flipkart/ranger/core/utils/RangerTestUtils.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ranger-core/src/test/java/com/flipkart/ranger/core/utils/RangerTestUtils.java b/ranger-core/src/test/java/com/flipkart/ranger/core/utils/RangerTestUtils.java index 3c30c006..53e3e122 100644 --- a/ranger-core/src/test/java/com/flipkart/ranger/core/utils/RangerTestUtils.java +++ b/ranger-core/src/test/java/com/flipkart/ranger/core/utils/RangerTestUtils.java @@ -57,7 +57,7 @@ public static Service getService(String namespace, String serviceName){ refreshed and other boolean flags throughout the code. */ public static void sleepUntil(int numSeconds) { - await().pollDelay(numSeconds, TimeUnit.SECONDS).until(() -> true); + await().pollDelay(Duration.ofSeconds(numSeconds)).until(() -> true); } /* @@ -65,7 +65,7 @@ public static void sleepUntil(int numSeconds) { keeping the start intervals in mind. */ public static void sleepUntil(int numSeconds, Callable conditionEvaluator) { - await().pollDelay(numSeconds, TimeUnit.SECONDS).until(conditionEvaluator); + await().pollDelay(Duration.ofSeconds(numSeconds)).until(conditionEvaluator); } /*