Logo Questions Linux Laravel Mysql Ubuntu Git Menu
HTML CSS JAVASCRIPT SQL PYTHON PHP BOOTSTRAP JAVA JQUERY R React Kotlin
 

Spring cloud with ribbon not ignoring down servers

Tags:

spring-boot

load-balancing

spring-cloud

spring-cloud-netflix

I was followig spring guide on client side load balancing with eureka:

https://spring.io/guides/gs/client-side-load-balancing/

But I'm getting a connection refuse exception when after having 3 server instances I shut down any of them. Instead of the ignoring the servers that fail the ping, the client is still trying to send requests to those servers. I get an exception when those servers are pinged and the same exception when I the client tries to send requests to those servers. It seems like the ribbon is performing some sort of blind round-robbin through all servers listed in the configurations and it is ignoring the servers that I shut down.

application.yml

spring:
  application:
    name: user

server:
  port: 8888

say-hello:
  ribbon:
    eureka:
      enabled: false
    listOfServers: localhost:8090,localhost:9092,localhost:9999
    ServerListRefreshInterval: 15000

SayHelloConfiguration.java

package hello;

import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;

import com.netflix.client.config.IClientConfig;
import com.netflix.loadbalancer.IPing;
import com.netflix.loadbalancer.IRule;
import com.netflix.loadbalancer.PingUrl;
import com.netflix.loadbalancer.AvailabilityFilteringRule;

public class SayHelloConfiguration {

    @Autowired
    IClientConfig ribbonClientConfig;

    @Bean
    public IPing ribbonPing(IClientConfig config) {
        return new PingUrl();
    }

    @Bean
    public IRule ribbonRule(IClientConfig config) {
        return new AvailabilityFilteringRule();
    }

}

UserApplication.java

package hello;

import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;

import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.client.RestTemplate;
import org.springframework.cloud.client.loadbalancer.LoadBalanced;
import org.springframework.cloud.netflix.ribbon.RibbonClient;

@SpringBootApplication
@RestController
@RibbonClient(name = "say-hello", configuration = SayHelloConfiguration.class)
public class UserApplication {

    @LoadBalanced
    @Bean
    RestTemplate restTemplate(){
        return new RestTemplate();
    }

    @Autowired
    RestTemplate restTemplate;

    @RequestMapping("/hi")
    public String hi(@RequestParam(value="name", defaultValue="Artaban") String name) {
        String greeting = this.restTemplate.getForObject("http://say-hello/greeting", String.class);
        return String.format("%s, %s!", greeting, name);
    }

    public static void main(String[] args) {
        SpringApplication.run(UserApplication.class, args);
    }
}

pom.xml

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>hello</groupId>
    <artifactId>user</artifactId>
    <version>1.0-SNAPSHOT</version>
    <packaging>jar</packaging>

    <name>user</name>
    <description>Demo project for Spring Boot</description>

    <parent>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-parent</artifactId>
        <version>1.5.2.RELEASE</version>
        <relativePath/> <!-- lookup parent from repository -->
    </parent>

    <properties>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
        <java.version>1.8</java.version>
    </properties>

    <dependencies>
        <dependency>
            <groupId>org.springframework.cloud</groupId>
            <artifactId>spring-cloud-starter-ribbon</artifactId>
        </dependency>
        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-web</artifactId>
        </dependency>

        <dependency>
            <groupId>org.springframework.boot</groupId>
            <artifactId>spring-boot-starter-test</artifactId>
            <scope>test</scope>
        </dependency>
    </dependencies>

    <dependencyManagement>
        <dependencies>
            <dependency>
                <groupId>org.springframework.cloud</groupId>
                <artifactId>spring-cloud-dependencies</artifactId>
                <version>Camden.SR5</version>
                <type>pom</type>
                <scope>import</scope>
            </dependency>
        </dependencies>
    </dependencyManagement>

    <build>
        <plugins>
            <plugin>
                <groupId>org.springframework.boot</groupId>
                <artifactId>spring-boot-maven-plugin</artifactId>
            </plugin>
        </plugins>
    </build>

    <repositories>
        <repository>
            <id>spring-snapshots</id>
            <name>Spring Snapshots</name>
            <url>https://repo.spring.io/snapshot</url>
            <snapshots>
                <enabled>true</enabled>
            </snapshots>
        </repository>
        <repository>
            <id>spring-milestones</id>
            <name>Spring Milestones</name>
            <url>https://repo.spring.io/milestone</url>
            <snapshots>
                <enabled>false</enabled>
            </snapshots>
        </repository>
    </repositories>

</project>

This is the exception I'm getting when pinging the down servers:

java.net.ConnectException: Connection refused: connect
    at java.net.DualStackPlainSocketImpl.connect0(Native Method)
    at java.net.DualStackPlainSocketImpl.socketConnect(DualStackPlainSocketImpl.java:79)
    at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
    at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
    at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
    at java.net.PlainSocketImpl.connect(PlainSocketImpl.java:172)
    at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
    at java.net.Socket.connect(Socket.java:589)
    at org.apache.http.conn.scheme.PlainSocketFactory.connectSocket(PlainSocketFactory.java:121)
    at org.apache.http.impl.conn.DefaultClientConnectionOperator.openConnection(DefaultClientConnectionOperator.java:180)
    at org.apache.http.impl.conn.ManagedClientConnectionImpl.open(ManagedClientConnectionImpl.java:326)
    at org.apache.http.impl.client.DefaultRequestDirector.tryConnect(DefaultRequestDirector.java:610)
    at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:445)
    at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:835)
    at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:83)
    at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:108)
    at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:56)
    at com.netflix.loadbalancer.PingUrl.isAlive(PingUrl.java:126)
    at com.netflix.loadbalancer.BaseLoadBalancer$SerialPingStrategy.pingServers(BaseLoadBalancer.java:927)
    at com.netflix.loadbalancer.BaseLoadBalancer$Pinger.runPinger(BaseLoadBalancer.java:676)
    at com.netflix.loadbalancer.BaseLoadBalancer$PingTask.run(BaseLoadBalancer.java:629)
    at java.util.TimerThread.mainLoop(Timer.java:555)
    at java.util.TimerThread.run(Timer.java:505)

This is the exception I'm getting when the client app sends requests to the down servers:

java.net.ConnectException: Connection refused: connect
    at java.net.DualStackPlainSocketImpl.connect0(Native Method) ~[na:1.8.0_101]
    at java.net.DualStackPlainSocketImpl.socketConnect(DualStackPlainSocketImpl.java:79) ~[na:1.8.0_101]
    at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350) ~[na:1.8.0_101]
    at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206) ~[na:1.8.0_101]
    at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188) ~[na:1.8.0_101]
    at java.net.PlainSocketImpl.connect(PlainSocketImpl.java:172) ~[na:1.8.0_101]
    at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) ~[na:1.8.0_101]
    at java.net.Socket.connect(Socket.java:589) ~[na:1.8.0_101]
    at java.net.Socket.connect(Socket.java:538) ~[na:1.8.0_101]
    at sun.net.NetworkClient.doConnect(NetworkClient.java:180) ~[na:1.8.0_101]
    at sun.net.www.http.HttpClient.openServer(HttpClient.java:432) ~[na:1.8.0_101]
    at sun.net.www.http.HttpClient.openServer(HttpClient.java:527) ~[na:1.8.0_101]
    at sun.net.www.http.HttpClient.<init>(HttpClient.java:211) ~[na:1.8.0_101]
    at sun.net.www.http.HttpClient.New(HttpClient.java:308) ~[na:1.8.0_101]
    at sun.net.www.http.HttpClient.New(HttpClient.java:326) ~[na:1.8.0_101]
    at sun.net.www.protocol.http.HttpURLConnection.getNewHttpClient(HttpURLConnection.java:1169) ~[na:1.8.0_101]
    at sun.net.www.protocol.http.HttpURLConnection.plainConnect0(HttpURLConnection.java:1105) ~[na:1.8.0_101]
    at sun.net.www.protocol.http.HttpURLConnection.plainConnect(HttpURLConnection.java:999) ~[na:1.8.0_101]
    at sun.net.www.protocol.http.HttpURLConnection.connect(HttpURLConnection.java:933) ~[na:1.8.0_101]
    at org.springframework.http.client.SimpleBufferingClientHttpRequest.executeInternal(SimpleBufferingClientHttpRequest.java:78) ~[spring-web-4.3.7.RELEASE.jar:4.3.7.RELEASE]
    at org.springframework.http.client.AbstractBufferingClientHttpRequest.executeInternal(AbstractBufferingClientHttpRequest.java:48) ~[spring-web-4.3.7.RELEASE.jar:4.3.7.RELEASE]
    at org.springframework.http.client.AbstractClientHttpRequest.execute(AbstractClientHttpRequest.java:53) ~[spring-web-4.3.7.RELEASE.jar:4.3.7.RELEASE]
    at org.springframework.http.client.InterceptingClientHttpRequest$InterceptingRequestExecution.execute(InterceptingClientHttpRequest.java:99) ~[spring-web-4.3.7.RELEASE.jar:4.3.7.RELEASE]
    at org.springframework.cloud.client.loadbalancer.LoadBalancerRequestFactory$1.apply(LoadBalancerRequestFactory.java:63) ~[spring-cloud-commons-1.1.7.RELEASE.jar:1.1.7.RELEASE]
    at org.springframework.cloud.client.loadbalancer.LoadBalancerRequestFactory$1.apply(LoadBalancerRequestFactory.java:52) ~[spring-cloud-commons-1.1.7.RELEASE.jar:1.1.7.RELEASE]
    at org.springframework.cloud.netflix.ribbon.RibbonLoadBalancerClient.execute(RibbonLoadBalancerClient.java:98) ~[spring-cloud-netflix-core-1.2.5.RELEASE.jar:1.2.5.RELEASE]
    at org.springframework.cloud.client.loadbalancer.RetryLoadBalancerInterceptor$1.doWithRetry(RetryLoadBalancerInterceptor.java:88) ~[spring-cloud-commons-1.1.7.RELEASE.jar:1.1.7.RELEASE]
    at org.springframework.cloud.client.loadbalancer.RetryLoadBalancerInterceptor$1.doWithRetry(RetryLoadBalancerInterceptor.java:76) ~[spring-cloud-commons-1.1.7.RELEASE.jar:1.1.7.RELEASE]
    at org.springframework.retry.support.RetryTemplate.doExecute(RetryTemplate.java:286) ~[spring-retry-1.2.0.RELEASE.jar:na]
    at org.springframework.retry.support.RetryTemplate.execute(RetryTemplate.java:163) ~[spring-retry-1.2.0.RELEASE.jar:na]
    at org.springframework.cloud.client.loadbalancer.RetryLoadBalancerInterceptor.intercept(RetryLoadBalancerInterceptor.java:76) ~[spring-cloud-commons-1.1.7.RELEASE.jar:1.1.7.RELEASE]
    at org.springframework.http.client.InterceptingClientHttpRequest$InterceptingRequestExecution.execute(InterceptingClientHttpRequest.java:86) ~[spring-web-4.3.7.RELEASE.jar:4.3.7.RELEASE]
    at org.springframework.http.client.InterceptingClientHttpRequest.executeInternal(InterceptingClientHttpRequest.java:70) ~[spring-web-4.3.7.RELEASE.jar:4.3.7.RELEASE]
    at org.springframework.http.client.AbstractBufferingClientHttpRequest.executeInternal(AbstractBufferingClientHttpRequest.java:48) ~[spring-web-4.3.7.RELEASE.jar:4.3.7.RELEASE]
    at org.springframework.http.client.AbstractClientHttpRequest.execute(AbstractClientHttpRequest.java:53) ~[spring-web-4.3.7.RELEASE.jar:4.3.7.RELEASE]
    at org.springframework.web.client.RestTemplate.doExecute(RestTemplate.java:652) ~[spring-web-4.3.7.RELEASE.jar:4.3.7.RELEASE]
    at org.springframework.web.client.RestTemplate.execute(RestTemplate.java:613) ~[spring-web-4.3.7.RELEASE.jar:4.3.7.RELEASE]
    at org.springframework.web.client.RestTemplate.getForObject(RestTemplate.java:287) ~[spring-web-4.3.7.RELEASE.jar:4.3.7.RELEASE]
    at hello.UserApplication.hi(UserApplication.java:31) ~[classes/:na]
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[na:1.8.0_101]
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[na:1.8.0_101]
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.8.0_101]
    at java.lang.reflect.Method.invoke(Method.java:498) ~[na:1.8.0_101]
    at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205) ~[spring-web-4.3.7.RELEASE.jar:4.3.7.RELEASE]
    at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:133) ~[spring-web-4.3.7.RELEASE.jar:4.3.7.RELEASE]
    at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:116) ~[spring-webmvc-4.3.7.RELEASE.jar:4.3.7.RELEASE]
    at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:827) ~[spring-webmvc-4.3.7.RELEASE.jar:4.3.7.RELEASE]
    at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:738) ~[spring-webmvc-4.3.7.RELEASE.jar:4.3.7.RELEASE]
    at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:85) ~[spring-webmvc-4.3.7.RELEASE.jar:4.3.7.RELEASE]
    at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:963) ~[spring-webmvc-4.3.7.RELEASE.jar:4.3.7.RELEASE]
    at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:897) ~[spring-webmvc-4.3.7.RELEASE.jar:4.3.7.RELEASE]
    at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:970) ~[spring-webmvc-4.3.7.RELEASE.jar:4.3.7.RELEASE]
    at org.springframework.web.servlet.FrameworkServlet.doGet(FrameworkServlet.java:861) ~[spring-webmvc-4.3.7.RELEASE.jar:4.3.7.RELEASE]
    at javax.servlet.http.HttpServlet.service(HttpServlet.java:622) ~[tomcat-embed-core-8.5.11.jar:8.5.11]
    at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:846) ~[spring-webmvc-4.3.7.RELEASE.jar:4.3.7.RELEASE]
    at javax.servlet.http.HttpServlet.service(HttpServlet.java:729) ~[tomcat-embed-core-8.5.11.jar:8.5.11]
    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:230) ~[tomcat-embed-core-8.5.11.jar:8.5.11]
    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:165) ~[tomcat-embed-core-8.5.11.jar:8.5.11]
    at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:52) ~[tomcat-embed-websocket-8.5.11.jar:8.5.11]
    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:192) ~[tomcat-embed-core-8.5.11.jar:8.5.11]
    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:165) ~[tomcat-embed-core-8.5.11.jar:8.5.11]
    at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:99) ~[spring-web-4.3.7.RELEASE.jar:4.3.7.RELEASE]
    at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107) ~[spring-web-4.3.7.RELEASE.jar:4.3.7.RELEASE]
    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:192) ~[tomcat-embed-core-8.5.11.jar:8.5.11]
    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:165) ~[tomcat-embed-core-8.5.11.jar:8.5.11]
    at org.springframework.web.filter.HttpPutFormContentFilter.doFilterInternal(HttpPutFormContentFilter.java:105) ~[spring-web-4.3.7.RELEASE.jar:4.3.7.RELEASE]
    at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107) ~[spring-web-4.3.7.RELEASE.jar:4.3.7.RELEASE]
    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:192) ~[tomcat-embed-core-8.5.11.jar:8.5.11]
    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:165) ~[tomcat-embed-core-8.5.11.jar:8.5.11]
    at org.springframework.web.filter.HiddenHttpMethodFilter.doFilterInternal(HiddenHttpMethodFilter.java:81) ~[spring-web-4.3.7.RELEASE.jar:4.3.7.RELEASE]
    at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107) ~[spring-web-4.3.7.RELEASE.jar:4.3.7.RELEASE]
    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:192) ~[tomcat-embed-core-8.5.11.jar:8.5.11]
    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:165) ~[tomcat-embed-core-8.5.11.jar:8.5.11]
    at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:197) ~[spring-web-4.3.7.RELEASE.jar:4.3.7.RELEASE]
    at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:107) ~[spring-web-4.3.7.RELEASE.jar:4.3.7.RELEASE]
    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:192) ~[tomcat-embed-core-8.5.11.jar:8.5.11]
    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:165) ~[tomcat-embed-core-8.5.11.jar:8.5.11]
    at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:198) ~[tomcat-embed-core-8.5.11.jar:8.5.11]
    at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:96) [tomcat-embed-core-8.5.11.jar:8.5.11]
    at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:474) [tomcat-embed-core-8.5.11.jar:8.5.11]
    at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:140) [tomcat-embed-core-8.5.11.jar:8.5.11]
    at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:79) [tomcat-embed-core-8.5.11.jar:8.5.11]
    at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:87) [tomcat-embed-core-8.5.11.jar:8.5.11]
    at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:349) [tomcat-embed-core-8.5.11.jar:8.5.11]
    at org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:783) [tomcat-embed-core-8.5.11.jar:8.5.11]
    at org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:66) [tomcat-embed-core-8.5.11.jar:8.5.11]
    at org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:798) [tomcat-embed-core-8.5.11.jar:8.5.11]
    at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1434) [tomcat-embed-core-8.5.11.jar:8.5.11]
    at org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49) [tomcat-embed-core-8.5.11.jar:8.5.11]
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) [na:1.8.0_101]
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) [na:1.8.0_101]
    at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61) [tomcat-embed-core-8.5.11.jar:8.5.11]
    at java.lang.Thread.run(Thread.java:745) [na:1.8.0_101]

What am I missing? The expected behaviour would be that after the ping fails to the servers shut down, the client app would ignore those servers and sent no more requests to them.

Thanks

like image 642
jmdinis Avatar asked Mar 14 '17 11:03

jmdinis

3 Answers

Ok, so I came across this problem and here is how I tried to solve it. Not sure if this is the ideal way to go, but this seems sufficient for my needs right now.

The IPing implementation only helps determine if the server is alive or dead. It does not remove the server from the list of available servers.

As per the Javadoc here, looks like Ribbon leaves that the decision to client on when to mark a server as unhealthy. It seems a safe choice. Should the server be marked unavailable if it fails 1 ping vs. 3 pings and should there be a backoff while pinging - all this would be dependent on the kind of application you have at hand, so it is only the client that can make a safe choice.

However, there is no clear indication as to how to add a ServerStatusChangeListener to the load balancer in the context of Spring Cloud Netflix. So, I think if we had that ability, that would be the right place to call the BaseLoadBalancer.markServerDown(Server) from within the listener. Spring Cloud guru's (@spencergibb) can weigh in here.

Moving on to how I implemented my solution. In addition to the IPing, I have also configured a IRule and extended my rule from the com.netflix.loadbalancer.AvailabilityFilteringRule. Since this rule filters the servers that are circuit tripped, the only thing you would have to be adding is a check if the server itself is alive or not before return it as the chosen server.

@Configuration
public class MyLoadBalancerConfig {

    @Bean
    public IRule ribbonRule(IClientConfig config) {
        return new AvailabilityBasedServerSelectionRule();
    }

    @Bean
    public IPing ribbonPing() {
        return server -> {
            try {
                // TODO: ping your server to decide if health check has passed or not
                Thread.sleep(2000);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }

            return true;
        };
    }

    @Bean
    public ServerList<Server> ribbonServerList(IClientConfig config) {
        return new ServerList<Server>() {
            @Override
            public List<Server> getInitialListOfServers() {
                Server server1 = new Server("server1");
                Server server2 = new Server("server2");
                return new ArrayList<>(Arrays.asList(server1, server2));
            }

            @Override
            public List<Server> getUpdatedListOfServers() {
                Server server3 = new Server("server3");
                Server server4 = new Server("server4");

                return new ArrayList<>(Arrays.asList(server3, server4));
            }
        };
    }
}

and 

public class AvailabilityBasedServerSelectionRule extends AvailabilityFilteringRule {

    @Override
    public Server choose(Object key) {

        Server chosenServer = super.choose(key);

        int count = 1;
        List<Server> reachableServers = this.getLoadBalancer().getReachableServers();
        List<Server> allServers = this.getLoadBalancer().getAllServers();

        if(reachableServers.size() > 0) {
            while(!reachableServers.contains(chosenServer) && count++ < allServers.size()) {
                chosenServer = super.choose(key);
            }
        }

        System.out.println("chosenServer.isAlive() = " + chosenServer.isAlive());
        return chosenServer;
    }
}

Thus, the AvailabilityBasedServerSelectionRule always returns servers that are alive and NOT circuit tripped. I am yet to find any issues with this solution approach. I will update this post, if I find any as I go forward with this approach. Good luck!

like image 115
Satya Avatar answered Oct 05 '22 10:10

Satya

You can tell ribbon to retry next server if it is not there:

say-hello:
  ribbon:
    MaxAutoRetriesNextServer: 1

This also works when using eureka. Since service registry removes servers delayed if they not crash and not deregister.

Additionally, you need to add the following dependency:

<dependency>
    <groupId>org.springframework.retry</groupId>
    <artifactId>spring-retry</artifactId>
</dependency>
like image 45
Marcus A. Avatar answered Oct 05 '22 10:10

Marcus A.

After debugging BaseLoadBalancer I realized the following thing: Ribbon pings all sever instances from the list and determines which of them are dead. Ribbon does it in SerialPingStrategy, which is default implementation of IPingStrategy. These ping results are processed and the list of servers which status has changed since last ping process are send to ServerStatusChangeListener instances if at least some of them exist. But there is no any implementation provided in given tutorial. I suppose that if you provide some implementation of ServerStatusChangeListener and add it in application context, then there may be different behaviour.

like image 21
Yury Yaroshevich Avatar answered Oct 05 '22 11:10

Yury Yaroshevich
Sign in to Comment

Related questions

How to register a custom BasicAuthenticationFilter AuthenticationProvider in Spring Boot using Spring Security OAuth2
Spring boot not complaining about two beans with the same name
Spring Boot with multiple data sources using same repositories and model classes?
Spring to distinguish browser visitors from API calls to the endpoints
Migration to Spring Boot 2 - Security Encoded password does not look like BCrypt
Spring HATEOAS build link to paged resource
Weird (Loop) behavior when using Spring @TransactionalEventListener to publish event
How to speed up SpringBoot 2.0 basic authentication?
Spring boot Rest responding with empty body for exceptions other than the ones overridden in my @ControllerAdvice
Spring boot + tomcat 8.5 + mongoDB, AsyncRequestTimeoutException
Combining @PathVariable and @RequestBody
Spring JUnit Test not loading full Application Context
Spring Boot and Spring Data application with multiple DataSources created in runtime
Spring boot with authentication - login page not found (404)
how can i load jpa properties to datasource in Spring?
Spring Boot get property of a package in Gradle
Spring boot No default constructor found on @SpringBootApplication class
How to choose what implementation get's injected in to an autowired constructor
Why does MockMvc always return empty content()?
How to configure logback-access.xml with Spring Boot

Donate For Us

If you love us? You can donate to us via Paypal or buy me a coffee so we can maintain and grow! Thank you!