My supervisor crashes when I try to start it from eshell?

Question

I'm very new to OTP, I'm trying to create simple example to understand supervisor behaviour:

Here is simple increment server

-module( inc_serv ).
-behaviour( gen_server ).
-export( [ start/0, inc/1, stop/0 ] ).
-export( [ init/1, handle_call/3, terminate/2 ] ).

start() ->
        gen_server:start_link( { local, ?MODULE }, ?MODULE, no_args, [] ).

stop() ->
        gen_server:call( ?MODULE, stop ).

inc( Num ) ->
        gen_server:call( ?MODULE, { num, Num } ).

init( no_args ) ->
        io:format( "~p~n", [ "Increment server started :)" ] ),
        { ok, no_state }.

handle_call( { num, Num }, _From, no_state ) ->
        { reply, Num + 1, no_state };
handle_call( stop, _From, no_state ) ->
        { stop, normal, ok, no_state }.

terminate( Reason, no_state ) ->
        io:format( "~p~n", [ "Increment server stopped" ] ).

And I'd like to make it supervised by this module:

-module( supervisor_inc ).
-behaviour( supervisor ).

-export( [ start/0 ] ).
-export( [ init/1 ] ).

start() ->
        supervisor:start_link( { local, ?MODULE }, ?MODULE, no_args ).

init( no_args ) ->
        process_flag( trap_exit, true ),
        Supervisor_Spec = { one_for_one, 1, 1 },
        IncServ_Spec = {
                inc_serv,
                { inc_serv, start, [] },
                permanent, 2000, worker, [ inc_serv ] },
        { ok, { Supervisor_Spec, [ IncServ_Spec ] } }.

After that I've performed in erlang shell next steps:

1> 
1> c(inc_serv).
{ok,inc_serv}
2> 
2> c(supervisor_inc).
{ok,supervisor_inc}
3> 
3> supervisor_inc:start().
"Increment server started :)"
{ok,<0.43.0>}
4> 
4> inc_serv:inc( 7 ).
8
5> inc_serv:inc( 8 ).
9

After this I've tried next (as I expected I've got error):

6> inc_serv:inc( bad_arg ).
"Increment server stopped"
"Increment server started :)"

=ERROR REPORT==== 23-Aug-2012::19:32:06 ===
** Generic server inc_serv terminating 
** Last message in was {num,bad_arg}
** When Server state == no_state
** Reason for termination == 
** {badarith,[{inc_serv,handle_call,3,[{file,"inc_serv.erl"},{line,22}]},
              {gen_server,handle_msg,5,[{file,"gen_server.erl"},{line,588}]},
              {proc_lib,init_p_do_apply,3,
                        [{file,"proc_lib.erl"},{line,227}]}]}

=ERROR REPORT==== 23-Aug-2012::19:32:06 ===
** Generic server supervisor_inc terminating 
** Last message in was {'EXIT',<0.31.0>,
                           {{{badarith,
                                 [{inc_serv,handle_call,3,
                                      [{file,"inc_serv.erl"},{line,22}]},
                                  {gen_server,handle_msg,5,
                                      [{file,"gen_server.erl"},{line,588}]},
                                  {proc_lib,init_p_do_apply,3,
                                      [{file,"proc_lib.erl"},{line,227}]}]},
                             {gen_server,call,[inc_serv,{num,bad_arg}]}},
                            [{gen_server,call,2,
                                 [{file,"gen_server.erl"},{line,180}]},
                             {erl_eval,do_apply,6,
                                 [{file,"erl_eval.erl"},{line,576}]},
                             {shell,exprs,7,[{file,"shell.erl"},{line,668}]},
                             {shell,eval_exprs,7,
                                 [{file,"shell.erl"},{line,623}]},
                             {shell,eval_loop,3,
                                 [{file,"shell.erl"},{line,608}]}]}}
** When Server state == {state,
                            {local,supervisor_inc},
                            one_for_one,
                            [{child,<0.48.0>,inc_serv,
                                 {inc_serv,start,[]},
                                 permanent,2000,worker,
                                 [inc_serv]}],
                            undefined,1,1,
                            [{1345,739526,107495}],
                            supervisor_inc,no_args}
** Reason for termination == 
** {{{badarith,[{inc_serv,handle_call,3,[{file,"inc_serv.erl"},{line,22}]},
                {gen_server,handle_msg,5,[{file,"gen_server.erl"},{line,588}]},
                {proc_lib,init_p_do_apply,3,
                          [{file,"proc_lib.erl"},{line,227}]}]},
     {gen_server,call,[inc_serv,{num,bad_arg}]}},
    [{gen_server,call,2,[{file,"gen_server.erl"},{line,180}]},
     {erl_eval,do_apply,6,[{file,"erl_eval.erl"},{line,576}]},
     {shell,exprs,7,[{file,"shell.erl"},{line,668}]},
     {shell,eval_exprs,7,[{file,"shell.erl"},{line,623}]},
     {shell,eval_loop,3,[{file,"shell.erl"},{line,608}]}]}
** exception exit: {{badarith,[{inc_serv,handle_call,3,
                                         [{file,"inc_serv.erl"},{line,22}]},
                               {gen_server,handle_msg,5,
                                           [{file,"gen_server.erl"},{line,588}]},
                               {proc_lib,init_p_do_apply,3,
                                         [{file,"proc_lib.erl"},{line,227}]}]},
                    {gen_server,call,[inc_serv,{num,bad_arg}]}}
     in function  gen_server:call/2 (gen_server.erl, line 180)

After this I've expected - my supervisor to restart inc_serv. But it didn't:

7> inc_serv:inc( 8 ).      
** exception exit: {noproc,{gen_server,call,[inc_serv,{num,8}]}}
     in function  gen_server:call/2 (gen_server.erl, line 180)

Could you help me to understand what happened? And how should I rewrite my supervisor, to make it able to restart inc_serv

Thanks

Answer 1

This is actually a kind of race condition.

As you might know, the Erlang shell itself is a normal Erlang process. When you start your supervisor from the shell, the supervisor is linked to the shell (because you use supervisor:start_link/3).

When you call your gen_server process, that process crashes (and is correctly restarted by the supervisor, as you can see by the subsequent "Increment server started :)" output).

However, at the same time, your call to gen_server:call/2 will result in the same crash (a gen_server crashing during the call will emit the same crash through the gen_server:call/2 function). This then crashes the shell process, which is linked to your supervisor, which in turn crashes with the same reason (badarith).

Basically, your supervisor is backstabbed by your shell process, after it loyally restarted your gen_server. Like so:

       +---------(6)exit----------+    +---------(5)restart---------+
       |                          |    |                            |
       |                          v    |                            v
     Shell ---(1)start_link---> supervisor ---(2)start_link---> gen_server
     |  ^                         ^    |                         ^  |   ^
     |  |                         |    |                         |  |   |
     |  |                         |    +---------(7)exit---------+  |   |
     |  |                         |                                 |   |
     |  +-------------------------+--------------(4)exit------------+   |
     |                                                                  |
     +---------------------------(3)call--------------------------------+

You can avoid this by calling catch inc_serv:inc(bad_arg). in your shell:

90> inc_serv:inc(7).        
8
91> catch inc_serv:inc(bad_arg).
"Increment server stopped"

=ERROR REPORT==== 23-Aug-2012::22:10:02 ===
** Generic server inc_serv terminating 
** Last message in was {num,bad_arg}
** When Server state == no_state
** Reason for termination == 
** {badarith,[{inc_serv,handle_call,3,[{file,"inc_serv.erl"},{line,20}]},
              {gen_server,handle_msg,5,[{file,"gen_server.erl"},{line,588}]},
              {proc_lib,init_p_do_apply,3,
                        [{file,"proc_lib.erl"},{line,227}]}]}
"Increment server started :)"
{'EXIT',{{badarith,[{inc_serv,handle_call,3,
                              [{file,"inc_serv.erl"},{line,20}]},
                    {gen_server,handle_msg,5,
                              [{file,"gen_server.erl"},{line,588}]},
                    {proc_lib,init_p_do_apply,3,
                              [{file,"proc_lib.erl"},{line,227}]}]},
                    {gen_server,call,[inc_serv,{num,bad_arg}]}}}
92> inc_serv:inc(7).            
8