Federation supervisor crash during upgrade to 4.2.x on multi-node cluster

[gomoripeti]

Community Support Policy

• I have read RabbitMQ's Community Support Policy

RabbitMQ version used

4.2.x

How is RabbitMQ deployed?

Debian package

Steps to reproduce the behavior in question

• Create a 3 node cluster on 4.1.7 and another (single node) cluster that will serve as federation upstream
• Set up exchange federation from the 3 node cluster to the upstream cluster
• Upgrade the 3 node cluster to 4.2.2 in a rolling fashion

Supervisor rabbit_federation_exchange_link_sup_sup crashes with the following crash and is not running any more

<0.748.0>     supervisor: {local,rabbit_federation_exchange_link_sup_sup}
<0.748.0>     errorContext: child_terminated
<0.748.0>     reason: {shutdown,
<0.748.0>              [{{'EXIT',
<0.748.0>                 {undef,
<0.748.0>                  [{rabbit_federation_link_sup,start_link,

full crash log

``` <0.533.0> rabbit on node 'rabbit@host-03' down <0.748.0> supervisor: {local,rabbit_federation_exchange_link_sup_sup} <0.748.0> errorContext: child_terminated <0.748.0> reason: {shutdown, <0.748.0> [{{'EXIT', <0.748.0> {undef, <0.748.0> [{rabbit_federation_link_sup,start_link, <0.748.0> [{exchange, <0.748.0> {resource,<<"vhost1">>,exchange,<<"fed_ex3">>}, <0.748.0> fanout,true,false,false,[],undefined, <0.748.0> [{vhost,<<"vhost1">>}, <0.748.0> {name,<<"fed_on_upgrade_policy">>}, <0.748.0> {pattern,<<"^fed_.*$">>}, <0.748.0> {'apply-to',<<"all">>}, <0.748.0> {definition, <0.748.0> [{<<"federation-upstream">>,<<"fed_upstream">>}]}, <0.748.0> {priority,2}], <0.748.0> undefined, <0.748.0> {[], <0.748.0> [rabbit_federation_exchange, <0.748.0> rabbit_event_exchange_decorator]}, <0.748.0> #{user => <<"vhost1">>}}], <0.748.0> []}, <0.748.0> {supervisor2,do_start_child_i,3, <0.748.0> [{file,"supervisor2.erl"},{line,441}]}, <0.748.0> {supervisor2,do_start_child,2, <0.748.0> [{file,"supervisor2.erl"},{line,427}]}, <0.748.0> {supervisor2,handle_start_child,2, <0.748.0> [{file,"supervisor2.erl"},{line,750}]}, <0.748.0> {supervisor2,handle_call,3, <0.748.0> [{file,"supervisor2.erl"},{line,482}]}, <0.748.0> {gen_server,try_handle_call,4, <0.748.0> [{file,"gen_server.erl"},{line,2470}]}, <0.748.0> {gen_server,handle_msg,3, <0.748.0> [{file,"gen_server.erl"},{line,2499}]}, <0.748.0> {proc_lib,init_p_do_apply,3, <0.748.0> [{file,"proc_lib.erl"},{line,333}]}]}}, <0.748.0> {child,undefined, <0.748.0> {exchange, <0.748.0> {resource,<<"vhost1">>,exchange,<<"fed_ex3">>}, <0.748.0> fanout,true,false,false,[],none, <0.748.0> [{vhost,<<"vhost1">>}, <0.748.0> {name,<<"fed_on_upgrade_policy">>}, <0.748.0> {pattern,<<"^fed_.*$">>}, <0.748.0> {'apply-to',<<"all">>}, <0.748.0> {definition, <0.748.0> [{<<"federation-upstream">>,<<"fed_upstream">>}]}, <0.748.0> {priority,2}], <0.748.0> undefined,none, <0.748.0> #{user => <<"vhost1">>}}, <0.748.0> {rabbit_federation_link_sup,start_link, <0.748.0> [{exchange, <0.748.0> {resource,<<"vhost1">>,exchange,<<"fed_ex3">>}, <0.748.0> fanout,true,false,false,[],undefined, <0.748.0> [{vhost,<<"vhost1">>}, <0.748.0> {name,<<"fed_on_upgrade_policy">>}, <0.748.0> {pattern,<<"^fed_.*$">>}, <0.748.0> {'apply-to',<<"all">>}, <0.748.0> {definition, <0.748.0> [{<<"federation-upstream">>,<<"fed_upstream">>}]}, <0.748.0> {priority,2}], <0.748.0> undefined, <0.748.0> {[], <0.748.0> [rabbit_federation_exchange, <0.748.0> rabbit_event_exchange_decorator]}, <0.748.0> #{user => <<"vhost1">>}}]}, <0.748.0> transient,infinity,supervisor, <0.748.0> [rabbit_federation_link_sup]}}, <0.748.0> {{'EXIT', <0.748.0> {undef, <0.748.0> [{rabbit_federation_link_sup,start_link, <0.748.0> [{exchange, <0.748.0> {resource,<<"vhost1">>,exchange,<<"fed_ex2">>}, <0.748.0> fanout,true,false,false,[],undefined, <0.748.0> [{vhost,<<"vhost1">>}, <0.748.0> {name,<<"fed_on_upgrade_policy">>}, <0.748.0> {pattern,<<"^fed_.*$">>}, <0.748.0> {'apply-to',<<"all">>}, <0.748.0> {definition, <0.748.0> [{<<"federation-upstream">>,<<"fed_upstream">>}]}, <0.748.0> {priority,2}], <0.748.0> undefined, <0.748.0> {[], <0.748.0> [rabbit_federation_exchange, <0.748.0> rabbit_event_exchange_decorator]}, <0.748.0> #{user => <<"vhost1">>}}], <0.748.0> []}, <0.748.0> {supervisor2,do_start_child_i,3, <0.748.0> [{file,"supervisor2.erl"},{line,441}]}, <0.748.0> {supervisor2,do_start_child,2, <0.748.0> [{file,"supervisor2.erl"},{line,427}]}, <0.748.0> {supervisor2,handle_start_child,2, <0.748.0> [{file,"supervisor2.erl"},{line,750}]}, <0.748.0> {supervisor2,handle_call,3, <0.748.0> [{file,"supervisor2.erl"},{line,482}]}, <0.748.0> {gen_server,try_handle_call,4, <0.748.0> [{file,"gen_server.erl"},{line,2470}]}, <0.748.0> {gen_server,handle_msg,3, <0.748.0> [{file,"gen_server.erl"},{line,2499}]}, <0.748.0> {proc_lib,init_p_do_apply,3, <0.748.0> [{file,"proc_lib.erl"},{line,333}]}]}}, <0.748.0> {child,undefined, <0.748.0> {exchange, <0.748.0> {resource,<<"vhost1">>,exchange,<<"fed_ex2">>}, <0.748.0> fanout,true,false,false,[],none, <0.748.0> [{vhost,<<"vhost1">>}, <0.748.0> {name,<<"fed_on_upgrade_policy">>}, <0.748.0> {pattern,<<"^fed_.*$">>}, <0.748.0> {'apply-to',<<"all">>}, <0.748.0> {definition, <0.748.0> [{<<"federation-upstream">>,<<"fed_upstream">>}]}, <0.748.0> {priority,2}], <0.748.0> undefined,none, <0.748.0> #{user => <<"vhost1">>}}, <0.748.0> {rabbit_federation_link_sup,start_link, <0.748.0> [{exchange, <0.748.0> {resource,<<"vhost1">>,exchange,<<"fed_ex2">>}, <0.748.0> fanout,true,false,false,[],undefined, <0.748.0> [{vhost,<<"vhost1">>}, <0.748.0> {name,<<"fed_on_upgrade_policy">>}, <0.748.0> {pattern,<<"^fed_.*$">>}, <0.748.0> {'apply-to',<<"all">>}, <0.748.0> {definition, <0.748.0> [{<<"federation-upstream">>,<<"fed_upstream">>}]}, <0.748.0> {priority,2}], <0.748.0> undefined, <0.748.0> {[], <0.748.0> [rabbit_federation_exchange, <0.748.0> rabbit_event_exchange_decorator]}, <0.748.0> #{user => <<"vhost1">>}}]}, <0.748.0> transient,infinity,supervisor, <0.748.0> [rabbit_federation_link_sup]}}, <0.748.0> {{'EXIT', <0.748.0> {undef, <0.748.0> [{rabbit_federation_link_sup,start_link, <0.748.0> [{exchange, <0.748.0> {resource,<<"vhost1">>,exchange,<<"fed_ex">>}, <0.748.0> fanout,true,false,false,[],undefined, <0.748.0> [{vhost,<<"vhost1">>}, <0.748.0> {name,<<"fed_on_upgrade_policy">>}, <0.748.0> {pattern,<<"^fed_.*$">>}, <0.748.0> {'apply-to',<<"all">>}, <0.748.0> {definition, <0.748.0> [{<<"federation-upstream">>,<<"fed_upstream">>}]}, <0.748.0> {priority,2}], <0.748.0> undefined, <0.748.0> {[], <0.748.0> [rabbit_federation_exchange, <0.748.0> rabbit_event_exchange_decorator]}, <0.748.0> #{user => <<"vhost1">>}}], <0.748.0> []}, <0.748.0> {supervisor2,do_start_child_i,3, <0.748.0> [{file,"supervisor2.erl"},{line,441}]}, <0.748.0> {supervisor2,do_start_child,2, <0.748.0> [{file,"supervisor2.erl"},{line,427}]}, <0.748.0> {supervisor2,handle_start_child,2, <0.748.0> [{file,"supervisor2.erl"},{line,750}]}, <0.748.0> {supervisor2,handle_call,3, <0.748.0> [{file,"supervisor2.erl"},{line,482}]}, <0.748.0> {gen_server,try_handle_call,4, <0.748.0> [{file,"gen_server.erl"},{line,2470}]}, <0.748.0> {gen_server,handle_msg,3, <0.748.0> [{file,"gen_server.erl"},{line,2499}]}, <0.748.0> {proc_lib,init_p_do_apply,3, <0.748.0> [{file,"proc_lib.erl"},{line,333}]}]}}, <0.748.0> {child,undefined, <0.748.0> {exchange, <0.748.0> {resource,<<"vhost1">>,exchange,<<"fed_ex">>}, <0.748.0> fanout,true,false,false,[],none, <0.748.0> [{vhost,<<"vhost1">>}, <0.748.0> {name,<<"fed_on_upgrade_policy">>}, <0.748.0> {pattern,<<"^fed_.*$">>}, <0.748.0> {'apply-to',<<"all">>}, <0.748.0> {definition, <0.748.0> [{<<"federation-upstream">>,<<"fed_upstream">>}]}, <0.748.0> {priority,2}], <0.748.0> undefined,none, <0.748.0> #{user => <<"vhost1">>}}, <0.748.0> {rabbit_federation_link_sup,start_link, <0.748.0> [{exchange, <0.748.0> {resource,<<"vhost1">>,exchange,<<"fed_ex">>}, <0.748.0> fanout,true,false,false,[],undefined, <0.748.0> [{vhost,<<"vhost1">>}, <0.748.0> {name,<<"fed_on_upgrade_policy">>}, <0.748.0> {pattern,<<"^fed_.*$">>}, <0.748.0> {'apply-to',<<"all">>}, <0.748.0> {definition, <0.748.0> [{<<"federation-upstream">>,<<"fed_upstream">>}]}, <0.748.0> {priority,2}], <0.748.0> undefined, <0.748.0> {[], <0.748.0> [rabbit_federation_exchange, <0.748.0> rabbit_event_exchange_decorator]}, <0.748.0> #{user => <<"vhost1">>}}]}, <0.748.0> transient,infinity,supervisor, <0.748.0> [rabbit_federation_link_sup]}}]} <0.748.0> offender: [{pid,<0.750.0>}, <0.748.0> {id,mirroring}, <0.748.0> {mfargs, <0.748.0> {mirrored_supervisor,start_internal, <0.748.0> [rabbit_federation_exchange_link_sup_sup,[]]}}, <0.748.0> {restart_type,permanent}, <0.748.0> {shutdown,4294967295}, <0.748.0> {child_type,worker}] <0.748.0> supervisor: {local,rabbit_federation_exchange_link_sup_sup} <0.748.0> errorContext: shutdown <0.748.0> reason: reached_max_restart_intensity <0.748.0> offender: [{pid,<0.750.0>}, <0.748.0> {id,mirroring}, <0.748.0> {mfargs, <0.748.0> {mirrored_supervisor,start_internal, <0.748.0> [rabbit_federation_exchange_link_sup_sup,[]]}}, <0.748.0> {restart_type,permanent}, <0.748.0> {shutdown,4294967295}, <0.748.0> {child_type,worker}] ```

4> supervisor:which_children(rabbit_exchange_federation_sup).
[{x_links,undefined,supervisor,
          [rabbit_federation_exchange_link_sup_sup]}]
5> whereis(rabbit_federation_exchange_link_sup_sup).
undefined

This is because rabbit_federation_link_sup:start_link had arity 1 until 4.1.x. PR #13921 converted it to arity 2 (https://github.com/rabbitmq/rabbitmq-server/pull/13921/changes#diff-331ecadea9665c8db269b5f30f87851eddeb09b6968ec5daeae0c34355cbd4fbR39). But in a mixed version cluster the mirrored supervisor preserves the child definitions which still include a call with arity 1 (without the link module).

Maybe adding a compatibility start_link/1 would solve this?

modified   deps/rabbitmq_federation_common/src/rabbit_federation_link_sup.erl
@@ -17,6 +17,13 @@
 
 -export([start_link/2, adjust/4, restart/2]).
 -export([init/1]).
+-export([start_link/1]).
+
+%% For backwards compatibility
+start_link(X = #exchange{}) ->
+    start_link(rabbit_federation_exchange_link, X);
+start_link(Q) when ?is_amqqueue(Q) ->
+    start_link(rabbit_federation_queue_link, Q).
 
 start_link(LinkMod, Q) ->
     supervisor2:start_link(?MODULE, [LinkMod, Q]).

[michaelklishin]

@gomoripeti the proposed change looks acceptable to me. If we want to include it, we must do it early next week because 4.1.8 will ship in mid-January and it will be the last publicly available 4.1.x release.

[gomoripeti]

splitting the federation plugin was included in 4.2.0. So I think the patch to be backwards compatible should be included in 4.2.3 and not making 4.1.8 forward compatible. (With the caveat that on multi-node with federation, upgrading from 3.13.x/4.0.x/4.1.x to 4.2.0..4.2.2 does not work)

I will try to write a mixed version test case for this scenario.