2 * Copyright (c) 2005 MontaVista Software, Inc.
3 * Copyright (c) 2006 Red Hat, Inc.
4 * Copyright (c) 2006 Sun Microsystems, Inc.
8 * Author: Steven Dake (sdake@mvista.com)
10 * This software licensed under BSD license, the text of which follows:
12 * Redistribution and use in source and binary forms, with or without
13 * modification, are permitted provided that the following conditions are met:
15 * - Redistributions of source code must retain the above copyright notice,
16 * this list of conditions and the following disclaimer.
17 * - Redistributions in binary form must reproduce the above copyright notice,
18 * this list of conditions and the following disclaimer in the documentation
19 * and/or other materials provided with the distribution.
20 * - Neither the name of the MontaVista Software, Inc. nor the names of its
21 * contributors may be used to endorse or promote products derived from this
22 * software without specific prior written permission.
24 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
25 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
27 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
28 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
31 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
32 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
33 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
34 * THE POSSIBILITY OF SUCH DAMAGE.
40 #include <sys/types.h>
42 #include <sys/socket.h>
45 #include <sys/ioctl.h>
46 #include <sys/param.h>
47 #include <netinet/in.h>
48 #include <arpa/inet.h>
60 #include "../include/queue.h"
61 #include "../include/sq.h"
62 #include "../include/list.h"
63 #include "../include/hdb.h"
69 struct totemrrp_instance
;
70 struct passive_instance
{
71 struct totemrrp_instance
*rrp_instance
;
73 unsigned int *token_recv_count
;
74 unsigned int *mcast_recv_count
;
75 unsigned char token
[15000];
76 unsigned int token_len
;
77 poll_timer_handle timer_expired_token
;
78 poll_timer_handle timer_problem_decrementer
;
79 void *totemrrp_context
;
80 unsigned int token_xmit_iface
;
81 unsigned int msg_xmit_iface
;
84 struct active_instance
{
85 struct totemrrp_instance
*rrp_instance
;
87 unsigned int *last_token_recv
;
88 unsigned int *counter_problems
;
89 unsigned char token
[15000];
90 unsigned int token_len
;
91 unsigned int last_token_seq
;
92 poll_timer_handle timer_expired_token
;
93 poll_timer_handle timer_problem_decrementer
;
94 void *totemrrp_context
;
100 void * (*initialize
) (
101 struct totemrrp_instance
*rrp_instance
,
102 int interface_count
);
105 struct totemrrp_instance
*instance
,
106 unsigned int iface_no
,
109 unsigned int msg_len
);
111 void (*mcast_noflush_send
) (
112 struct totemrrp_instance
*instance
,
114 unsigned int iov_len
);
116 void (*mcast_flush_send
) (
117 struct totemrrp_instance
*instance
,
119 unsigned int iov_len
);
122 struct totemrrp_instance
*instance
,
123 unsigned int iface_no
,
126 unsigned int msg_len
,
127 unsigned int token_seqid
);
130 struct totemrrp_instance
*instance
,
132 unsigned int iov_len
);
135 struct totemrrp_instance
*instance
);
138 struct totemrrp_instance
*instance
);
140 void (*iface_check
) (
141 struct totemrrp_instance
*instance
);
143 void (*processor_count_set
) (
144 struct totemrrp_instance
*instance
,
145 unsigned int processor_count
);
147 void (*token_target_set
) (
148 struct totemrrp_instance
*instance
,
149 struct totem_ip_address
*token_target
,
150 unsigned int iface_no
);
152 void (*ring_reenable
) (
153 struct totemrrp_instance
*instance
);
156 struct totemrrp_instance
{
157 poll_handle totemrrp_poll_handle
;
159 struct totem_interface
*interfaces
;
161 struct rrp_algo
*rrp_algo
;
165 char *status
[INTERFACE_MAX
];
167 void (*totemrrp_deliver_fn
) (
172 void (*totemrrp_iface_change_fn
) (
174 struct totem_ip_address
*iface_addr
,
175 unsigned int iface_no
);
177 void (*totemrrp_token_seqid_get
) (
180 unsigned int *token_is
);
182 unsigned int (*totemrrp_msgs_missing
) (void);
185 * Function and data used to log messages
187 int totemrrp_log_level_security
;
189 int totemrrp_log_level_error
;
191 int totemrrp_log_level_warning
;
193 int totemrrp_log_level_notice
;
195 int totemrrp_log_level_debug
;
197 void (*totemrrp_log_printf
) (char *file
, int line
, int level
, char *format
, ...) __attribute__((format(printf
, 4, 5)));
199 totemrrp_handle handle
;
201 totemnet_handle
*net_handles
;
203 void *rrp_algo_instance
;
211 struct totem_config
*totem_config
;
215 * None Replication Forward Declerations
217 static void none_mcast_recv (
218 struct totemrrp_instance
*instance
,
219 unsigned int iface_no
,
222 unsigned int msg_len
);
224 static void none_mcast_noflush_send (
225 struct totemrrp_instance
*instance
,
227 unsigned int iov_len
);
229 static void none_mcast_flush_send (
230 struct totemrrp_instance
*instance
,
232 unsigned int iov_len
);
234 static void none_token_recv (
235 struct totemrrp_instance
*instance
,
236 unsigned int iface_no
,
239 unsigned int msg_len
,
240 unsigned int token_seqid
);
242 static void none_token_send (
243 struct totemrrp_instance
*instance
,
245 unsigned int iov_len
);
247 static void none_recv_flush (
248 struct totemrrp_instance
*instance
);
250 static void none_send_flush (
251 struct totemrrp_instance
*instance
);
253 static void none_iface_check (
254 struct totemrrp_instance
*instance
);
256 static void none_processor_count_set (
257 struct totemrrp_instance
*instance
,
258 unsigned int processor_count_set
);
260 static void none_token_target_set (
261 struct totemrrp_instance
*instance
,
262 struct totem_ip_address
*token_target
,
263 unsigned int iface_no
);
265 static void none_ring_reenable (
266 struct totemrrp_instance
*instance
);
269 * Passive Replication Forward Declerations
271 static void *passive_instance_initialize (
272 struct totemrrp_instance
*rrp_instance
,
273 int interface_count
);
275 static void passive_mcast_recv (
276 struct totemrrp_instance
*instance
,
277 unsigned int iface_no
,
280 unsigned int msg_len
);
282 static void passive_mcast_noflush_send (
283 struct totemrrp_instance
*instance
,
285 unsigned int iov_len
);
287 static void passive_mcast_flush_send (
288 struct totemrrp_instance
*instance
,
290 unsigned int iov_len
);
292 static void passive_token_recv (
293 struct totemrrp_instance
*instance
,
294 unsigned int iface_no
,
297 unsigned int msg_len
,
298 unsigned int token_seqid
);
300 static void passive_token_send (
301 struct totemrrp_instance
*instance
,
303 unsigned int iov_len
);
305 static void passive_recv_flush (
306 struct totemrrp_instance
*instance
);
308 static void passive_send_flush (
309 struct totemrrp_instance
*instance
);
311 static void passive_iface_check (
312 struct totemrrp_instance
*instance
);
314 static void passive_processor_count_set (
315 struct totemrrp_instance
*instance
,
316 unsigned int processor_count_set
);
318 static void passive_token_target_set (
319 struct totemrrp_instance
*instance
,
320 struct totem_ip_address
*token_target
,
321 unsigned int iface_no
);
323 static void passive_ring_reenable (
324 struct totemrrp_instance
*instance
);
327 * Active Replication Forward Definitions
329 static void *active_instance_initialize (
330 struct totemrrp_instance
*rrp_instance
,
331 int interface_count
);
333 static void active_mcast_recv (
334 struct totemrrp_instance
*instance
,
335 unsigned int iface_no
,
338 unsigned int msg_len
);
340 static void active_mcast_noflush_send (
341 struct totemrrp_instance
*instance
,
343 unsigned int iov_len
);
345 static void active_mcast_flush_send (
346 struct totemrrp_instance
*instance
,
348 unsigned int iov_len
);
350 static void active_token_recv (
351 struct totemrrp_instance
*instance
,
352 unsigned int iface_no
,
355 unsigned int msg_len
,
356 unsigned int token_seqid
);
358 static void active_token_send (
359 struct totemrrp_instance
*instance
,
361 unsigned int iov_len
);
363 static void active_recv_flush (
364 struct totemrrp_instance
*instance
);
366 static void active_send_flush (
367 struct totemrrp_instance
*instance
);
369 static void active_iface_check (
370 struct totemrrp_instance
*instance
);
372 static void active_processor_count_set (
373 struct totemrrp_instance
*instance
,
374 unsigned int processor_count_set
);
376 static void active_token_target_set (
377 struct totemrrp_instance
*instance
,
378 struct totem_ip_address
*token_target
,
379 unsigned int iface_no
);
381 static void active_ring_reenable (
382 struct totemrrp_instance
*instance
);
384 static void active_timer_expired_token_start (
385 struct active_instance
*active_instance
);
387 static void active_timer_expired_token_cancel (
388 struct active_instance
*active_instance
);
390 static void active_timer_problem_decrementer_start (
391 struct active_instance
*active_instance
);
393 static void active_timer_problem_decrementer_cancel (
394 struct active_instance
*active_instance
);
396 struct rrp_algo none_algo
= {
399 .mcast_recv
= none_mcast_recv
,
400 .mcast_noflush_send
= none_mcast_noflush_send
,
401 .mcast_flush_send
= none_mcast_flush_send
,
402 .token_recv
= none_token_recv
,
403 .token_send
= none_token_send
,
404 .recv_flush
= none_recv_flush
,
405 .send_flush
= none_send_flush
,
406 .iface_check
= none_iface_check
,
407 .processor_count_set
= none_processor_count_set
,
408 .token_target_set
= none_token_target_set
,
409 .ring_reenable
= none_ring_reenable
412 struct rrp_algo passive_algo
= {
414 .initialize
= passive_instance_initialize
,
415 .mcast_recv
= passive_mcast_recv
,
416 .mcast_noflush_send
= passive_mcast_noflush_send
,
417 .mcast_flush_send
= passive_mcast_flush_send
,
418 .token_recv
= passive_token_recv
,
419 .token_send
= passive_token_send
,
420 .recv_flush
= passive_recv_flush
,
421 .send_flush
= passive_send_flush
,
422 .iface_check
= passive_iface_check
,
423 .processor_count_set
= passive_processor_count_set
,
424 .token_target_set
= passive_token_target_set
,
425 .ring_reenable
= passive_ring_reenable
428 struct rrp_algo active_algo
= {
430 .initialize
= active_instance_initialize
,
431 .mcast_recv
= active_mcast_recv
,
432 .mcast_noflush_send
= active_mcast_noflush_send
,
433 .mcast_flush_send
= active_mcast_flush_send
,
434 .token_recv
= active_token_recv
,
435 .token_send
= active_token_send
,
436 .recv_flush
= active_recv_flush
,
437 .send_flush
= active_send_flush
,
438 .iface_check
= active_iface_check
,
439 .processor_count_set
= active_processor_count_set
,
440 .token_target_set
= active_token_target_set
,
441 .ring_reenable
= active_ring_reenable
444 struct rrp_algo
*rrp_algos
[] = {
450 #define RRP_ALGOS_COUNT 3
453 * All instances in one database
455 static struct hdb_handle_database totemrrp_instance_database
= {
459 .mutex
= PTHREAD_MUTEX_INITIALIZER
462 #define log_printf(level, format, args...) \
463 rrp_instance->totemrrp_log_printf (__FILE__, __LINE__, level, format, ##args)
466 * None Replication Implementation
469 static void none_mcast_recv (
470 struct totemrrp_instance
*rrp_instance
,
471 unsigned int iface_no
,
474 unsigned int msg_len
)
476 rrp_instance
->totemrrp_deliver_fn (
482 static void none_mcast_flush_send (
483 struct totemrrp_instance
*instance
,
485 unsigned int iov_len
)
487 totemnet_mcast_flush_send (instance
->net_handles
[0], iovec
, iov_len
);
490 static void none_mcast_noflush_send (
491 struct totemrrp_instance
*instance
,
493 unsigned int iov_len
)
495 totemnet_mcast_noflush_send (instance
->net_handles
[0], iovec
, iov_len
);
498 static void none_token_recv (
499 struct totemrrp_instance
*rrp_instance
,
500 unsigned int iface_no
,
503 unsigned int msg_len
,
504 unsigned int token_seq
)
506 rrp_instance
->totemrrp_deliver_fn (
512 static void none_token_send (
513 struct totemrrp_instance
*instance
,
515 unsigned int iov_len
)
517 totemnet_token_send (
518 instance
->net_handles
[0],
522 static void none_recv_flush (struct totemrrp_instance
*instance
)
524 totemnet_recv_flush (instance
->net_handles
[0]);
527 static void none_send_flush (struct totemrrp_instance
*instance
)
529 totemnet_send_flush (instance
->net_handles
[0]);
532 static void none_iface_check (struct totemrrp_instance
*instance
)
534 totemnet_iface_check (instance
->net_handles
[0]);
537 static void none_processor_count_set (
538 struct totemrrp_instance
*instance
,
539 unsigned int processor_count
)
541 totemnet_processor_count_set (instance
->net_handles
[0],
545 static void none_token_target_set (
546 struct totemrrp_instance
*instance
,
547 struct totem_ip_address
*token_target
,
548 unsigned int iface_no
)
550 totemnet_token_target_set (instance
->net_handles
[0], token_target
);
553 static void none_ring_reenable (
554 struct totemrrp_instance
*instance
)
562 * Passive Replication Implementation
564 void *passive_instance_initialize (
565 struct totemrrp_instance
*rrp_instance
,
568 struct passive_instance
*instance
;
570 instance
= malloc (sizeof (struct passive_instance
));
574 memset (instance
, 0, sizeof (struct passive_instance
));
576 instance
->faulty
= malloc (sizeof (int) * interface_count
);
577 if (instance
->faulty
== 0) {
582 memset (instance
->faulty
, 0, sizeof (int) * interface_count
);
584 instance
->token_recv_count
= malloc (sizeof (int) * interface_count
);
585 if (instance
->token_recv_count
== 0) {
586 free (instance
->faulty
);
591 memset (instance
->token_recv_count
, 0, sizeof (int) * interface_count
);
593 instance
->mcast_recv_count
= malloc (sizeof (int) * interface_count
);
594 if (instance
->mcast_recv_count
== 0) {
595 free (instance
->token_recv_count
);
596 free (instance
->faulty
);
601 memset (instance
->mcast_recv_count
, 0, sizeof (int) * interface_count
);
604 return ((void *)instance
);
607 static void timer_function_passive_token_expired (void *context
)
609 struct passive_instance
*passive_instance
= (struct passive_instance
*)context
;
610 struct totemrrp_instance
*rrp_instance
= passive_instance
->rrp_instance
;
612 rrp_instance
->totemrrp_deliver_fn (
613 passive_instance
->totemrrp_context
,
614 passive_instance
->token
,
615 passive_instance
->token_len
);
619 static void timer_function_passive_problem_decrementer (void *context)
621 // struct passive_instance *passive_instance = (struct passive_instance *)context;
622 // struct totemrrp_instance *rrp_instance = passive_instance->rrp_instance;
628 static void passive_timer_expired_token_start (
629 struct passive_instance
*passive_instance
)
632 passive_instance
->rrp_instance
->poll_handle
,
633 passive_instance
->rrp_instance
->totem_config
->rrp_token_expired_timeout
,
634 (void *)passive_instance
,
635 timer_function_passive_token_expired
,
636 &passive_instance
->timer_expired_token
);
639 static void passive_timer_expired_token_cancel (
640 struct passive_instance
*passive_instance
)
643 passive_instance
->rrp_instance
->poll_handle
,
644 passive_instance
->timer_expired_token
);
648 static void passive_timer_problem_decrementer_start (
649 struct passive_instance *passive_instance)
652 passive_instance->rrp_instance->poll_handle,
653 passive_instance->rrp_instance->totem_config->rrp_problem_count_timeout,
654 (void *)passive_instance,
655 timer_function_passive_problem_decrementer,
656 &passive_instance->timer_problem_decrementer);
659 static void passive_timer_problem_decrementer_cancel (
660 struct passive_instance *passive_instance)
663 passive_instance->rrp_instance->poll_handle,
664 passive_instance->timer_problem_decrementer);
669 static void passive_mcast_recv (
670 struct totemrrp_instance
*rrp_instance
,
671 unsigned int iface_no
,
674 unsigned int msg_len
)
676 struct passive_instance
*passive_instance
= (struct passive_instance
*)rrp_instance
->rrp_algo_instance
;
680 rrp_instance
->totemrrp_deliver_fn (
685 if (rrp_instance
->totemrrp_msgs_missing() == 0 &&
686 passive_instance
->timer_expired_token
) {
688 * Delivers the last token
690 rrp_instance
->totemrrp_deliver_fn (
691 passive_instance
->totemrrp_context
,
692 passive_instance
->token
,
693 passive_instance
->token_len
);
694 passive_timer_expired_token_cancel (passive_instance
);
698 * Monitor for failures
699 * TODO doesn't handle wrap-around of the mcast recv count
701 passive_instance
->mcast_recv_count
[iface_no
] += 1;
703 for (i
= 0; i
< rrp_instance
->interface_count
; i
++) {
704 if (max
< passive_instance
->mcast_recv_count
[i
]) {
705 max
= passive_instance
->mcast_recv_count
[i
];
709 for (i
= 0; i
< rrp_instance
->interface_count
; i
++) {
710 if ((passive_instance
->faulty
[i
] == 0) &&
711 (max
- passive_instance
->mcast_recv_count
[i
] >
712 rrp_instance
->totem_config
->rrp_problem_count_threshold
)) {
713 passive_instance
->faulty
[i
] = 1;
714 sprintf (rrp_instance
->status
[i
],
715 "Marking ringid %u interface %s FAULTY - adminisrtative intervention required.",
717 totemnet_iface_print (rrp_instance
->net_handles
[i
]));
719 rrp_instance
->totemrrp_log_level_error
,
720 rrp_instance
->status
[i
]);
725 static void passive_mcast_flush_send (
726 struct totemrrp_instance
*instance
,
728 unsigned int iov_len
)
730 struct passive_instance
*passive_instance
= (struct passive_instance
*)instance
->rrp_algo_instance
;
733 passive_instance
->msg_xmit_iface
= (passive_instance
->msg_xmit_iface
+ 1) % instance
->interface_count
;
734 } while (passive_instance
->faulty
[passive_instance
->msg_xmit_iface
] == 1);
736 totemnet_mcast_flush_send (instance
->net_handles
[passive_instance
->msg_xmit_iface
], iovec
, iov_len
);
739 static void passive_mcast_noflush_send (
740 struct totemrrp_instance
*instance
,
742 unsigned int iov_len
)
744 struct passive_instance
*passive_instance
= (struct passive_instance
*)instance
->rrp_algo_instance
;
747 passive_instance
->msg_xmit_iface
= (passive_instance
->msg_xmit_iface
+ 1) % instance
->interface_count
;
748 } while (passive_instance
->faulty
[passive_instance
->msg_xmit_iface
] == 1);
751 totemnet_mcast_noflush_send (instance
->net_handles
[passive_instance
->msg_xmit_iface
], iovec
, iov_len
);
754 static void passive_token_recv (
755 struct totemrrp_instance
*rrp_instance
,
756 unsigned int iface_no
,
759 unsigned int msg_len
,
760 unsigned int token_seq
)
762 struct passive_instance
*passive_instance
= (struct passive_instance
*)rrp_instance
->rrp_algo_instance
;
766 passive_instance
->totemrrp_context
= context
; // this should be in totemrrp_instance ? TODO
768 if (rrp_instance
->totemrrp_msgs_missing() == 0) {
769 rrp_instance
->totemrrp_deliver_fn (
774 memcpy (passive_instance
->token
, msg
, msg_len
);
775 passive_timer_expired_token_start (passive_instance
);
780 * Monitor for failures
781 * TODO doesn't handle wrap-around of the token
783 passive_instance
->token_recv_count
[iface_no
] += 1;
785 for (i
= 0; i
< rrp_instance
->interface_count
; i
++) {
786 if (max
< passive_instance
->token_recv_count
[i
]) {
787 max
= passive_instance
->token_recv_count
[i
];
791 for (i
= 0; i
< rrp_instance
->interface_count
; i
++) {
792 if ((passive_instance
->faulty
[i
] == 0) &&
793 (max
- passive_instance
->token_recv_count
[i
] >
794 rrp_instance
->totem_config
->rrp_problem_count_threshold
)) {
795 passive_instance
->faulty
[i
] = 1;
796 sprintf (rrp_instance
->status
[i
],
797 "Marking seqid %d ringid %u interface %s FAULTY - adminisrtative intervention required.",
800 totemnet_iface_print (rrp_instance
->net_handles
[i
]));
802 rrp_instance
->totemrrp_log_level_error
,
803 rrp_instance
->status
[i
]);
808 static void passive_token_send (
809 struct totemrrp_instance
*instance
,
811 unsigned int iov_len
)
813 struct passive_instance
*passive_instance
= (struct passive_instance
*)instance
->rrp_algo_instance
;
816 passive_instance
->token_xmit_iface
= (passive_instance
->token_xmit_iface
+ 1) % instance
->interface_count
;
817 } while (passive_instance
->faulty
[passive_instance
->token_xmit_iface
] == 1);
819 totemnet_token_send (
820 instance
->net_handles
[passive_instance
->token_xmit_iface
],
825 static void passive_recv_flush (struct totemrrp_instance
*instance
)
827 struct passive_instance
*rrp_algo_instance
= (struct passive_instance
*)instance
->rrp_algo_instance
;
830 for (i
= 0; i
< instance
->interface_count
; i
++) {
831 if (rrp_algo_instance
->faulty
[i
] == 0) {
833 totemnet_recv_flush (instance
->net_handles
[i
]);
838 static void passive_send_flush (struct totemrrp_instance
*instance
)
840 struct passive_instance
*rrp_algo_instance
= (struct passive_instance
*)instance
->rrp_algo_instance
;
843 for (i
= 0; i
< instance
->interface_count
; i
++) {
844 if (rrp_algo_instance
->faulty
[i
] == 0) {
846 totemnet_send_flush (instance
->net_handles
[i
]);
851 static void passive_iface_check (struct totemrrp_instance
*instance
)
853 struct passive_instance
*rrp_algo_instance
= (struct passive_instance
*)instance
->rrp_algo_instance
;
856 for (i
= 0; i
< instance
->interface_count
; i
++) {
857 if (rrp_algo_instance
->faulty
[i
] == 0) {
859 totemnet_iface_check (instance
->net_handles
[i
]);
864 static void passive_processor_count_set (
865 struct totemrrp_instance
*instance
,
866 unsigned int processor_count
)
868 struct passive_instance
*rrp_algo_instance
= (struct passive_instance
*)instance
->rrp_algo_instance
;
871 for (i
= 0; i
< instance
->interface_count
; i
++) {
872 if (rrp_algo_instance
->faulty
[i
] == 0) {
874 totemnet_processor_count_set (instance
->net_handles
[i
],
880 static void passive_token_target_set (
881 struct totemrrp_instance
*instance
,
882 struct totem_ip_address
*token_target
,
883 unsigned int iface_no
)
885 totemnet_token_target_set (instance
->net_handles
[iface_no
], token_target
);
888 static void passive_ring_reenable (
889 struct totemrrp_instance
*instance
)
891 struct passive_instance
*rrp_algo_instance
= (struct passive_instance
*)instance
->rrp_algo_instance
;
893 memset (rrp_algo_instance
->mcast_recv_count
, 0, sizeof (unsigned int) *
894 instance
->interface_count
);
895 memset (rrp_algo_instance
->token_recv_count
, 0, sizeof (unsigned int) *
896 instance
->interface_count
);
897 memset (rrp_algo_instance
->faulty
, 0, sizeof (unsigned int) *
898 instance
->interface_count
);
902 * Active Replication Implementation
904 void *active_instance_initialize (
905 struct totemrrp_instance
*rrp_instance
,
908 struct active_instance
*instance
;
910 instance
= malloc (sizeof (struct active_instance
));
914 memset (instance
, 0, sizeof (struct active_instance
));
916 instance
->faulty
= malloc (sizeof (int) * interface_count
);
917 if (instance
->faulty
== 0) {
922 memset (instance
->faulty
, 0, sizeof (unsigned int) * interface_count
);
924 instance
->last_token_recv
= malloc (sizeof (int) * interface_count
);
925 if (instance
->last_token_recv
== 0) {
926 free (instance
->faulty
);
931 memset (instance
->last_token_recv
, 0, sizeof (unsigned int) * interface_count
);
933 instance
->counter_problems
= malloc (sizeof (int) * interface_count
);
934 if (instance
->counter_problems
== 0) {
935 free (instance
->last_token_recv
);
936 free (instance
->faulty
);
941 memset (instance
->counter_problems
, 0, sizeof (unsigned int) * interface_count
);
943 instance
->timer_expired_token
= 0;
945 instance
->timer_problem_decrementer
= 0;
947 instance
->rrp_instance
= rrp_instance
;
950 return ((void *)instance
);
952 static void timer_function_active_problem_decrementer (void *context
)
954 struct active_instance
*active_instance
= (struct active_instance
*)context
;
955 struct totemrrp_instance
*rrp_instance
= active_instance
->rrp_instance
;
956 unsigned int problem_found
= 0;
959 for (i
= 0; i
< rrp_instance
->interface_count
; i
++) {
960 if (active_instance
->counter_problems
[i
] > 0) {
962 active_instance
->counter_problems
[i
] -= 1;
963 if (active_instance
->counter_problems
[i
] == 0) {
964 sprintf (rrp_instance
->status
[i
],
965 "ring %d active with no faults", i
);
967 sprintf (rrp_instance
->status
[i
],
968 "Decrementing problem counter for iface %s to [%d of %d]",
969 totemnet_iface_print (rrp_instance
->net_handles
[i
]),
970 active_instance
->counter_problems
[i
],
971 rrp_instance
->totem_config
->rrp_problem_count_threshold
);
974 rrp_instance
->totemrrp_log_level_warning
,
975 rrp_instance
->status
[i
]);
979 active_timer_problem_decrementer_start (active_instance
);
981 active_instance
->timer_problem_decrementer
= 0;
985 static void timer_function_active_token_expired (void *context
)
987 struct active_instance
*active_instance
= (struct active_instance
*)context
;
988 struct totemrrp_instance
*rrp_instance
= active_instance
->rrp_instance
;
991 for (i
= 0; i
< rrp_instance
->interface_count
; i
++) {
992 if (active_instance
->last_token_recv
[i
] == 0) {
993 active_instance
->counter_problems
[i
] += 1;
995 if (active_instance
->timer_problem_decrementer
== 0) {
996 active_timer_problem_decrementer_start (active_instance
);
998 sprintf (rrp_instance
->status
[i
],
999 "Incrementing problem counter for seqid %d iface %s to [%d of %d]",
1000 active_instance
->last_token_seq
,
1001 totemnet_iface_print (rrp_instance
->net_handles
[i
]),
1002 active_instance
->counter_problems
[i
],
1003 rrp_instance
->totem_config
->rrp_problem_count_threshold
);
1005 rrp_instance
->totemrrp_log_level_warning
,
1006 rrp_instance
->status
[i
]);
1009 for (i
= 0; i
< rrp_instance
->interface_count
; i
++) {
1010 if (active_instance
->counter_problems
[i
] >= rrp_instance
->totem_config
->rrp_problem_count_threshold
)
1012 active_instance
->faulty
[i
] = 1;
1013 sprintf (rrp_instance
->status
[i
],
1014 "Marking seqid %d ringid %u interface %s FAULTY - adminisrtative intervention required.",
1015 active_instance
->last_token_seq
,
1017 totemnet_iface_print (rrp_instance
->net_handles
[i
]));
1019 rrp_instance
->totemrrp_log_level_error
,
1020 rrp_instance
->status
[i
]);
1021 active_timer_problem_decrementer_cancel (active_instance
);
1025 rrp_instance
->totemrrp_deliver_fn (
1026 active_instance
->totemrrp_context
,
1027 active_instance
->token
,
1028 active_instance
->token_len
);
1031 static void active_timer_expired_token_start (
1032 struct active_instance
*active_instance
)
1035 active_instance
->rrp_instance
->poll_handle
,
1036 active_instance
->rrp_instance
->totem_config
->rrp_token_expired_timeout
,
1037 (void *)active_instance
,
1038 timer_function_active_token_expired
,
1039 &active_instance
->timer_expired_token
);
1042 static void active_timer_expired_token_cancel (
1043 struct active_instance
*active_instance
)
1046 active_instance
->rrp_instance
->poll_handle
,
1047 active_instance
->timer_expired_token
);
1050 static void active_timer_problem_decrementer_start (
1051 struct active_instance
*active_instance
)
1054 active_instance
->rrp_instance
->poll_handle
,
1055 active_instance
->rrp_instance
->totem_config
->rrp_problem_count_timeout
,
1056 (void *)active_instance
,
1057 timer_function_active_problem_decrementer
,
1058 &active_instance
->timer_problem_decrementer
);
1061 static void active_timer_problem_decrementer_cancel (
1062 struct active_instance
*active_instance
)
1065 active_instance
->rrp_instance
->poll_handle
,
1066 active_instance
->timer_problem_decrementer
);
1071 * active replication
1073 static void active_mcast_recv (
1074 struct totemrrp_instance
*instance
,
1075 unsigned int iface_no
,
1078 unsigned int msg_len
)
1080 instance
->totemrrp_deliver_fn (
1086 static void active_mcast_flush_send (
1087 struct totemrrp_instance
*instance
,
1088 struct iovec
*iovec
,
1089 unsigned int iov_len
)
1092 struct active_instance
*rrp_algo_instance
= (struct active_instance
*)instance
->rrp_algo_instance
;
1094 for (i
= 0; i
< instance
->interface_count
; i
++) {
1095 if (rrp_algo_instance
->faulty
[i
] == 0) {
1096 totemnet_mcast_flush_send (instance
->net_handles
[i
], iovec
, iov_len
);
1101 static void active_mcast_noflush_send (
1102 struct totemrrp_instance
*instance
,
1103 struct iovec
*iovec
,
1104 unsigned int iov_len
)
1107 struct active_instance
*rrp_algo_instance
= (struct active_instance
*)instance
->rrp_algo_instance
;
1109 for (i
= 0; i
< instance
->interface_count
; i
++) {
1110 if (rrp_algo_instance
->faulty
[i
] == 0) {
1111 totemnet_mcast_noflush_send (instance
->net_handles
[i
], iovec
, iov_len
);
1116 static void active_token_recv (
1117 struct totemrrp_instance
*instance
,
1118 unsigned int iface_no
,
1121 unsigned int msg_len
,
1122 unsigned int token_seq
)
1125 struct active_instance
*active_instance
= (struct active_instance
*)instance
->rrp_algo_instance
;
1127 active_instance
->totemrrp_context
= context
; // this should be in totemrrp_instance ?
1128 if (token_seq
> active_instance
->last_token_seq
) {
1129 memcpy (active_instance
->token
, msg
, msg_len
);
1130 active_instance
->token_len
= msg_len
;
1131 for (i
= 0; i
< instance
->interface_count
; i
++) {
1132 active_instance
->last_token_recv
[i
] = 0;
1135 active_instance
->last_token_recv
[iface_no
] = 1;
1136 active_timer_expired_token_start (active_instance
);
1139 active_instance
->last_token_seq
= token_seq
;
1141 if (token_seq
== active_instance
->last_token_seq
) {
1142 active_instance
->last_token_recv
[iface_no
] = 1;
1143 for (i
= 0; i
< instance
->interface_count
; i
++) {
1144 if ((active_instance
->last_token_recv
[i
] == 0) &&
1145 active_instance
->faulty
[i
] == 0) {
1146 return; /* don't deliver token */
1149 active_timer_expired_token_cancel (active_instance
);
1151 instance
->totemrrp_deliver_fn (
1158 static void active_token_send (
1159 struct totemrrp_instance
*instance
,
1160 struct iovec
*iovec
,
1161 unsigned int iov_len
)
1163 struct active_instance
*rrp_algo_instance
= (struct active_instance
*)instance
->rrp_algo_instance
;
1166 for (i
= 0; i
< instance
->interface_count
; i
++) {
1167 if (rrp_algo_instance
->faulty
[i
] == 0) {
1168 totemnet_token_send (
1169 instance
->net_handles
[i
],
1176 static void active_recv_flush (struct totemrrp_instance
*instance
)
1178 struct active_instance
*rrp_algo_instance
= (struct active_instance
*)instance
->rrp_algo_instance
;
1181 for (i
= 0; i
< instance
->interface_count
; i
++) {
1182 if (rrp_algo_instance
->faulty
[i
] == 0) {
1184 totemnet_recv_flush (instance
->net_handles
[i
]);
1189 static void active_send_flush (struct totemrrp_instance
*instance
)
1191 struct active_instance
*rrp_algo_instance
= (struct active_instance
*)instance
->rrp_algo_instance
;
1194 for (i
= 0; i
< instance
->interface_count
; i
++) {
1195 if (rrp_algo_instance
->faulty
[i
] == 0) {
1197 totemnet_send_flush (instance
->net_handles
[i
]);
1202 static void active_iface_check (struct totemrrp_instance
*instance
)
1204 struct active_instance
*rrp_algo_instance
= (struct active_instance
*)instance
->rrp_algo_instance
;
1207 for (i
= 0; i
< instance
->interface_count
; i
++) {
1208 if (rrp_algo_instance
->faulty
[i
] == 0) {
1210 totemnet_iface_check (instance
->net_handles
[i
]);
1215 static void active_processor_count_set (
1216 struct totemrrp_instance
*instance
,
1217 unsigned int processor_count
)
1219 struct active_instance
*rrp_algo_instance
= (struct active_instance
*)instance
->rrp_algo_instance
;
1222 for (i
= 0; i
< instance
->interface_count
; i
++) {
1223 if (rrp_algo_instance
->faulty
[i
] == 0) {
1225 totemnet_processor_count_set (instance
->net_handles
[i
],
1231 static void active_token_target_set (
1232 struct totemrrp_instance
*instance
,
1233 struct totem_ip_address
*token_target
,
1234 unsigned int iface_no
)
1236 totemnet_token_target_set (instance
->net_handles
[iface_no
], token_target
);
1239 static void active_ring_reenable (
1240 struct totemrrp_instance
*instance
)
1242 struct active_instance
*rrp_algo_instance
= (struct active_instance
*)instance
->rrp_algo_instance
;
1244 memset (rrp_algo_instance
->last_token_recv
, 0, sizeof (unsigned int) *
1245 instance
->interface_count
);
1246 memset (rrp_algo_instance
->faulty
, 0, sizeof (unsigned int) *
1247 instance
->interface_count
);
1248 memset (rrp_algo_instance
->counter_problems
, 0, sizeof (unsigned int) *
1249 instance
->interface_count
);
1252 struct deliver_fn_context
{
1253 struct totemrrp_instance
*instance
;
1258 static void totemrrp_instance_initialize (struct totemrrp_instance
*instance
)
1260 memset (instance
, 0, sizeof (struct totemrrp_instance
));
1263 static int totemrrp_algorithm_set (
1264 struct totem_config
*totem_config
,
1265 struct totemrrp_instance
*instance
)
1267 unsigned int res
= -1;
1270 for (i
= 0; i
< RRP_ALGOS_COUNT
; i
++) {
1271 if (strcmp (totem_config
->rrp_mode
, rrp_algos
[i
]->name
) == 0) {
1272 instance
->rrp_algo
= rrp_algos
[i
];
1273 if (rrp_algos
[i
]->initialize
) {
1274 instance
->rrp_algo_instance
= rrp_algos
[i
]->initialize (
1276 totem_config
->interface_count
);
1282 for (i
= 0; i
< totem_config
->interface_count
; i
++) {
1283 instance
->status
[i
] = malloc (1024);
1284 sprintf (instance
->status
[i
], "ring %d active with no faults", i
);
1289 void rrp_deliver_fn (
1294 unsigned int token_seqid
;
1295 unsigned int token_is
;
1297 struct deliver_fn_context
*deliver_fn_context
= (struct deliver_fn_context
*)context
;
1299 deliver_fn_context
->instance
->totemrrp_token_seqid_get (
1306 * Deliver to the token receiver for this rrp algorithm
1308 deliver_fn_context
->instance
->rrp_algo
->token_recv (
1309 deliver_fn_context
->instance
,
1310 deliver_fn_context
->iface_no
,
1311 deliver_fn_context
->context
,
1317 * Deliver to the mcast receiver for this rrp algorithm
1319 deliver_fn_context
->instance
->rrp_algo
->mcast_recv (
1320 deliver_fn_context
->instance
,
1321 deliver_fn_context
->iface_no
,
1322 deliver_fn_context
->context
,
1328 void rrp_iface_change_fn (
1330 struct totem_ip_address
*iface_addr
)
1332 struct deliver_fn_context
*deliver_fn_context
= (struct deliver_fn_context
*)context
;
1334 deliver_fn_context
->instance
->totemrrp_iface_change_fn (
1335 deliver_fn_context
->context
,
1337 deliver_fn_context
->iface_no
);
1340 int totemrrp_finalize (
1341 totemrrp_handle handle
)
1343 struct totemrrp_instance
*instance
;
1347 res
= hdb_handle_get (&totemrrp_instance_database
, handle
,
1354 for (i
= 0; i
< instance
->interface_count
; i
++) {
1355 totemnet_finalize (instance
->net_handles
[i
]);
1358 hdb_handle_put (&totemrrp_instance_database
, handle
);
1365 * Totem Redundant Ring interface
1366 * depends on poll abstraction, POSIX, IPV4
1370 * Create an instance
1372 int totemrrp_initialize (
1373 poll_handle poll_handle
,
1374 totemrrp_handle
*handle
,
1375 struct totem_config
*totem_config
,
1378 void (*deliver_fn
) (
1383 void (*iface_change_fn
) (
1385 struct totem_ip_address
*iface_addr
,
1386 unsigned int iface_no
),
1388 void (*token_seqid_get
) (
1390 unsigned int *seqid
,
1391 unsigned int *token_is
),
1393 unsigned int (*msgs_missing
) (void))
1395 struct totemrrp_instance
*instance
;
1399 res
= hdb_handle_create (&totemrrp_instance_database
,
1400 sizeof (struct totemrrp_instance
), handle
);
1404 res
= hdb_handle_get (&totemrrp_instance_database
, *handle
,
1410 totemrrp_instance_initialize (instance
);
1412 instance
->totem_config
= totem_config
;
1414 res
= totemrrp_algorithm_set (
1415 instance
->totem_config
,
1424 instance
->totemrrp_log_level_security
= totem_config
->totem_logging_configuration
.log_level_security
;
1425 instance
->totemrrp_log_level_error
= totem_config
->totem_logging_configuration
.log_level_error
;
1426 instance
->totemrrp_log_level_warning
= totem_config
->totem_logging_configuration
.log_level_warning
;
1427 instance
->totemrrp_log_level_notice
= totem_config
->totem_logging_configuration
.log_level_notice
;
1428 instance
->totemrrp_log_level_debug
= totem_config
->totem_logging_configuration
.log_level_debug
;
1429 instance
->totemrrp_log_printf
= totem_config
->totem_logging_configuration
.log_printf
;
1431 instance
->interfaces
= totem_config
->interfaces
;
1433 instance
->totemrrp_poll_handle
= poll_handle
;
1435 instance
->totemrrp_deliver_fn
= deliver_fn
;
1437 instance
->totemrrp_iface_change_fn
= iface_change_fn
;
1439 instance
->totemrrp_token_seqid_get
= token_seqid_get
;
1441 instance
->totemrrp_msgs_missing
= msgs_missing
;
1443 instance
->interface_count
= totem_config
->interface_count
;
1445 instance
->net_handles
= malloc (sizeof (totemnet_handle
) * totem_config
->interface_count
);
1447 instance
->context
= context
;
1449 instance
->poll_handle
= poll_handle
;
1451 for (i
= 0; i
< totem_config
->interface_count
; i
++) {
1452 struct deliver_fn_context
*deliver_fn_context
;
1454 deliver_fn_context
= malloc (sizeof (struct deliver_fn_context
));
1455 assert (deliver_fn_context
);
1456 deliver_fn_context
->instance
= instance
;
1457 deliver_fn_context
->context
= context
;
1458 deliver_fn_context
->iface_no
= i
;
1460 totemnet_initialize (
1462 &instance
->net_handles
[i
],
1465 (void *)deliver_fn_context
,
1467 rrp_iface_change_fn
);
1470 totemnet_net_mtu_adjust (totem_config
);
1473 hdb_handle_put (&totemrrp_instance_database
, *handle
);
1477 hdb_handle_put (&totemrrp_instance_database
, *handle
);
1479 hdb_handle_destroy (&totemrrp_instance_database
, *handle
);
1483 int totemrrp_processor_count_set (
1484 totemrrp_handle handle
,
1485 unsigned int processor_count
)
1487 struct totemrrp_instance
*instance
;
1490 res
= hdb_handle_get (&totemrrp_instance_database
, handle
,
1497 instance
->rrp_algo
->processor_count_set (instance
, processor_count
);
1499 instance
->processor_count
= processor_count
;
1501 hdb_handle_put (&totemrrp_instance_database
, handle
);
1507 int totemrrp_token_target_set (
1508 totemrrp_handle handle
,
1509 struct totem_ip_address
*addr
,
1510 unsigned int iface_no
)
1512 struct totemrrp_instance
*instance
;
1515 res
= hdb_handle_get (&totemrrp_instance_database
, handle
,
1522 instance
->rrp_algo
->token_target_set (instance
, addr
, iface_no
);
1524 hdb_handle_put (&totemrrp_instance_database
, handle
);
1529 int totemrrp_recv_flush (totemrrp_handle handle
)
1531 struct totemrrp_instance
*instance
;
1534 res
= hdb_handle_get (&totemrrp_instance_database
, handle
,
1541 instance
->rrp_algo
->recv_flush (instance
);
1543 hdb_handle_put (&totemrrp_instance_database
, handle
);
1549 int totemrrp_send_flush (totemrrp_handle handle
)
1551 struct totemrrp_instance
*instance
;
1554 res
= hdb_handle_get (&totemrrp_instance_database
, handle
,
1561 instance
->rrp_algo
->send_flush (instance
);
1563 hdb_handle_put (&totemrrp_instance_database
, handle
);
1569 int totemrrp_token_send (
1570 totemrrp_handle handle
,
1571 struct iovec
*iovec
,
1572 unsigned int iov_len
)
1574 struct totemrrp_instance
*instance
;
1577 res
= hdb_handle_get (&totemrrp_instance_database
, handle
,
1584 instance
->rrp_algo
->token_send (instance
, iovec
, iov_len
);
1586 hdb_handle_put (&totemrrp_instance_database
, handle
);
1592 int totemrrp_mcast_flush_send (
1593 totemrrp_handle handle
,
1594 struct iovec
*iovec
,
1595 unsigned int iov_len
)
1597 struct totemrrp_instance
*instance
;
1600 res
= hdb_handle_get (&totemrrp_instance_database
, handle
,
1607 // TODO this needs to return the result
1608 instance
->rrp_algo
->mcast_flush_send (instance
, iovec
, iov_len
);
1610 hdb_handle_put (&totemrrp_instance_database
, handle
);
1615 int totemrrp_mcast_noflush_send (
1616 totemrrp_handle handle
,
1617 struct iovec
*iovec
,
1618 unsigned int iov_len
)
1620 struct totemrrp_instance
*instance
;
1623 res
= hdb_handle_get (&totemrrp_instance_database
, handle
,
1631 * merge detects go out through mcast_flush_send so it is safe to
1632 * flush these messages if we are only one processor. This avoids
1633 * an encryption/hmac and decryption/hmac
1635 if (instance
->processor_count
> 1) {
1637 // TODO this needs to return the result
1638 instance
->rrp_algo
->mcast_noflush_send (instance
, iovec
, iov_len
);
1641 hdb_handle_put (&totemrrp_instance_database
, handle
);
1646 int totemrrp_iface_check (totemrrp_handle handle
)
1648 struct totemrrp_instance
*instance
;
1651 res
= hdb_handle_get (&totemrrp_instance_database
, handle
,
1658 instance
->rrp_algo
->iface_check (instance
);
1660 hdb_handle_put (&totemrrp_instance_database
, handle
);
1665 int totemrrp_ifaces_get (
1666 totemrrp_handle handle
,
1668 unsigned int *iface_count
)
1670 struct totemrrp_instance
*instance
;
1673 res
= hdb_handle_get (&totemrrp_instance_database
, handle
,
1680 *status
= instance
->status
;
1683 *iface_count
= instance
->interface_count
;
1686 hdb_handle_put (&totemrrp_instance_database
, handle
);
1692 int totemrrp_ring_reenable (
1693 totemrrp_handle handle
)
1695 struct totemrrp_instance
*instance
;
1699 printf ("totemrrp ring reenable\n");
1700 res
= hdb_handle_get (&totemrrp_instance_database
, handle
,
1707 instance
->rrp_algo
->ring_reenable (instance
);
1709 for (i
= 0; i
< instance
->interface_count
; i
++) {
1710 sprintf (instance
->status
[i
], "ring %d active with no faults", i
);
1713 hdb_handle_put (&totemrrp_instance_database
, handle
);