Fix various bugs; see ChangeLog

[clumanager.git] / src / daemons / clusvcmgrd.c

/*
  Copyright Red Hat, Inc. 2002-2003

  This program is free software; you can redistribute it and/or modify it
  under the terms of the GNU General Public License as published by the
  Free Software Foundation; either version 2, or (at your option) any
  later version.

  This program is distributed in the hope that it will be useful, but
  WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  General Public License for more details.

  You should have received a copy of the GNU General Public License
  along with this program; see the file COPYING.  If not, write to the
  Free Software Foundation, Inc.,  675 Mass Ave, Cambridge, 
  MA 02139, USA.
*/
/** @file
 * Service Manager for RHCM.  This is the 1.0.x service manager with
 * extras to make it multi-node capable.
 *
 * Author: Brian Stevens (bstevens at redhat.com)
 *         Lon Hohberger (lhh at redhat.com)
 *
 */

/*static const char *version __attribute__ ((unused)) = "$Revision: 1.71 $";*/

#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <getopt.h>
#include <signal.h>
#include <string.h>
#include <errno.h>
#include <sys/types.h>
#include <sys/param.h>
#include <sys/wait.h>
#include <sys/syslog.h>
#include <sys/time.h>
#include <linux/reboot.h>
#include <sys/reboot.h>
#include <sched.h>
#include <clusterdefs.h>
#include <xmlwrap.h>
#include <clu_lock.h>
#include <msgsvc.h>
#include <svcmgr.h>
#include <clulog.h>
#include <quorum.h>
#include <clushared.h>
#include <sharedstate.h>
#include <namespace.h>
#include <findproc.h>

#ifdef TESTING
#ifdef reboot
#undef reboot
#endif
#define reboot(arg) {\
        clulog(LOG_EMERG, "reboot(%s) @ %s:%d\n", #arg, __FILE__, __LINE__); \
        raise(SIGSTOP); \
}
#endif

#define svc_fail(x) \
do { \
        clulog(LOG_DEBUG, "Service %d failed @ %s:%d\n",\
               x, __FILE__, __LINE__); \
        _svc_fail(x); \
} while(0)

#define HEARTBEAT_INTERVAL      60
#define CHECK_INTERVAL          5
#define MSG_TIMEOUT             10

#define SVCF_START_DISABLED             1
#define SVCF_PENDING                    2
#define SVCF_RESTART                    4
#define SVCF_CLEAR_FAILURES             8
#define SVCF_RESTARTFAILED              16

static int myNodeID;
static int myNodeState;
static int services_locked = 0;
#ifdef OLD_CLU_ALIAS
static int alias_owner = -1;
#endif
static char *myNodeName = NULL;
static int ticks[MAX_SERVICES];
static memb_mask_t membership, mask_panic;
static int sighup_received = 0, sigterm_received = 0;

struct child_sm {
        pid_t cs_pid;
        int cs_rq;
};

static struct child_sm svc_children[MAX_SERVICES];

/*
 * from clusvcmgrd_cfg.c:
 */
int check_config_file(void);
int check_config_data(void);
int rebuild_config_lockless(void);
int rebuild_config(void);
int handle_config_update(memb_mask_t mask, int my_node_id);
int boot_config_init(void);


/*
 * Service action string table
 */
static char *serviceActionStrings[] = {
        SVC_NONE_STR,
        SVC_ADD_STR,
        SVC_REMOVE_STR,
        SVC_START_STR,
        SVC_STOP_STR,
        SVC_CHECK_STR,
        SVC_DISABLE_STR,
        SVC_RELOCATE_STR,
        SVC_STATUS_INQUIRY_STR,
        SVC_FAILBACK_STR,
        SVC_START_PENDING_STR,
        SVC_START_RELOCATE_STR,
        "reconfigure", /* XXX */
        SVC_RESTART_STR
};

extern void daemon_init(char *);

static int svc_stop(int, int);
static int svc_stop_unclean(int);
static int _svc_fail(int svcID);

#ifdef OLD_CLU_ALIAS
static int clu_alias(int);
#endif

static int request_failback(int);
static int failback(int);

static int relocate_service(int svcID, int request, int target);
static void handle_svc_request(int, int, int, msg_handle_t);

int svc_report_failure(int svcID);
int setServiceStatus(ServiceBlock *svcblk);
int getServiceStatus(int svcNum, ServiceBlock *svcblk);
int removeService(int svcNum);


/**
 * Block the given signal.
 *
 * @param sig           Signal to block.
 * @return              See man sigprocmask.
 */
static int
block_signal(int sig)
{
        sigset_t set;

        sigemptyset(&set);
        sigaddset(&set, sig);
        
        return(sigprocmask(SIG_BLOCK, &set, NULL));
}


/**
 * unblock the given signal.
 *
 * @param sig           Signal to unblock.
 * @return              See man sigprocmask.
 */
static int
unblock_signal(int sig)
{
        sigset_t set;

        sigemptyset(&set);
        sigaddset(&set, sig);
        
        return(sigprocmask(SIG_UNBLOCK, &set, NULL));
}


static void
set_facility(void)
{
        char *p;
        if (CFG_Get((char *) "cluster%logfacility", NULL, &p) == CFG_OK) {
                if (p)
                        clu_set_facility(p);
        }
}


/**
 * Send a SVC_FAILBACK request to the given partner member.
 *
 * @param partner       Partner we are sending request to.
 * @return              FAIL or SUCCESS
 * @see failback
 */
int
request_failback(int partner)
{
        msg_handle_t fd_failback;
        SmMessageSt msg_failback;

        if (partner == myNodeID)
                return SUCCESS;

        /*
         * Fork here to avoid deadlock.
         */
        switch(fork()) {
        case 0:
                break;
        case -1:
                return FAIL;
        default:
                return SUCCESS;
        }

        msg_failback.sm_hdr.gh_magic = GENERIC_HDR_MAGIC;
        msg_failback.sm_hdr.gh_command = SVC_ACTION_REQUEST;
        msg_failback.sm_hdr.gh_length = sizeof (SmMessageSt);
        msg_failback.sm_data.d_action = SVC_FAILBACK;
        msg_failback.sm_data.d_svcOwner = myNodeID;
        msg_failback.sm_data.d_ret = 0;

        if ((fd_failback = msg_open(PROCID_CLUSVCMGRD, partner)) < 0) {
                clulog(LOG_DEBUG, "Failed opening connection to svcmgrd\n");
                exit(1);
        }

        /* Encode */
        swab_SmMessageSt(&msg_failback);

        if (msg_send(fd_failback, &msg_failback, sizeof (SmMessageSt)) !=
            sizeof (SmMessageSt)) {
                msg_close(fd_failback);
                clulog(LOG_ERR, "Error sending failback request.\n");
                exit(1);
        }
        msg_close(fd_failback);
        exit(0);
}


/**
 * Handle SVC_FAILBACK from a given node.  This shuts down services which
 * should be running on 'target' instead of 'myNodeID'.  Takes into account
 * service failover domain and preferred node ordering.  Services without
 * a failover domain will never be sent to the requesting node.
 *
 * @param target        Requestor which sent us SVC_FAILBACK
 * @return              SUCCESS
 * @see request_failback
 */
int
failback(int target)
{
        int svcID;

        for (svcID = 0; svcID < MAX_SERVICES; svcID++) {
                if (serviceExists(svcID) != YES)
                        continue;

                if (!svc_has_domain(svcID))
                        continue;

                /*
                 * If the service has a failover domain, and the requestor
                 * should run it and I shouldn't, then I will give the
                 * service back.
                 * 
                 * This relies on handle_svc_request to determine the
                 * state of the service.
                 */
                if (node_should_start(myNodeID, membership, svcID) <
                    node_should_start(target, membership, svcID))
                        handle_svc_request(svcID, SVC_RELOCATE, target, -1);
        }

        return SUCCESS;
}


/**
 * See if a child process operating on a specified service has exited.
 *
 * @param svcID         Service ID's child we're checking out.
 * @return              0 indicates that no child has exited.  1 indicates
 *                      that the child for the service has, indeed, been
 *                      cleaned up.
 */
int
cleanup_child(int svcID)
{
        /* Obvious check: is there even a child for this service? */
        if (!svc_children[svcID].cs_pid)
                return 1;
        if (waitpid(svc_children[svcID].cs_pid, NULL, WNOHANG) != -1)
                return 0;
        if (errno != ECHILD)
                return 0;

        svc_children[svcID].cs_pid = 0;
        return 1;
}


/**
 * Clean up children.  This is our SIGCHLD handler.
 */
void
reap_zombies(int __attribute__ ((unused)) sig)
{
        int svcID;
        int pid;
        //int nchildren = 0;
        int status;

        while ((pid = waitpid(-1, &status, WNOHANG)) != 0) {
                if (pid < 0) {
                        if (errno == EINTR)
                                continue;
                        break;          /* No children */
                }

                /*clulog(LOG_DEBUG, "waitpid reaped %d\n", pid);*/
                for (svcID = 0; svcID < MAX_SERVICES; svcID++) {
                        if (pid == svc_children[svcID].cs_pid) {
                                svc_children[svcID].cs_pid = 0;
                                //nchildren++;
                        }
                }
        }
        //return (nchildren);
}


/**
 * Clean up services and exit.
 *
 * @param status        Return value passed up to parent process.
 * @param clean         This is set to '1' when we're cleanly shutting down
 *                      and we have quorum.  Without quorum or during an
 *                      unclean shutdown, this is 0.
 * @return              If it returns, that's BAD
 */
static void
svcmgr_exit(int status, int clean)
{
        int svcID, fd;

#ifdef OLD_CLU_ALIAS
        clu_alias(0);
#endif

        for (svcID = 0; svcID < MAX_SERVICES; svcID++) {

                if (serviceExists(svcID) != YES)
                        continue;

                /* Wait for child process acting on this service to exit */
                while (!cleanup_child(svcID))
                        sleep(5);

                if (clean) {
                        switch(svc_stop(svcID, 0)) {
                        case FAIL:
                                svc_fail(svcID);
                                svc_report_failure(svcID);
                                break;
                        case ABORT:
                                /* Lock failure during shutdown == switch to
                                   unclean mode */
                                clulog(LOG_ERR, "Failed to acquire cluster lock "
                                       "during shutdown\n");
                                  
                                clean = 0;
                                break;
                        case SUCCESS:
                        default:
                                continue;
                        }
                }

                /* Succeed || Die */
                svc_stop_unclean(svcID);
        }

        /*
         * Tell the quorum daemon that we are leaving
         */
        clulog(LOG_DEBUG, "Sending message to quorumd that we are exiting\n");

        if ((fd = msg_open(PROCID_CLUQUORUMD, myNodeID)) < 0) {
                clulog(LOG_ERR, "msg_open failed to quorum daemon\n");
                exit(status);
        }

        if (msg_send_simple(fd, QUORUM_EXIT, status, 0) == -1) {
                clulog(LOG_ERR, "Failed sending exit message to cluquorumd\n");
                msg_close(fd);
                exit(status);
        }

        msg_close(fd);
        clulog(LOG_INFO, "Exiting\n");
        exit(status);
}


/**
 * NOTE: If someone kills the service manager during start, it's possible to
 * have a service script still running the stop phase.  This is OKAY!
 * This is our SIGTERM handler.
 *
 * @see svcmgr_exit
 */
static void
sigterm_handler(void)
{
        block_signal(SIGHUP);
        block_signal(SIGTERM);
        sigterm_received = 1;
}


/**
 * Retrieve our log level from the cluster database and set it accordingly.
 */
static void
set_loglevel(void)
{
        int level;

        if (getSvcMgrLogLevel(&level) == FAIL) {
                clulog(LOG_ERR,
                       "Failed getting log level for from config database\n");
                return;
        }

        if (clu_set_loglevel(level) == -1) {
                clulog(LOG_ERR, "Failed setting log level\n");
        }
}


/**
 * Noitify the local daemons that the on-disk configuration has changed, and
 * so needs to be reread.
 */
static void
notify_everybody(void)
{
        /*
         * Notify local daemons of the cofiguration update...
         */
        killall("clumembd", SIGHUP);
        killall("cluquorumd", SIGHUP);
        killall("clulockd", SIGHUP);
        killall("clurmtabd", SIGHUP);
}


/**
 * Handle an updated configuration.  This is called after we receive a SIGHUP.
 *
 * @see sighup_handler
 */
void
update_config(void)
{
        int really_updated = 1;

        block_signal(SIGHUP);
        /* XXX check for return code?? */
        /* We reload the msg service stuff inside handle_config_update */
        really_updated = handle_config_update(membership, myNodeID);

        if (really_updated == 0) {
                set_loglevel();
                notify_everybody();
        } 

        /*
         *  If we fail to update, the other service managers will reread
         * the shared config in a few seconds anyway.
         */
        unblock_signal(SIGHUP);
}


/**
 * When we receive SIGHUP, we set the global flag.  We soon after call
 * update_config.
 *
 * @see update_config
 */
void
sighup_handler(int __attribute__ ((unused)) sig)
{
        sighup_received = 1;
}


/**
 * Run the service script for a given service.  The service scripts handle
 * the real meat of starting/stopping services.
 *
 * @param action        The action to take (ie, start/stop/status)
 * @param svcID         The service ID we intend to take 'action' on.
 * @param block         Set to 0 if the service script should run in the
 *                      background, 1 if we should wait for it to complete
 *                      before continuing.
 * @param ret           The return code of the service script.
 * @return              SUCCESS or FAIL.
 */
static int
exec_service_script(char *action, int svcID, int block, int *ret)
{
        int pid;
        char svcIDstr[8];
        char *svcName;
        int local_ret = 0;
        sigset_t set;
        struct sched_param param;

        getSvcName(svcID, &svcName);

        clulog(LOG_DEBUG, "Exec of script %s, action %s, service %s\n",
               SVC_ACTION_SCRIPT, action, svcName);

        pid = fork();
        if (pid < 0) {
                clulog(LOG_ERR, "fork failed: %s", strerror(errno));
                return FAIL;
        }
        if (pid) {

                if (ret)
                        *ret = 0;

                if (block) {
                        do {
                                pid = waitpid(pid, &local_ret, 0);
                                if (pid < 0) {
                                        if (errno == EINTR)
                                                continue;
                                        clulog(LOG_DEBUG,
                                               "waitpid: %s",
                                               strerror(errno));
                                        /* Fake it. */
                                        local_ret = 0;
                                }
                        } while (0);

                        clulog(LOG_DEBUG,
                               "Exec of script for service %s returned %d\n",
                               svcName, local_ret);
                        if (ret)
                                *ret = local_ret;
                }
                return SUCCESS;
        }

        /**
         * we need to set the sched_priority back to normal in case clusvcmgrd
         * is running in a different prio b/c cluquorumd%rtp is set
         */
        if (sched_getscheduler(0) != SCHED_OTHER) {
                memset(&param,0,sizeof(param));
                param.sched_priority = 0;
                if (sched_setscheduler(0, SCHED_OTHER, (void *)&param) != 0)
                        clulog(LOG_WARNING, "Setting child to normal priority "
                               "failed: %s\n", strerror(errno));
                else
                        clulog(LOG_DEBUG, "Using normal priority\n");
        }

        /* lhh - Unblock signals so the user script doesn't break */
        sigfillset(&set);
        if (sigprocmask(SIG_UNBLOCK, &set, NULL) != 0) {
                clulog(LOG_WARNING, "Failed to unblock signals: %s\n",
                       strerror(errno));
        }

        snprintf(svcIDstr, sizeof (svcIDstr), "%d", svcID);
        local_ret =
            execl(SVC_ACTION_SCRIPT, SVC_ACTION_SCRIPT, action, svcIDstr, NULL);

        clulog(LOG_DEBUG, "Exec failed of %s, action %s, service %s, err %s\n",
               SVC_ACTION_SCRIPT, action, svcName, strerror(errno));

        exit(local_ret);
}


#ifdef OLD_CLU_ALIAS
static int
clu_alias(int req)
{
        int pid;
        int local_ret;

        if (req) {
                if (alias_owner == myNodeID)
                        return SUCCESS;
                alias_owner = myNodeID;
                clulog(LOG_DEBUG, "Start cluster alias request\n");
        } else {
                if (alias_owner != myNodeID)
                        return SUCCESS;
                alias_owner = -1;
                clulog(LOG_DEBUG, "Stop cluster alias request\n");
        }

        pid = fork();
        if (pid < 0) {
                clulog(LOG_ERR, "fork failed: %s", strerror(errno));
                return FAIL;
        }
        if (pid) {
                do {
                        pid = waitpid(pid, &local_ret, 0);
                        if ((pid < 0) && (errno == EINTR))
                                continue;
                } while (0);

                clulog(LOG_DEBUG, "Exec of alias script returned %d\n",
                       local_ret);
                return local_ret;
        }

        block_signal(SIGTERM);
        block_signal(SIGHUP);

        if (req)
                local_ret =
                    execl(CLU_ALIAS_SCRIPT, CLU_ALIAS_SCRIPT, "start", NULL);
        else
                local_ret =
                    execl(CLU_ALIAS_SCRIPT, CLU_ALIAS_SCRIPT, "stop", NULL);

        clulog(LOG_DEBUG, "Exec failed of %s, err %s\n", CLU_ALIAS_SCRIPT,
               strerror(errno));

        exit(local_ret);
}
#endif


/**
 * Initialize an on-disk service block.
 *
 * @param svcID         Service ID whose block we need to update.
 * @return              FAIL or SUCCESS.
 */
int
svc_add(int svcID)
{
        ServiceBlock svcStatus;

        clulog(LOG_DEBUG, "Initializing service #%d\n", svcID);

        /*
         * Make sure the service does not exist
         */

        if (clu_svc_lock(svcID) == -1) {
                clulog(LOG_ERR, "Unable to obtain cluster lock: %s\n",
                       strerror(errno));
                return FAIL;
        }

        if (getServiceStatus(svcID, &svcStatus) == SUCCESS) {
                clulog(LOG_ERR,
                       "Service #%d already exists!\n",
                       svcID);
                clu_svc_unlock(svcID);
                return FAIL;
        }

        svcStatus.sb_id = svcID;
        svcStatus.sb_owner = NODE_ID_NONE;
        svcStatus.sb_last_owner = NODE_ID_NONE;
        svcStatus.sb_state = SVC_DISABLED;
        svcStatus.sb_transition = (uint64_t)time(NULL);
        svcStatus.sb_restarts = 0;

        if (setServiceStatus(&svcStatus) != SUCCESS) {
                (void) removeService(svcID);
                clu_svc_unlock(svcID);
                return FAIL;
        }

        clu_svc_unlock(svcID);
        return SUCCESS;
}


/**
 * Set an on-disk service block's state to UNINITIALIZED.
 *
 * @param svcID         Service ID whose block we need to update.
 * @return              FAIL or SUCCESS.
 */
int
svc_remove(int svcID)
{
        clulog(LOG_DEBUG, "Removing service #%d from database\n", svcID);

        if (clu_svc_lock(svcID) == -1) {
                clulog(LOG_ERR, "Unable to obtain cluster lock: %s\n",
                       strerror(errno));
                return FAIL;
        }

        if (removeService(svcID) != 0) {
                clulog(LOG_ERR, "Failed removing service %d from database\n",
                       svcID);
                clu_svc_unlock(svcID);
                return FAIL;
        }

        clu_svc_unlock(svcID);
        return SUCCESS;
}


/**
 * Advise service manager as to whether or not to start a service, given
 * that we already know it's legal to run the service.
 *
 * @param svcStatus     Current service status.
 * @param svcName       Service name
 * @param flags         Specify whether or not it's legal to start a 
 *                      disabled service, etc.
 * @return              0 = DO NOT start service, return FAIL
 *                      1 = START service - return whatever it returns.
 *                      2 = DO NOT start service, return SUCCESS
 */
int
svc_advise_start(ServiceBlock *svcStatus, char *svcName, int flags)
{
        char *nodeName;
        
        switch(svcStatus->sb_state) {
        case SVC_FAILED:
                clulog(LOG_ERR, "Service %s has failed on all applicable "
                       "members; can not start.\n", svcName);
                return 0;
                
        case SVC_STARTED:
        case SVC_STOPPING:
                getNodeName(svcStatus->sb_owner, &nodeName);
                if ((svcStatus->sb_owner == myNodeID) || 
                    (memb_online(membership, svcStatus->sb_owner)==1) ||
                    (memb_online(mask_panic, svcStatus->sb_owner)==1)) {
                        /*
                         * Service is running and the owner is online!
                         */
                        clulog(LOG_DEBUG,
                               "Service is running on member %s.\n",
                               nodeName);
                        return 2;
                }

                /*
                 * Service is running but owner is down -> FAILOVER
                 */
                clulog(LOG_NOTICE,
                       "Taking over service %s from down member %s\n",
                       svcName, nodeName);
                return 1;

        case SVC_PENDING:
                /*
                 * Starting failed service...
                 */
                if (flags & SVCF_PENDING) {
                        clulog(LOG_NOTICE, "Starting failed service %s\n",
                               svcName);
                        svcStatus->sb_state = SVC_STOPPED;
                        /* Start! */
                        return 1;
                }

                /* Don't start, but return success. */
                clulog(LOG_DEBUG,
                       "Not starting %s: pending/transitional state\n",
                       svcName);
                return 2;

        case SVC_STOPPED:
                clulog(LOG_NOTICE, "Starting stopped service %s\n", svcName);
                return 1;
        
        case SVC_DISABLED:
        case SVC_UNINITIALIZED:
                if (flags & SVCF_START_DISABLED) {
                        clulog(LOG_NOTICE, "Starting disabled service %s\n",
                               svcName);
                        return 1;
                }

                clulog(LOG_DEBUG, "Not starting disabled service %s\n",
                       svcName);
                return 0;

        default:
                clulog(LOG_ERR,
                       "Cannot start service %s: Invalid State %d\n",
                       svcName, svcStatus->sb_state);
                return 0;
        }
}


/**
 * Start a cluster service.
 *
 * @param svcID         Service ID to start.
 * @param flags         Service-operation specific flags to take into account.
 * @see svc_advise_start
 * @return              FAIL, SUCCESS
 */
static int
svc_start(int svcID, int flags)
{
        char *svcName;
        ServiceBlock svcStatus;
        int ret;

        getSvcName(svcID, &svcName);
        clulog(LOG_DEBUG, "Handling start request for service %s\n", svcName);

        if (clu_svc_lock(svcID) == -1) {
                clulog(LOG_ERR, "Unable to obtain cluster lock: %s\n",
                       strerror(errno));
                return FAIL;
        }

        if (getServiceStatus(svcID, &svcStatus) != SUCCESS) {
                clu_svc_unlock(svcID);
                clulog(LOG_ERR, "Failed getting status for service %s\n",
                       svcName);
                return FAIL;
        }

        /* LOCK HELD */
        switch (svc_advise_start(&svcStatus, svcName, flags)) {
        case 0: /* Don't start service, return FAIL */
                clu_svc_unlock(svcID);
                return FAIL;
        case 1: /* Start service. */
                break;
        case 2: /* Don't start service, return SUCCESS */
                clu_svc_unlock(svcID);
                return SUCCESS;
        
        default:
                break;
        }

        /* LOCK HELD if we get here */
#if 0
        if (flags & SVCF_CLEAR_FAILURES) 
                memset(svcStatus.sb_failed_mask, 0, sizeof(memb_mask_t));
#endif

        svcStatus.sb_owner = myNodeID;
        svcStatus.sb_state = SVC_STARTED;
        svcStatus.sb_transition = (uint64_t)time(NULL);
        svcStatus.sb_checks = (uint16_t)0;

        if (flags & (SVCF_START_DISABLED|SVCF_PENDING))
                svcStatus.sb_false_starts = (uint16_t)0;

        if (flags & SVCF_RESTARTFAILED)
                svcStatus.sb_restarts++;
        else
                svcStatus.sb_restarts = 0;

        if (setServiceStatus(&svcStatus) != SUCCESS) {
                clulog(LOG_ERR, "Failed changing service status\n");
                clu_svc_unlock(svcID);
                return FAIL;
        }
        
        clu_svc_unlock(svcID);

        if ((exec_service_script(SVC_START_STR, svcID, 1, &ret) != SUCCESS) ||
            (ret)) {
                return FAIL;
        }

        return SUCCESS;
}


static int
flip_state(char *svcName, int svcID, int state, int last_owner_flip)
{
        ServiceBlock svcStatus;

        if (clu_svc_lock(svcID) == -1) {
                clulog(LOG_ERR, "Unable to obtain cluster lock: %s\n",
                       strerror(errno));
                return ABORT;
        }

        if (getServiceStatus(svcID, &svcStatus) != SUCCESS) {
                clu_svc_unlock(svcID);
                clulog(LOG_ERR, "Failed getting status for service %s\n",
                       svcName);
                return FAIL;
        }

        if (last_owner_flip) {
                svcStatus.sb_last_owner = svcStatus.sb_owner;
                svcStatus.sb_owner = NODE_ID_NONE;
        }
        svcStatus.sb_state = state;
        svcStatus.sb_transition = (uint64_t)time(NULL);
        if (setServiceStatus(&svcStatus) != SUCCESS) {
                clu_svc_unlock(svcID);
                clulog(LOG_ERR, "Failed changing service status\n");
                return FAIL;
        }
        clu_svc_unlock(svcID);
        return SUCCESS;
}



/**
 * Stop a cluster service.
 *
 * @param svcID         Service ID to stop.
 * @param flags         Service-operation specific flags to take into account.
 * @see svc_advise_start
 * @return              FAIL, SUCCESS
 */
static int
svc_stop(int svcID, int flags)
{
        ServiceBlock svcStatus;
        char *svcName;
        int ret;

        getSvcName(svcID, &svcName);
        clulog(LOG_DEBUG, "Handling stop request for service %s\n", svcName);

        if (clu_svc_lock(svcID) == -1) {
                clulog(LOG_ERR, "Unable to obtain cluster lock: %s\n",
                       strerror(errno));
                return ABORT;
        }

        if (getServiceStatus(svcID, &svcStatus) != SUCCESS) {
                clu_svc_unlock(svcID);
                clulog(LOG_ERR, "Failed getting status for service %s\n",
                       svcName);
                return FAIL;
        }

        if (((svcStatus.sb_state != SVC_STARTED) ||
             (svcStatus.sb_owner != myNodeID))
            && (svcStatus.sb_state != SVC_PENDING)) {
                clu_svc_unlock(svcID);
                clulog(LOG_DEBUG, "Unable to stop service %s in %s state\n",
                       svcName, serviceStateStrings[svcStatus.sb_state]);
                return SUCCESS;
        }

        svcStatus.sb_state = SVC_STOPPING;
        svcStatus.sb_transition = (uint64_t)time(NULL);
        if (setServiceStatus(&svcStatus) != SUCCESS) {
                clu_svc_unlock(svcID);
                clulog(LOG_ERR, "Failed changing service status\n");
                return FAIL;
        }
        clu_svc_unlock(svcID);

        if ((exec_service_script(SVC_STOP_STR, svcID, 1, &ret) != SUCCESS) ||
            (ret)) {
                return FAIL;
        }

        if (flags & SVCF_PENDING)
                ret = SVC_PENDING;
        else
                ret = SVC_STOPPED;

        flip_state(svcName, svcID, ret, 1);

        return SUCCESS;
}


/**
 * Stop a cluster service - without updating the on-disk-block.
 *
 * @param svcID         Service ID to stop.
 * @return              FAIL, SUCCESS
 */
static int
svc_stop_unclean(int svcID)
{
        int ret;
        char *svcName;

        /* 
         * Infanticide.
         */
        if (svc_children[svcID].cs_pid) {
                kill(svc_children[svcID].cs_pid, SIGKILL);

                do {
                        if ((waitpid(svc_children[svcID].cs_pid, NULL, 0)==-1)
                            && (errno == EINTR))
                                continue;
                } while (0);
        }

        getSvcName(svcID, &svcName);
        clulog(LOG_WARNING, "Forcing stop of service %s\n", svcName);

        if ((exec_service_script(SVC_STOP_STR, svcID, 1, &ret) != SUCCESS) ||
            (ret)) {
                clulog(LOG_EMERG,
                       "Failed to stop service %s uncleanly - REBOOTING\n",
                       svcName);
                sleep(1);
                REBOOT(RB_AUTOBOOT);
        }
        return SUCCESS;
}


/**
 * Disable a cluster service.  Services in the disabled state are never
 * automatically started by the service manager - one must send a SVC_START
 * message.
 *
 * @param svcID         Service ID to stop.
 * @return              FAIL, SUCCESS
 */
static int
svc_disable(int svcID)
{
        ServiceBlock svcStatus;
        char *svcName;
        int ret;

        if (clu_svc_lock(svcID) == -1) {
                clulog(LOG_ERR, "Unable to obtain cluster lock: %s\n",
                       strerror(errno));
                return FAIL;
        }

        getSvcName(svcID, &svcName);
        clulog(LOG_DEBUG, "Handling disable request for service %s\n", svcName);

        if (getServiceStatus(svcID, &svcStatus) != SUCCESS) {
                clu_svc_unlock(svcID);
                clulog(LOG_ERR, "Failed getting status for service %s\n",
                       svcName);
                return FAIL;
        }

        if (svcStatus.sb_state == SVC_DISABLED) {
                clu_svc_unlock(svcID);
                clulog(LOG_DEBUG, "Service %s already disabled\n", svcName);
                return SUCCESS;
        }

        if (((svcStatus.sb_state == SVC_STOPPING) && 
             (svcStatus.sb_owner != myNodeID)) &&
            (memb_online(membership, svcStatus.sb_owner)==1)) {
                clulog(LOG_WARNING,
                       "Service %s is in stop-transition on node %d"
                       ", cannot disable\n",
                       svcName);
                return SUCCESS;
        }

        if (((svcStatus.sb_state == SVC_STARTED) &&
             (svcStatus.sb_owner != myNodeID))
            || ((svcStatus.sb_state != SVC_STARTED)
                && (svcStatus.sb_state != SVC_STOPPING)
                && (svcStatus.sb_state != SVC_STOPPED)
                && (svcStatus.sb_state != SVC_PENDING)
                && (svcStatus.sb_state != SVC_FAILED))) {
                clu_svc_unlock(svcID);
                clulog(LOG_DEBUG, "Unable to disable service %s in %s state\n",
                       svcName, serviceStateStrings[svcStatus.sb_state]);
                return FAIL;
        }

        svcStatus.sb_state = SVC_STOPPING;
        svcStatus.sb_transition = (uint64_t)time(NULL);
        if (setServiceStatus(&svcStatus) != SUCCESS) {
                clu_svc_unlock(svcID);
                clulog(LOG_ERR, "Failed changing service status\n");
                return FAIL;
        }
        clu_svc_unlock(svcID);

        if ((exec_service_script(SVC_STOP_STR, svcID, 1, &ret) != SUCCESS) ||
            (ret)) {
                return FAIL;
        }

        flip_state(svcName, svcID, SVC_DISABLED, 1);

        return SUCCESS;
}


/**
 * Mark a cluster service as failed.  User intervention required.
 *
 * @param svcID         Service ID to stop.
 * @return              FAIL, SUCCESS
 */
static int
_svc_fail(int svcID)
{
        ServiceBlock svcStatus;
        char *svcName;

        if (clu_svc_lock(svcID) == -1) {
                clulog(LOG_ERR, "Unable to obtain cluster lock: %s\n",
                       strerror(errno));
                return FAIL;
        }

        getSvcName(svcID, &svcName);
        clulog(LOG_DEBUG, "Handling failure request for service %s\n", svcName);

        if (getServiceStatus(svcID, &svcStatus) != SUCCESS) {
                clu_svc_unlock(svcID);
                clulog(LOG_ERR, "Failed getting status for service %s\n",
                       svcName);
                return FAIL;
        }

        if ((svcStatus.sb_state == SVC_STARTED) &&
            (svcStatus.sb_owner != myNodeID)) {
                clu_svc_unlock(svcID);
                clulog(LOG_DEBUG, "Unable to disable service %s in %s state\n",
                       svcName, serviceStateStrings[svcStatus.sb_state]);
                return FAIL;
        }

        /*
         * Leave a bread crumb so we can debug the problem with the service!
         */
        if (svcStatus.sb_owner != NODE_ID_NONE) {
                svcStatus.sb_last_owner = svcStatus.sb_owner;
                svcStatus.sb_owner = NODE_ID_NONE;
        }
        svcStatus.sb_state = SVC_FAILED;
        svcStatus.sb_transition = (uint64_t)time(NULL);
        svcStatus.sb_restarts = 0;
        if (setServiceStatus(&svcStatus) != SUCCESS) {
                clu_svc_unlock(svcID);
                clulog(LOG_ERR, "Failed changing service status\n");
                return FAIL;
        }
        clu_svc_unlock(svcID);

        return SUCCESS;
}


/**
 * Check the status of a given service.  This execs the service script
 * with the argument 'status', and evaluates the return code.
 *
 * @param svcID         Service ID to check.
 * @return              FAIL or SUCCESS.
 */
static int
svc_check(int svcID)
{
        ServiceBlock svcStatus;
        char *svcName, *maxrestarts, *maxfs;
        int script_ret, ret;

        getSvcName(svcID, &svcName);
        clulog(LOG_DEBUG, "Handling check request for service %s\n", svcName);

        if (clu_svc_lock(svcID) == -1) {
                clulog(LOG_ERR, "Unable to obtain cluster lock: %s\n",
                       strerror(errno));
                return FAIL;
        }

        if (getServiceStatus(svcID, &svcStatus) != SUCCESS) {
                clu_svc_unlock(svcID);
                clulog(LOG_ERR, "Failed getting status for service %s\n",
                       svcName);
                return FAIL;
        }

        if ((svcStatus.sb_state != SVC_STARTED) ||
            (svcStatus.sb_owner != myNodeID)) {
                clu_svc_unlock(svcID);
                clulog(LOG_DEBUG, "Unable to check service %s in %s state\n",
                       svcName, serviceStateStrings[svcStatus.sb_state]);
                return FAIL;
        }
        clu_svc_unlock(svcID);

        if ((exec_service_script(SVC_CHECK_STR, svcID, 1, &ret) != SUCCESS) ||
            (ret))
                script_ret = FAIL;
        else 
                script_ret = SUCCESS;

        clu_svc_lock(svcID);
        if (getServiceStatus(svcID, &svcStatus) != SUCCESS) {
                clu_svc_unlock(svcID);
                clulog(LOG_ERR, "Failed getting status for service %s\n",
                       svcName);
                return FAIL;
        }

        if (script_ret == FAIL) {
                ret = FAIL;

                if (svcStatus.sb_checks == 0 &&
                    (getSvcMaxFalseStarts(svcID, &maxfs) == SUCCESS) &&
                    atoi(maxfs) > 0) {

                        /* If we've exceeded false-start count, relocate */
                        svcStatus.sb_false_starts++;
                        clulog(LOG_WARNING,
                               "Service %s false-start detected (%d/%d)\n",
                               svcName, svcStatus.sb_false_starts, atoi(maxfs));

                        if (svcStatus.sb_false_starts > atoi(maxfs)) {
                                clulog(LOG_ERR, "Max false starts for service %s"
                                       " exceeded.  Relocating\n", svcName);
                                ret = ABORT;
                        }

                        /* Update on-disk with new false start info */
                        setServiceStatus(&svcStatus);
                }

                if (getSvcMaxRestarts(svcID, &maxrestarts) == SUCCESS) {
                        if (atoi(maxrestarts) > 0) {
                                /* We're about to restart.  If we would exceed 
                                   our restart count, relocate. */
                                if (svcStatus.sb_restarts >=
                                    atoi(maxrestarts)) {
                                        clulog(LOG_ERR, "Max restarts for "
                                               "service %s exceeded.  "
                                               "Relocating\n", svcName);
                                        ret = ABORT;
                                }
                        } else if (atoi(maxrestarts) < 0) {
                                clulog(LOG_ERR, "Service %s failed.  "
                                       "Relocating\n", svcName);
                                ret = ABORT;
                        }
                }
        } else { /* SUCCESS */
                ret = SUCCESS;
                if (!svcStatus.sb_checks) {
                        svcStatus.sb_checks = 1;
                        svcStatus.sb_false_starts = 0;
                        setServiceStatus(&svcStatus);
                }
        }

        clu_svc_unlock(svcID);

        return ret;
}


static int
init_services(void)
{
        int svcID;
        ServiceBlock svcStatus;
        char *svcName;

        clulog(LOG_INFO, "Initializing services\n");

        for (svcID = 0; svcID < MAX_SERVICES; svcID++) {
        
                /* This takes a long time... Abort quickly if necessary */
                if (sigterm_received)
                        svcmgr_exit(0, 1);

                if (serviceExists(svcID) != YES)
                        continue;

                getSvcName(svcID, &svcName);

                /*
                 * If service is not on the shared service information disk,
                 * or it is running and owned by this node, reinitialized it.
                 */

                if (clu_svc_lock(svcID) == 0) {
                        if ((getServiceStatus(svcID, &svcStatus) != SUCCESS) ||
                            ((svcStatus.sb_owner == myNodeID) &&
                             ((svcStatus.sb_state == SVC_STARTED) ||
                              (svcStatus.sb_state == SVC_STOPPING))) ||
                            ((svcStatus.sb_owner == NODE_ID_NONE) &&
                             (svcStatus.sb_state == SVC_PENDING))) {
                                svcStatus.sb_id = svcID;
                                svcStatus.sb_last_owner = svcStatus.sb_owner;
                                svcStatus.sb_owner = NODE_ID_NONE;
                                svcStatus.sb_state = SVC_STOPPED;
                                svcStatus.sb_transition = (uint64_t)time(NULL);
                                svcStatus.sb_restarts = 0;
                                if (setServiceStatus(&svcStatus) != SUCCESS) {
                                        clulog(LOG_ERR, "Failed setting "
                                               "service status for %s\n",
                                               svcName);
                                }
                        }
                        clu_svc_unlock(svcID);
                } else {
                        clulog(LOG_WARNING,
                               "Unable to obtain lock for service %s: %s\n",
                               svcName,
                               strerror(errno));
                }

                /*
                 * We stop all services to clean up any state in the case
                 * that this system came down without gracefully stopping
                 * services.
                 */
                if (exec_service_script(SVC_STOP_STR, svcID, 1, NULL) !=
                    SUCCESS) {
                        clulog(LOG_ALERT,
                               "Failed stopping service %s during init\n",
                               svcName);
                        continue;
                }

        }

        return SUCCESS;
}


/*
 * Send a message to the target node to start the service.
 */
static int
relocate_service(int svcID, int request, int target)
{
        SmMessageSt msg_relo;
        int fd_relo, msg_ret;

        /* Build the message header */
        msg_relo.sm_hdr.gh_magic = GENERIC_HDR_MAGIC;
        msg_relo.sm_hdr.gh_command = SVC_ACTION_REQUEST;
        msg_relo.sm_hdr.gh_length = sizeof (SmMessageSt);
        msg_relo.sm_data.d_action = request;
        msg_relo.sm_data.d_svcID = svcID;
        msg_relo.sm_data.d_ret = 0;

        /* Open a connection to the other node */

        if ((fd_relo = msg_open(PROCID_CLUSVCMGRD, target)) < 0) {
                clulog(LOG_ERR, "Failed opening connection to member #%d\n",
                       target);
                return FAIL;
        }

        /* Encode */
        swab_SmMessageSt(&msg_relo);

        /* Send relocate message to the other node */
        if (msg_send(fd_relo, &msg_relo, sizeof (SmMessageSt)) !=
            sizeof (SmMessageSt)) {
                clulog(LOG_ERR,
                       "Error sending relocate request to member #%d\n",
                       target);
                msg_close(fd_relo);
                return FAIL;
        }

        clulog(LOG_DEBUG, "Sent relocate request.\n");

        /* Check the response */
        msg_ret = msg_receive(fd_relo, &msg_relo, sizeof (SmMessageSt));

        if (msg_ret != sizeof (SmMessageSt)) {
                /* 
                 * In this case, we don't restart the service, because the 
                 * service state is actually unknown to us at this time.
                 */
                clulog(LOG_ERR, "Mangled reply from member #%d during service "
                       "relocate\n", target);
                msg_close(fd_relo);
                return SUCCESS; /* XXX really UNKNOWN */
        }

        /* Got a valid response from other node. */
        msg_close(fd_relo);

        /* Decode */
        swab_SmMessageSt(&msg_relo);

        return msg_relo.sm_data.d_ret;
}


/**
 * Advise whether or not we should drop a particular request for a given
 * service.
 *
 * @param svcID         Service ID in question.
 * @param req           Particular request in question.
 * @param svcStatus     Current service status block.
 * @return              1 for TRUE (drop service request), 0 for FALSE (do not
 *                      drop given request)
 */
int
svc_advise_drop_request(int svcID, int req, ServiceBlock * svcStatus)
{
        /* 
         * Drop the request if it's not a DISABLE and not a START_PENDING 
         * if the service is in the PENDING state (ie, it failed on one node)
         */
        if ((svcStatus->sb_state == SVC_PENDING) &&
            ((req != SVC_START_PENDING) && (req != SVC_DISABLE))) {
                clulog(LOG_DEBUG,
                       "Dropping op %d for svc%d: Service Pending Start\n",
                       req, svcID);
                return 1;
        }
        
        /*
         * Drop the request if it's an SVC_CHECK and the service isn't started.
         */
        if ((req == SVC_CHECK) &&
            ((svcStatus->sb_state != SVC_STARTED) ||
             (svcStatus->sb_owner != myNodeID))) {
                clulog(LOG_DEBUG, "Dropping SVC_CHECK for svc%d: Service "
                       "not running locally\n", svcID);
                return 1;
        }

        /*
         * Drop the request if it's an SVC_CHECK and we're already doing
         * something to that service so that other requests may continue.
         */
        if ((req == SVC_CHECK) && svc_children[svcID].cs_pid) {
                clulog(LOG_DEBUG,
                       "Dropping SVC_CHECK for svc%d: PID%d has not completed",
                       svcID, svc_children[svcID].cs_pid);
                return 1;
        }

        /*
         * Drop the request if it's an SVC_START, we are the owner, and
         * the service is currently stopping
         */
        if ((req == SVC_START) && svc_children[svcID].cs_pid) {
                clulog(LOG_DEBUG,
                       "Dropping SVC_START for svc%d: PID%d has not completed",
                       svcID, svc_children[svcID].cs_pid);
                return 1;
        }

        return 0;
}


/**
 * Determine the target node we should relocate the service to if we are
 * not given one from cluadmin.  This checks the failover domain to see
 * the next node online in a given failover group.  
 *
 * @param rmask         The nodes allowed to be checked for when we are
 *                      trying to determine who should start the service.
 * @param current_owner The current owner of the service, or the node
 *                      who is requesting the information.  This is the
 *                      _last_ member allowed to run the service.
 * @param svcID         ID of the service in question.
 */
int
best_target_node(memb_mask_t rmask, int current_owner, int svcID)
{
        int x;

        x = current_owner + 1;
        if (x >= MAX_NODES)
                x = 0;

        do {
                if (node_should_start(x, rmask, svcID) == FOD_BEST) {
                        return x;
                }

                x++;
                if (x >= MAX_NODES)
                        x = 0;
        } while (x != current_owner);

        return current_owner;
}


#if 0
/**
 * clear_failure_mask(int svcID)
 *
 * @param svcID
 * @see mark_self_failed
 */
int
clear_failure_mask(int svcID)
{
        ServiceBlock svcStatus;

        if (clu_svc_lock(svcID) == -1) {
                clulog(LOG_ERR, "Couldn't obtain lock for service %d: %s\n",
                       svcID, strerror(errno));
                return FAIL;
        }

        if (getServiceStatus(svcID, &svcStatus) != SUCCESS) {
                clulog(LOG_ERR, "Couldn't obtain status for service %d\n",
                       svcID);
                clu_svc_unlock(svcID);
                return FAIL;
        }

        memset(svcStatus.sb_failed_mask, 0, sizeof(svcStatus.sb_failed_mask));
        if (setServiceStatus(&svcStatus) != SUCCESS) {
                clulog(LOG_ERR, "Couldn't set FAILURE status for service %d\n",
                       svcID);
                return FAIL;
        }

        clu_svc_unlock(svcID);
        return SUCCESS;
}

/**
 * Marks our bit in the failed_nodes bitmask in the service block on disk.
 * This is a signal to other members to _not_ send us the service again.
 * This mask is cleared when a service is successfully started.
 *
 * @param svcID
 * @see clear_failure_mask
 */
int
mark_self_failed(int svcID)
{
        ServiceBlock svcStatus;

        if (clu_svc_lock(svcID) == -1) {
                clulog(LOG_ERR, "Couldn't obtain lock for service %d: %s\n",
                       svcID, strerror(errno));
                return FAIL;
        }

        if (getServiceStatus(svcID, &svcStatus) != SUCCESS) {
                clulog(LOG_ERR, "Couldn't obtain status for service %d\n",
                       svcID);
                clu_svc_unlock(svcID);
                return FAIL;
        }

        /* Mark ourselves as FAILED for this service */
        memb_mark_up(svcStatus.sb_failed_mask, myNodeID);

        if (setServiceStatus(&svcStatus) != SUCCESS) {
                clulog(LOG_ERR, "Couldn't set FAILURE status for service %d\n",
                       svcID);
                return FAIL;
        }

        clu_svc_unlock(svcID);

        return SUCCESS;
}
#endif


int
svc_report_failure(int svcID)
{
        ServiceBlock svcStatus;
        char *svcName;
        char *nodeName;

        getSvcName(svcID, &svcName);

        if (clu_svc_lock(svcID) == -1) {
                clulog(LOG_ERR, "Couldn't obtain lock for service %s: %s\n",
                       svcName, strerror(errno));
                return FAIL;
        }

        if (getServiceStatus(svcID, &svcStatus) != SUCCESS) {
                clulog(LOG_ERR, "Couldn't obtain status for service %s\n",
                       svcName);
                clu_svc_unlock(svcID);
                return FAIL;
        }
        clu_svc_unlock(svcID);

        getNodeName(svcStatus.sb_last_owner, &nodeName);
        
        clulog(LOG_ALERT,
               "Service %s returned failure code.  Last Owner: %s\n",
               svcName, nodeName);
        clulog(LOG_ALERT,
               "Administrator intervention required.\n",
               svcName, nodeName);

        return SUCCESS;
}

/**
 * handle_relocate_req - Relocate a service.  This seems like a huge
 * deal, except it really isn't.
 *
 * @param svcID         Service ID in question.
 * @param flags         If (flags & SVCF_PENDING), we were called from
 *                      handle_start_req - and so we should ignore all local
 *                      restarts/stops - since handle_start_req does this
 *                      for us.
 * @param preferred_target      When sent a relocate message from the
 *                              management software, a destination node
 *                              is sent as well.  This causes us to try
 *                              starting the service on that node *first*,
 *                              but does NOT GUARANTEE that the service
 *                              will end up on that node.  It will end up
 *                              on whatever node actually successfully
 *                              starts it.
 * @param new_owner     Member who actually ends up owning the service.
 */
int
handle_relocate_req(int svcID, int flags, int preferred_target,
                    uint32_t *new_owner)
{
        memb_mask_t allowed_nodes;
        int target = preferred_target;
        int request;
        char *nodeName=NULL, *svcName=NULL;
        
        getSvcName(svcID, &svcName);
        request = (flags & SVCF_PENDING) ? SVC_START_PENDING :
                SVC_START_RELOCATE;

        /*
         * Stop the service - if we haven't already done so.
         */
        if (!(flags & SVCF_PENDING)) {
                if (svc_stop(svcID, flags) != SUCCESS) {
                        if (svc_start(svcID, flags) != SUCCESS)
                                svc_fail(svcID);
                        return FAIL;
                }
        }

        /*
         * First, see if it's legal to relocate to the target node.  Legal
         * means: the node is online and is in the [restricted] failover
         * domain of the service, or the service has no failover domain.
         */
        if (preferred_target >= 0 && preferred_target <= MAX_NODES) {

                memset(allowed_nodes, 0, sizeof(allowed_nodes));
                memb_mark_up(allowed_nodes, preferred_target);
                target = best_target_node(allowed_nodes, myNodeID, svcID);

                /*
                 * I am the ONLY one capable of running this service,
                 * PERIOD...
                 */
                if (target == myNodeID)
                        goto exhausted;

                if (target == preferred_target) {
                        /*
                         * It's legal to start the service on the given
                         * node.  Try to do so.
                         */
                        if (relocate_service(svcID, request, target) ==
                            SUCCESS) {
                                *new_owner = target;
                                /*
                                 * Great! We're done...
                                 */
                                return SUCCESS;
                        }
                }
        }

        /*
         * Ok, so, we failed to send it to the preferred target node.
         * Try to start it on all other nodes.
         */
        memcpy(allowed_nodes, membership, sizeof(memb_mask_t));
        memb_mark_down(allowed_nodes, myNodeID);

        /* Don't try to relocate to the preferred target more than once. */
        if (preferred_target >= 0 && preferred_target <= MAX_NODES)
                memb_mark_down(allowed_nodes, myNodeID);

        while (memb_count(allowed_nodes)) {
                target = best_target_node(allowed_nodes, myNodeID, svcID);
                if (target == myNodeID)
                        goto exhausted;

                switch (relocate_service(svcID, request, target)) {
                case FAIL:
                        memb_mark_down(allowed_nodes, target);
                        continue;
                case ABORT:
                        svc_report_failure(svcID);
                        return FAIL;
                case SUCCESS:
                        *new_owner = target;
                        getNodeName(target, &nodeName);
                        clulog(LOG_NOTICE,
                               "Service %s now running on member %s\n",
                               svcName, nodeName);
                        return SUCCESS;
                default:
                        clulog(LOG_ERR, "Invalid reply from member %d during"
                               " relocate operation!\n", target);
                }
        }

        /*
         * We got sent here from handle_start_req.
         * We're DONE.
         */
        if (flags & SVCF_PENDING)
                return FAIL;

        /*
         * All potential places for the service to start have been exhausted.
         * We're done.
         */
exhausted:
        clulog(LOG_WARNING, "Attempting to restart service %s locally.\n",
               svcName);
        if (svc_start(svcID, flags) == SUCCESS) {
                *new_owner = myNodeID;
                return FAIL;
        }
                
        if (svc_stop(svcID, 0) != SUCCESS) {
                svc_fail(svcID);
                svc_report_failure(svcID);
        }

        return FAIL;
}


/**
 * handle_start_req - Handle a generic start request from a user or during
 * service manager boot.
 *
 * @param svcID         Service ID to start.
 * @param flags
 * @param new_owner     Owner which actually started the service.
 * @return              FAIL - Failure.
 *                      SUCCESS - The service is running.
 */
int
handle_start_req(int svcID, int flags, uint32_t *new_owner)
{
        int ret, tolerance = FOD_BEST, target = -1;

        /*
         * When a service request is from a user application (eg, clusvcadm),
         * accept FOD_GOOD instead of FOD_BEST
         */
        if (flags & SVCF_START_DISABLED)
                tolerance = FOD_GOOD;
        
        if (!(flags & SVCF_RESTART) &&
            (node_should_start(myNodeID, membership, svcID) < tolerance)) {

                /* Try to send to someone else who might care about it */
                target = best_target_node(membership, myNodeID, svcID);
                ret = handle_relocate_req(svcID, SVCF_PENDING, target,
                                          new_owner);

                if (ret == FAIL)
                        svc_disable(svcID);
                return ret;
        }
        
        /*
         * Strip out all flags which are invalid.
         */
        clulog(LOG_DEBUG, "Starting service %d - flags 0x%08x\n", svcID,
               flags);

#if 0
        /*
         * This is a 'root' start request.  We need to clear out our failure
         * mask here - so that we can try all nodes if necessary.
         */
        flags |= SVCF_CLEAR_FAILURES;
#endif
        ret = svc_start(svcID, flags);

#if 0
        if (clear_failure_mask(svcID) != SUCCESS) {
                clulog(LOG_WARNING, "Could not clear failure bitmask for "
                       "service #%s!\n", svcName);
        }
#endif

        /*
         * If we succeeded, then we're done.
         */
        if (ret == SUCCESS) {
                *new_owner = myNodeID;
                return SUCCESS;
        }
        
        /* 
         * Keep the state open so the other nodes don't try to start
         * it.  This allows us to be the 'root' of a given service.
         */
        clulog(LOG_DEBUG, "Stopping failed service %d\n", svcID);
        if (svc_stop(svcID, SVCF_PENDING) != SUCCESS) {
                clulog(LOG_CRIT, "Service %d failed to stop cleanly", svcID);
                svc_fail(svcID);

                /*
                 * If we failed to stop the service, we're done.  At this
                 * point, we can't determine the service's status - so
                 * trying to start it on other nodes is right out.
                 */
                return ABORT;
        }
        
        /*
         * OK, it failed to start - but succeeded to stop.  Now,
         * we should relocate the service.
         */
        clulog(LOG_WARNING, "Relocating failed service %d\n", svcID);
        ret = handle_relocate_req(svcID, SVCF_PENDING, -1, new_owner);

        if (ret == FAIL)
                svc_disable(svcID);

        return ret;
}


/**
 * handle_start_remote_req - Handle a remote start request.
 *
 * @param svcID         Service ID to start.
 * @param flags         Flags to use to determine start behavior.
 * @return              FAIL - Local failure.  ABORT - Unrecoverable error:
 *                      the service didn't start, nor stop cleanly. SUCCESS
 *                      - We started the service.
 */
int
handle_start_remote_req(int svcID, int flags)
{
        memb_mask_t rmask;
        int tolerance = FOD_BEST;

        memset(rmask, 0, sizeof(rmask));
        memb_mark_up(rmask, myNodeID);

        if (flags & SVCF_START_DISABLED)
                tolerance = FOD_GOOD;

        /*
         * See if we agree with our ability to start the given service.
         */
        if (node_should_start(myNodeID, rmask, svcID) < tolerance)
                return FAIL;

        if (svc_start(svcID, flags) == SUCCESS)
                return SUCCESS;

#if 0
        if (mark_self_failed(svcID) == FAIL) {
                svc_fail(svcID);
                return ABORT;
        }
#endif

        if (svc_stop(svcID, 0) == SUCCESS)
                return FAIL;

        svc_fail(svcID);
        return ABORT;
}


/**
 * Handle a request regarding a service.
 *
 * @param svcID         ID of service in question.
 * @param action        Action to be performed on the service.
 * @param target        In the case of a relocate, target/destination node
 *                      we're relocating to.
 * @param fd            File descriptor on which we send our response.
 */
void
handle_svc_request(int svcID, int action, int target, msg_handle_t fd)
{
        char *svcName;
        SmMessageSt msg_sm;
        int ret = FAIL;
        ServiceBlock svcStatus;
        int flags = 0;
        uint32_t new_owner = NODE_ID_NONE;
        char child = 0;

        getSvcName(svcID, &svcName);
        clulog(LOG_DEBUG, "Service %s request %d\n", svcName, action);

        if (myNodeState != NODE_UP)
                goto out;

        /*
         * Don't assume the service exists...
         */
        if (serviceExists(svcID) != YES) {
                goto out;
        }

        if (clu_svc_lock(svcID) == -1) {
                clulog(LOG_ERR, "Unable to obtain cluster lock: %s\n",
                       strerror(errno));
                goto out;
        }

        if (getServiceStatus(svcID, &svcStatus) != SUCCESS) {
                clulog(LOG_ERR, "Cannot get status for service %d\n", svcID);
                clu_svc_unlock(svcID);
                goto out;       
        }

        clu_svc_unlock(svcID);

        /*
         * Check to see if we should drop the service request.  This
         * is based on the current servuce status, the action required,
         * etc.
         */
        if (svc_advise_drop_request(svcID, action, &svcStatus)) {
                ret = SUCCESS;
                goto out;
        }

        /*
         * Fork so that we can run service actions in parallel.
         */
        while (svc_children[svcID].cs_pid != 0) {

                if (svcStatus.sb_state == SVC_PENDING) {
                        /*
                         * Shouldn't get here, but if so, avoid deadlock.
                         */
                        clulog(LOG_ERR, "%s failed during "
                               "relocate-to-preferred member operation",
                               svcName);
                        ret = FAIL;
                        goto out;
                }

                clulog(LOG_DEBUG,
                       "Proc %d already running action on service %s\n",
                       svc_children[svcID].cs_pid, svcName);

                /* See if we missed the SIGCHLD */
                if (!cleanup_child(svcID))
                        sleep(5);
        }

        /* Record what the child will be doing */
        svc_children[svcID].cs_rq = action;

        if ((svc_children[svcID].cs_pid = fork())) {
                if (svc_children[svcID].cs_pid < 0) {
                        /*
                         * Fork failed.
                         */
                        clulog(LOG_DEBUG,
                               "Fork failed handling action request.\n");
                        svc_children[svcID].cs_pid = 0;
                        svc_children[svcID].cs_rq = 0;

                        /* Send reply, if applicable */
                        goto out;
                }

                clulog(LOG_DEBUG, "[M] Pid %d -> %s for service %s\n",
                       svc_children[svcID].cs_pid, serviceActionStrings[action],
                       svcName);

                return;
        }

        block_signal(SIGTERM);
        block_signal(SIGHUP);
        child = 1;
        clulog(LOG_DEBUG, "[C] Pid %d handling %s request for service %s\n",
               getpid(), serviceActionStrings[action], svcName);

        switch (action) {
        case SVC_START:
start_top:
                flags |= ((fd == -1) ? 0 : SVCF_START_DISABLED);
                
                ret = handle_start_req(svcID, flags, &new_owner);
                break;

        case SVC_START_PENDING:
                /*
                 * We allow starting of pending requests only if
                 * explicitly asked for from someone else - never on
                 * a local node event.
                 */
                flags = SVCF_PENDING;
        case SVC_START_RELOCATE:
                /*
                 * We use fd as an indicator to see whether or not we
                 * were called on behalf of a node event.  Generally,
                 * fd is set, but we usually don't handle relocation of
                 * disabled services -- it's kind of an anomaly.
                 */
                flags |= (fd == -1) ? 0 : SVCF_START_DISABLED;
                ret = handle_start_remote_req(svcID, flags);
                break;

        case SVC_STOP:
        case SVC_RESTART:
restart_top:
                if ((ret = svc_stop(svcID, 0)) == SUCCESS) {
                        /*
                         * Ok, we did the stop - now do the whole start
                         * process, including relocating in the case of
                         * failure.
                         */
                        if (action == SVC_RESTART) {
                                target = myNodeID;
                                flags = SVCF_RESTART;
                                goto start_top;
                        }
                        
                        break;
                }

                ret = FAIL;

                if (svc_start(svcID, 0) == SUCCESS)
                        break;

                svc_fail(svcID);
                break;

        case SVC_DISABLE:

                if ((ret = svc_disable(svcID)) == SUCCESS)
                        break;

                /*
                 * We don't run svc_fail here because svc_fail could
                 * put us back where we were.  Always allow disable.
                 */
                ret = FAIL;
                break;

        case SVC_CHECK:

                if ((ret = svc_check(svcID)) == SUCCESS)
                        break;

                if (ret == ABORT) {
                        /* Try to relocate service at this point */
                        ret = handle_relocate_req(svcID, 0, -1, &new_owner);
                        break;
                }

                ret = FAIL;

                clulog(LOG_WARNING, "Restarting locally failed service %s\n",
                       svcName);

                (void) svc_stop(svcID, 0);

                /*
                 * Try the whole start process, including relocating it in
                 * the case that it failed to restart locally.
                 */
                flags = SVCF_RESTART | SVCF_RESTARTFAILED;
                goto start_top;

        case SVC_RELOCATE:

                if (svcStatus.sb_state == SVC_DISABLED) {
                        clulog(LOG_DEBUG,
                               "Can not relocate disabled service %s\n",
                               svcName);
                        ret = FAIL;
                        break;
                }

                if (target == myNodeID)
                        goto restart_top;

                ret = handle_relocate_req(svcID, 0, target, &new_owner);
                break;

        default:
                clulog(LOG_ERR, "Invalid service request %d\n", action);
                ret = FAIL;
                break;
        }

        /*
         * If fd is valid, the request was on behalf of a client who is
         * blocking for the status reply.
         */
out:              
        if (fd != -1) {
                msg_sm.sm_data.d_svcOwner = new_owner;
                msg_sm.sm_data.d_ret = ret;

                /* Encode before responding... */
                swab_SmMessageSt(&msg_sm);

                if (msg_send(fd, &msg_sm, sizeof (SmMessageSt)) !=
                    sizeof (SmMessageSt)) {
                        clulog(LOG_ERR, "Error replying to action request.\n");
                }
        }

        if (child)
                exit(ret);      /* child exit */
}


/**
 * Check to see if we need to kill a child process - and do so if necessary.
 * We do not need to reset the cs_pid field.  This should only be called
 * during a remote node-down event to determine if we had a relocate-request
 * or other request out to that node.  If so, we need to kill the child
 * handling that request.
 *
 * @param svcID         Service ID
 * @param svc           Service block (status of svcID)
 */
void
consider_reapage(int svcID, ServiceBlock * svc)
{
        /*
         * Since PENDING is only a valid state when BOTH nodes are up, and
         * given that the remote node just died, mark the service as
         * 'stopped' if it was in the 'pending' state.  Kill the child
         * process if it exists.
         */
        if (svc->sb_state == SVC_PENDING) {
                if (svc_children[svcID].cs_pid)
                        kill(svc_children[svcID].cs_pid, SIGKILL);      

                clulog(LOG_DEBUG, "Marking %d (state %d) as stopped", svcID,
                       svc->sb_state);

                /* Mark state -> stopped */
                if (clu_svc_lock(svcID) == -1) {
                        clulog(LOG_ERR, "Unable to obtain cluster lock: %s\n",
                               strerror(errno));
                        return;
                }

                if (getServiceStatus(svcID, svc) == SUCCESS) {
                        svc->sb_last_owner = svc->sb_owner;
                        svc->sb_owner = NODE_ID_NONE;
                        svc->sb_state = SVC_STOPPED;
                        if (setServiceStatus(svc) != SUCCESS) {
                                clulog(LOG_ERR,
                                       "Failed marking service %d as stopped\n",
                                       svcID);
                        }
                }
                clu_svc_unlock(svcID);
                return;
        }

        if (!svc_children[svcID].cs_pid)
                return;

        /*
         * The child was SVC_START and the other node is marked as the owner.
         * This means we tried to start it locally, failed, and send a
         * REMOTE_START to the other node, but the other node died before we
         * received a response.
         *
         * Simplify: Kill child whenever our partner owns the service.
         */
        if (svc->sb_owner != myNodeID) {
                clulog(LOG_INFO,
                       "Killing child PID%d: Remote member went down!",
                       svc_children[svcID].cs_pid);
                kill(svc_children[svcID].cs_pid, SIGKILL);
                return;
        }

        /*
         * Our last case is an explicit relocate (eg, from cluadmin).  The other
         * node went down, and we received its node-down event.  This could have
         * been taken care of above, but we still need to catch the cases where
         * it hasn't been taken care of yet...
         */
        if (svc_children[svcID].cs_rq == SVC_RELOCATE) {
                clulog(LOG_INFO,
                       "Killing child PID%d: Remote member went down!",
                       svc_children[svcID].cs_pid);
                kill(svc_children[svcID].cs_pid, SIGKILL);
        }
}


/**
 * Rewrite a service block as 'stopped' if all members of its
 * restricted failover domain went offline.
 *
 * @param svcID         Service ID to stop.
 * @return              FAIL, SUCCESS
 */
static int
check_rdomain_crash(int svcID, ServiceBlock *svcStatus)
{
        char *svcName;
        memb_mask_t allowed_nodes;

        if (memb_online(membership, svcStatus->sb_owner) ||
            (svcStatus->sb_state == SVC_STOPPED))
                return SUCCESS;
        
        memcpy(allowed_nodes, membership, sizeof(memb_mask_t));
        memb_mark_down(allowed_nodes, svcStatus->sb_owner);
        if (best_target_node(allowed_nodes, svcStatus->sb_owner, svcID) !=
            svcStatus->sb_owner)
                return SUCCESS;

        if (clu_svc_lock(svcID) == -1) {
                clulog(LOG_ERR, "Unable to obtain cluster lock: %s\n",
                       strerror(errno));
                return FAIL;
        }

        if (getServiceStatus(svcID, svcStatus) != SUCCESS) {
                clu_svc_unlock(svcID);
                clulog(LOG_ERR, "Failed getting status for service %s\n",
                       svcName);
                return FAIL;
        }

        if ((svcStatus->sb_state != SVC_STARTED) ||
            (svcStatus->sb_owner == myNodeID) ||
            memb_online(membership, svcStatus->sb_owner)) {
                clu_svc_unlock(svcID);
                return SUCCESS;
        }

        svcStatus->sb_last_owner = svcStatus->sb_owner;
        svcStatus->sb_owner = NODE_ID_NONE;
        svcStatus->sb_state = SVC_STOPPED;
        svcStatus->sb_transition = (uint64_t)time(NULL);
        if (setServiceStatus(svcStatus) != SUCCESS) {
                clu_svc_unlock(svcID);
                clulog(LOG_ERR, "Failed changing service status\n");
                return FAIL;
        }
        clu_svc_unlock(svcID);
        return SUCCESS;
}


/**
 * Called to decide what services to start locally during a node_event.
 * Originally a part of node_event, it is now its own function to cut down
 * on the length of node_event.
 * 
 * @see                 node_event
 */
void
eval_services(int local, int nodeStatus)
{
        int svcID;
        char *svcName, *nodeName;
        ServiceBlock svcStatus;

        if (services_locked)
                return;

        for (svcID = 0; svcID < MAX_SERVICES; svcID++) {

                if (serviceExists(svcID) != YES)
                        continue;

                getSvcName(svcID, &svcName);

                /*
                 * Lock the service information and get the current service
                 * status.
                 */
                if (clu_svc_lock(svcID) == -1) {
                        clulog(LOG_ERR, "Unable to obtain cluster lock: %s\n",
                               strerror(errno));
                        return;
                }
                
                if (getServiceStatus(svcID, &svcStatus) != SUCCESS) {
                        clulog(LOG_ERR, "Cannot get status for service %s\n",
                               svcName);
                        clu_svc_unlock(svcID);
                        continue;
                }
                clu_svc_unlock(svcID);

                if (svcStatus.sb_owner == NODE_ID_NONE)
                        nodeName = "none";
                else
                        getNodeName(svcStatus.sb_owner, &nodeName);

                if ((svcStatus.sb_state == SVC_DISABLED) ||
                    (svcStatus.sb_state == SVC_FAILED))
                        continue;

                clulog(LOG_DEBUG, "Evaluating service %s, state %s, owner "
                       "%s\n", svcName,
                       serviceStateStrings[svcStatus.sb_state], nodeName);

                if (local && (nodeStatus == NODE_UP)) {

                        /*
                         * Start any stopped services, or started services
                         * that are owned by a down node.
                         */
                        if (node_should_start(myNodeID, membership, svcID) ==
                            FOD_BEST)
                                handle_svc_request(svcID, SVC_START, -1, -1);

                        continue;
                }

                if (!local && (nodeStatus == NODE_DOWN)) {

                        /*
                         * Start any stopped services, or started services
                         * that are owned by a down node.
                         */
                        consider_reapage(svcID, &svcStatus);
                        if (node_should_start(myNodeID, membership, svcID) ==
                            FOD_BEST)
                                handle_svc_request(svcID, SVC_START, -1, -1);
                        else
                                check_rdomain_crash(svcID, &svcStatus);
                        /*
                         * TODO
                         * Mark a service as 'stopped' if no members in its restricted
                         * fail-over domain are running.
                         */
                }
        }
}


/**
 * Called to handle the transition of a cluster member from up->down or
 * down->up.  This handles initializing services (in the local node-up case),
 * exiting due to loss of quorum (local node-down), and service fail-over
 * (remote node down).
 *
 * @param nodeID                ID of the member which has come up/gone down.
 * @param nodeStatus            New state of the member in question.
 * @see eval_services
 */
void
node_event(int nodeID, int nodeStatus)
{
        int local = 0;
        int partner;

        local = (nodeID == myNodeID);
        if (local) {
                if (nodeStatus == NODE_UP) {

                        if (myNodeState == NODE_UP)
                                return;

                        myNodeState = NODE_UP;

#ifdef OLD_CLU_ALIAS
                        clu_alias(0);
#endif

                        clulog(LOG_DEBUG,
                               "local member up, initializing services\n");

                        /*
                         * Initialize all services we own. We needed to wait
                         * for a NODE_UP event as we need the locking
                         * subsystem for this.
                         */

                        if (init_services() != SUCCESS) {
                                clulog(LOG_ERR, "Cannot initialize services\n");
                                svcmgr_exit(1, 0);
                        }
                }

                if (nodeStatus == NODE_DOWN) {
                        svcmgr_exit(0, 0);
                        /* NOT REACHED */
                }
        } else {

                /*
                 * Nothing to do for events from other nodes if we are not up.
                 */

                if (myNodeState != NODE_UP)
                        return;
        }

#ifdef OLD_CLU_ALIAS
        if (myNodeID == memb_high_node(membership)) {
                clu_alias(1);
        } else {
                clu_alias(0);
        }
#endif

        eval_services(local, nodeStatus);

        /* If we just came up, and our partner is up request a failback */
        if (local && (nodeStatus == NODE_UP)) {

                for (partner = 0; partner < MAX_NODES; partner++) {
                        if (partner == myNodeID)
                                continue;

                        if (memb_online(membership, partner)) {
                                if (request_failback(partner) != SUCCESS) {
                                        clulog(LOG_ERR,
                                               "Unable to inform partner "
                                               "to start failback\n");
                                }
                        }
                }
        }
}


/**
 * Run service status scripts on all services which (a) we are running and
 * (b) have check intervals set.
 *
 * @param elapsed               Number of elapsed seconds since last time
 *                              check_services was run.
 */
void
check_services(int elapsed)
{
        int svcID;
        char *svcName;
        ServiceBlock svcStatus;
        char *intervalStr;
        int interval;

        for (svcID = 0; svcID < MAX_SERVICES; svcID++) {

                if (serviceExists(svcID) != YES)
                        continue;

                getSvcName(svcID, &svcName);

                /* 
                 * Check service interval first, since it doesn't
                 * require a lock.
                 */
                if (getSvcCheckInterval(svcID, &intervalStr) == SUCCESS)
                        interval = atoi(intervalStr);
                else
                        interval = 0;

                if (!interval)
                        continue;

                /*
                 * Check service status
                 */
                if (clu_svc_lock(svcID) == -1) {
                        clulog(LOG_ERR, "Unable to obtain cluster lock: %s\n",
                               strerror(errno));
                        return;
                }

                if (getServiceStatus(svcID, &svcStatus) != SUCCESS) {
                        clu_svc_unlock(svcID);
                        clulog(LOG_ERR,
                               "Failed getting status for service %s\n",
                               svcName);
                        continue;
                }
                clu_svc_unlock(svcID);

                if ((svcStatus.sb_owner != myNodeID)
                    || (svcStatus.sb_state != SVC_STARTED))
                        continue;

                ticks[svcID] += elapsed;

                clulog(LOG_DEBUG,
                       "Check interval for service %s is %d, elapsed %d\n",
                       svcName, interval, ticks[svcID]);

                if (ticks[svcID] < interval) {
                        clulog(LOG_DEBUG, "Too early to check service %s\n",
                               svcName);
                        continue;
                }

                ticks[svcID] = 0;
                handle_svc_request(svcID, SVC_CHECK, -1, -1);
        }
}


/**
 * Handle a QUORUM or QUORUM_GAINED message from the quorum daemon.  This
 * updates our local membership view and handles whether or not we should
 * exit, as well as determines node transitions (thus, calling node_event()).
 *
 * @param msg_quorum            Cluster event from the quorum daemin.
 * @see                         node_event
 * @return                      0
 */
int
handle_quorum_msg(cm_event_t *msg_quorum)
{
        memb_mask_t     node_delta, old_membership;
        int             x;
        char            *nodeName;
        int             me = 0;

        memcpy(old_membership, membership, sizeof(memb_mask_t));
        memcpy(membership, cm_quorum_mask(msg_quorum), sizeof(memb_mask_t));

        lock_set_quorum_view(cm_quorum_view(msg_quorum));

        clulog(LOG_INFO, "Quorum Event: View #%d %s\n",
               (int)cm_quorum_view(msg_quorum),
               memb_mask_str(cm_quorum_mask(msg_quorum)));

        /*
         * Handle nodes lost.  Do our local node event first.
         */
        memb_mask_lost(node_delta, old_membership, membership);

        me = memb_online(node_delta, myNodeID);
        if (me) {
                /* Should not happen */
                clulog(LOG_INFO, "State change: LOCAL OFFLINE\n");
                node_event(myNodeID, NODE_DOWN);
                /* NOTREACHED */
        }

        for (x=0; x<MAX_NODES; x++) {
                if (x == myNodeID)
                        continue;
                /*
                 * If a node loses its panic status and is not online,
                 * take over services.  That is - someone decided *for sure*
                 * that said member is DOWN - so its state is no longer
                 * unknown.  (ie, disk-tiebreaker lost quorum...)
                 */
                getNodeName(x, &nodeName);

                if (memb_online(mask_panic, x) &&
                    !memb_online(cm_quorum_mask_panic(msg_quorum),x) &&
                    !memb_online(cm_quorum_mask(msg_quorum),x)) {
                        memb_mark_down(mask_panic, x);
                        node_event(x, NODE_DOWN);
                        clulog(LOG_INFO, "State change: %s DOWN\n",
                               nodeName);
                        continue;
                }

                if (!memb_online(node_delta, x))
                        continue;

                if (memb_online(cm_quorum_mask_panic(msg_quorum), x)) {
                        clulog(LOG_WARNING, "Member %s's state is uncertain: "
                               "Some services may be unavailable!",
                               nodeName);
                        continue;
                }

                node_event(x, NODE_DOWN);
                clulog(LOG_INFO, "State change: %s DOWN\n",
                       nodeName);
        }

        /*
         * Store our panic nodemask.
         */
        memcpy(mask_panic, cm_quorum_mask_panic(msg_quorum),
               sizeof(memb_mask_t));

        /*
         * Handle nodes gained.  Do our local node event first.
         */
        me = memb_mask_gained(node_delta, old_membership, membership);
        if (me) {
                clulog(LOG_INFO, "State change: Local UP\n");
                node_event(myNodeID, NODE_UP);
        }

        for (x=0; x<MAX_NODES; x++) {
                if (!memb_online(node_delta, x))
                        continue;

                if (x == myNodeID)
                        continue;

                node_event(x, NODE_UP);
                getNodeName(x, &nodeName);
                clulog(LOG_INFO, "State change: %s UP\n",
                       nodeName);
        }

        return 0;
}


/**
 * Read a message on a file descriptor (the one which is connected to
 * the quorumd daemon) and process it accordingly.
 *
 * @param fd            File descriptor connected to the quorum daemon.
 * @return              FAIL - no message waiting/empty message,
 *                      SUCCESS - successfully handled message.
 * @see                 dispatch_msg
 */
int
quorum_msg(msg_handle_t fd)
{
        cm_event_t      *msg_quorum;

        msg_quorum = cm_ev_read(fd);

        if (!msg_quorum)
                return FAIL;

        switch (cm_ev_event(msg_quorum)) {
        case EV_QUORUM_LOST:
                clulog(LOG_CRIT,"Halting services due to loss of quorum\n");
                svcmgr_exit(1, 0);
                /* NOT REACHED */
                break;

        case EV_QUORUM:
        case EV_QUORUM_GAINED:
                handle_quorum_msg(msg_quorum);
                break;

        case EV_NO_QUORUM:
                /* idle(); */
                break;

        default:
                clulog(LOG_DEBUG, "unhandled message request %d\n",
                       cm_ev_event(msg_quorum));
                break;
        }

        cm_ev_free(msg_quorum);
        return SUCCESS;
}


/**
 * Receive and process a message on a file descriptor and decide what to
 * do with it.  This function doesn't handle messages from the quorum daemon.
 *
 * @param fd            File descriptor with a waiting message.S
 * @return              FAIL - failed to receive/handle message, or invalid
 *                      data received.  SUCCESS - handled message successfully.
 * @see                 quorum_msg
 */
int
dispatch_msg(msg_handle_t fd)
{
        int ret;
        generic_msg_hdr msg_hdr;
        SmMessageSt     msg_sm;

        /* Peek-a-boo */
        ret = msg_peek(fd, &msg_hdr, sizeof(msg_hdr));
        if (ret != sizeof (generic_msg_hdr)) {
                clulog(LOG_ERR, "error receiving message header\n");
                return FAIL;
        }

        /* Decode the header */
        swab_generic_msg_hdr(&msg_hdr);
        if ((msg_hdr.gh_magic != GENERIC_HDR_MAGIC)) {
                clulog(LOG_ERR, "Invalid magic: Wanted 0x%08x, got 0x%08x\n",
                       GENERIC_HDR_MAGIC, msg_hdr.gh_magic);
                return FAIL;
        }

        clulog(LOG_DEBUG, "received message, fd %d\n", fd);

        switch (msg_hdr.gh_command) {
        case SVC_CONFIG_UPDATE:
                clulog(LOG_INFO, "Rereading configuration...\n");

                rebuild_config_lockless();

                msg_svc_init(1);
                set_loglevel();
                notify_everybody();
                break;

        case SVC_LOCK:
                clulog(LOG_NOTICE, "Service states locked\n");
                services_locked = 1;
                break;

        case SVC_UNLOCK:
                clulog(LOG_NOTICE, "Service states unlocked\n");
                services_locked = 0;
                break;

        case SVC_QUERY_LOCK:
                msg_send_simple(fd, services_locked?SVC_LOCK:SVC_UNLOCK, 0, 0);
                break;

        case SVC_ACTION_REQUEST:

                ret = msg_receive_timeout(fd, &msg_sm, sizeof(msg_sm), 
                                          MSG_TIMEOUT);
                if (ret != sizeof(msg_sm)) {
                        clulog(LOG_ERR, "receiving message data from client "
                               "error: %d\n", ret);
                        return FAIL;
                }

                /* Decode SmMessageSt message */
                swab_SmMessageSt(&msg_sm);

                if (services_locked) {
                        msg_sm.sm_data.d_ret = FAIL;
                        /* Encode before responding... */
                        swab_SmMessageSt(&msg_sm);

                        if (msg_send(fd, &msg_sm, sizeof (SmMessageSt)) !=
                            sizeof (SmMessageSt))
                                clulog(LOG_ERR,
                                       "Error replying to action request.\n");

                        break;
                }

                if (msg_sm.sm_data.d_action == SVC_FAILBACK) {
                        failback(msg_sm.sm_data.d_svcOwner);
                        break;
                }

                handle_svc_request(msg_sm.sm_data.d_svcID,
                                   msg_sm.sm_data.d_action, 
                                   msg_sm.sm_data.d_svcOwner, fd);
                break;

        default:
                clulog(LOG_DEBUG, "unhandled message request %d\n",
                       msg_hdr.gh_command);
                break;
        }
        return SUCCESS;
}


int
main(int argc, char **argv)
{
        struct timeval timeout, tv1, tv2;
        int elapsed_secs;
        int check_period = 0;
        msg_handle_t fd;
        int i;
        msg_handle_t listen_fd, quorum_fd;
        sigset_t set;
        fd_set rfds;
        extern char *optarg;
        int foreground = 0, debug = 0, opt, retries = 0;

        while ((opt = getopt(argc, argv, "fd")) != EOF) {
                switch (opt) {
                case 'd':
                        debug = 1;
                        break;
                case 'f':
                        foreground = 1;
                default:
                        break;
                }
        }

        if (!debug)
                (void) clu_set_loglevel(LOG_INFO);
        else
                (void) clu_set_loglevel(LOG_DEBUG);

        if (!foreground)
                daemon_init(argv[0]);
        else
                clu_log_console(1);

        /*
         * Generally, you do this when you know you have quorum.
         * However, the service manager simply doesn't get here without
         * quorum... (The quorum daemon spawns it when it achieves quorum)
         */
        shared_storage_init();
        switch(boot_config_init()) {
        case -1:
                clulog(LOG_CRIT, "Configuration invalid!\n");
                return -1;
        case 1:
                notify_everybody();
                break;
        case 0:
        default:
                break;
        }

        memset(membership,0,sizeof(memb_mask_t));

        set_facility();
        if (!debug)
                set_loglevel();

        clulog(LOG_DEBUG, "Service Manager starting\n");

        /*
         * daemon_init() blocks most signals, so we need to add the
         * ones the Service Manager is interested in.
         */
        sigemptyset(&set);
        sigaddset(&set, SIGINT);
        sigaddset(&set, SIGTERM);
        sigaddset(&set, SIGHUP);
        sigaddset(&set, SIGCHLD);
        sigprocmask(SIG_UNBLOCK, &set, NULL);
        (void) signal(SIGINT, (void (*)(int)) sigterm_handler);
        (void) signal(SIGTERM, (void (*)(int)) sigterm_handler);
        (void) signal(SIGHUP, (void (*)(int)) sighup_handler);
        (void) signal(SIGCHLD, (void (*)(int)) reap_zombies);

        /*
         * Retrieve our node id
         */
        myNodeID = memb_local_id();

        getNodeName(myNodeID, &myNodeName);
        myNodeName = strdup(myNodeName);
        myNodeState = NODE_DOWN;

        for (i = 0; i < MAX_SERVICES; i++) {
                ticks[i] = 0;
                svc_children[i].cs_pid = 0;
                svc_children[i].cs_rq = 0;
        }

        /*
         * Set up the message service
         */
        do {
                listen_fd = msg_listen(PROCID_CLUSVCMGRD);
                if (listen_fd >= 0)
                        break;

                if (++retries < 30) {
                        sleep(1);       /* Arbitrary... */
                        continue;
                }

                /* Could be that we lost and regained quorum really quickly */
                clulog(LOG_ERR, "Error setting up message listener: %s\n",
                               strerror(errno));
                clulog(LOG_ERR, "%s process may already be running.\n",
                       argv[0]);
                exit(1);
        } while (1);

        /*
         * Register for quorum events
         */
        do {
                quorum_fd = cm_ev_register(EC_QUORUM);
                if (quorum_fd >= 0)
                        break;

                if (++retries < 10) {
                        sleep(1);
                        continue;
                }
                
                clulog(LOG_CRIT, "Couldn't register with the quorum daemon!");
                exit(1);
        } while(1);

        while (1) {

                gettimeofday(&tv1, NULL);
#if 0
                /*
                 * Reap any zombied service scripts, as we do not synchronously
                 * wait on any of the service scripts. If the process was
                 * handling a service action, clear out the indication that it
                 * was running.
                 */

                reap_zombies();
#endif
                if (sighup_received) {
                        sighup_received = 0;
                        update_config();
                }

                if (sigterm_received)
                        svcmgr_exit(0, 1);

                FD_ZERO(&rfds);
                FD_SET(listen_fd, &rfds);
                FD_SET(quorum_fd, &rfds);
                timeout.tv_sec = 2;
                timeout.tv_usec = 0;

                i = select(MAX(listen_fd,quorum_fd) + 1, &rfds, NULL, NULL,
                           &timeout);

                /*
                 * We used to not check the return from the select call.
                 * However, this is necessary now because clusvcmgrd needs
                 * to properly handle SIGHUP
                 */
                if (i <= 0) {
                        FD_ZERO(&rfds);
                        if ((i == -1) && (errno != EINTR))
                                clulog(LOG_WARNING, "select: %s\n",
                                       strerror(errno));
                }

                if (FD_ISSET(listen_fd, &rfds)) {
                        fd = msg_accept_timeout(listen_fd, 1);
                        /*
                         * Process any waiting messages.
                         */
                        if (fd != -1) {
                                dispatch_msg(fd);
                                msg_close(fd);
                        }
                }

                if (FD_ISSET(quorum_fd, &rfds)) {
                        clulog(LOG_DEBUG, "Processing quorum event\n");
                        if (quorum_msg(quorum_fd) == -1) {
                                clulog(LOG_WARNING, "Invalid message from "
                                       "Quorum Daemon.  Reconnecting\n");
                                /* Failed to process it?  Try reconnecting */
                                cm_ev_unregister(quorum_fd);
                                sleep(2);
                                if (((quorum_fd =
                                      cm_ev_register(EC_QUORUM)) == -1) &&
                                    !sigterm_received) {
                                        
                                        clulog(LOG_EMERG, "Couldn't reconnect "
                                               "to the quorum daemon! "
                                               "REBOOTING");
                                        REBOOT(RB_AUTOBOOT);                                    
                                }
                        }
                }

                gettimeofday(&tv2, NULL);
                elapsed_secs = tv2.tv_sec - tv1.tv_sec;

                /*
                 * Check the status of running services and the cluster 
                 * configuration file (/etc/cluster.xml).
                 */
                if ((check_period += elapsed_secs) >= CHECK_INTERVAL) {
                        check_config_file();
                        if (check_config_data() == 1) {
                                rebuild_config_lockless();
                                msg_svc_init(1);
                                set_loglevel();
                                notify_everybody();
                        }
                        check_services(check_period);
                        check_period = 0;
                }
        }
}