cmogstored.c

   1 /*
   2  * Copyright (C) 2012-2020 all contributors <cmogstored-public@yhbt.net>
   3  * License: GPL-3.0+ <https://www.gnu.org/licenses/gpl-3.0.txt>
   4  */
   5 #include "cmogstored.h"
   6 #include "cfg.h"
   7 #include "nostd/setproctitle.h"
   8 #define THIS "cmogstored"
   9 #define DESC "alternative mogstored implementation for MogileFS"
  10 static char summary[] = THIS " -- "DESC;
  11 const char *argp_program_bug_address = PACKAGE_BUGREPORT;
  12 const char *argp_program_version = THIS" "PACKAGE_VERSION;
  13 static struct mog_fd *master_selfwake;
  14 static sig_atomic_t sigchld_hit;
  15 static sig_atomic_t do_exit;
  16 static sig_atomic_t do_upgrade;
  17 static pid_t master_pid;
  18 static pid_t upgrade_pid;
  19 static bool iostat_running;
  20
  21 static struct mog_main mog_main;
  22
  23 #define CFG_KEY(f) -((int)offsetof(struct mog_cfg,f) + 1)
  24 static struct argp_option options[] = {
  25         { .name = "daemonize", .key = 'd',
  26           .doc = "Daemonize" },
  27         { .name = "config", .key = CFG_KEY(configfile),
  28           .arg = "<file>",
  29           .doc = "Set config file (no default, unlike mogstored)" },
  30         { .name = "httplisten", .key = CFG_KEY(httplisten),
  31           .arg = "<ip:port>",
  32           .doc = "IP/Port HTTP server listens on" },
  33         /*
  34          * NOT adding httpgetlisten here to avoid (further) breaking
  35          * compatibility with Perl mogstored.  Most of these command-line
  36          * options suck anyways.
  37          */
  38         { .name = "mgmtlisten", .key = CFG_KEY(mgmtlisten),
  39           .arg = "<ip:port>",
  40           .doc = "IP/Port management/sidechannel listens on" },
  41         { .name = "docroot", .key = CFG_KEY(docroot),
  42           .arg = "<path>",
  43           .doc = "Docroot above device mount points.  "
  44                  "Defaults to "MOG_DEFAULT_DOCROOT
  45         },
  46         { .name = "maxconns", .key = CFG_KEY(maxconns),
  47           .arg = "<number>",
  48           .doc = "Number of simultaneous clients to serve. "
  49                  "Default " MOG_STR(MOG_DEFAULT_MAXCONNS) },
  50         { .name = "pidfile", .key = CFG_KEY(pidfile),
  51           .arg = "<path>",
  52           .doc = "path to PID file" },
  53         { .name = "server", .key = CFG_KEY(server), .flags = OPTION_HIDDEN,
  54           .arg = "(perlbal|none)"
  55         },
  56         {
  57           /* hidden for now, don't break compat with Perl mogstored */
  58           .name = "multi", .key = -'M', .flags = OPTION_HIDDEN
  59         },
  60         {
  61           /* hidden for now, don't break compat with Perl mogstored */
  62           .name = "worker-processes", .key = -'W', .flags = OPTION_HIDDEN,
  63           .arg = "<number>",
  64         },
  65         /* we don't load a default config file like Perl mogstored at all */
  66         { .name = "skipconfig", .key = 's', .flags = OPTION_HIDDEN },
  67         { NULL }
  68 };
  69
  70 static void new_cfg_or_die(const char *config)
  71 {
  72         struct mog_cfg *cfg = mog_cfg_new(config);
  73
  74         if (!cfg) die("invalid (or duplicate) config=%s", config);
  75         if (mog_cfg_load(cfg) == 0) return;
  76
  77         die("failed to load config=%s: %s",
  78             config, errno ? strerror(errno) : "parser error");
  79 }
  80
  81 static void check_strtoul(unsigned long *dst, const char *s, const char *key)
  82 {
  83         char *end;
  84
  85         errno = 0;
  86         *dst = strtoul(s, &end, 10);
  87         if (errno)
  88                 die_errno("failed to parse --%s=%s", key, s);
  89         if (*end)
  90                 die("failed to parse --%s=%s (invalid character: %c)",
  91                     key, s, *end);
  92 }
  93
  94 static void addr_or_die(struct mog_addrinfo **dst, const char *key, char *s)
  95 {
  96         *dst = mog_listen_parse(s);
  97         if (!*dst)
  98                 die("failed to parse %s=%s", key, s);
  99 }
 100
 101 static error_t parse_opt(int key, char *arg, struct argp_state *state)
 102 {
 103         struct mog_cfg *cfg = state->input;
 104         int rv = 0;
 105
 106         switch (key) {
 107         case 'd': cfg->daemonize = true; break;
 108         case 's': /* no-op, we don't load the default config file */; break;
 109         case CFG_KEY(docroot): cfg->docroot = xstrdup(arg); break;
 110         case CFG_KEY(pidfile): cfg->pidfile = xstrdup(arg); break;
 111         case CFG_KEY(configfile): new_cfg_or_die(arg); break;
 112         case CFG_KEY(maxconns):
 113                 check_strtoul(&cfg->maxconns, arg, "maxconns");
 114                 break;
 115         case CFG_KEY(httplisten):
 116                 addr_or_die(&cfg->httplisten, "httplisten", arg);
 117                 break;
 118         case CFG_KEY(mgmtlisten):
 119                 addr_or_die(&cfg->mgmtlisten, "mgmtlisten", arg);
 120                 break;
 121         case CFG_KEY(server):
 122                 cfg->server = xstrdup(arg);
 123                 mog_cfg_check_server(cfg);
 124                 break;
 125         case -'M': mog_cfg_multi = true; break;
 126         case -'W':
 127                 check_strtoul(&mog_main.worker_processes, arg,
 128                                 "worker-processes");
 129                 if (mog_main.worker_processes > UINT_MAX)
 130                         die("--worker-processes exceeded (max=%u)", UINT_MAX);
 131                 break;
 132         case ARGP_KEY_ARG:
 133                 argp_usage(state);
 134         case ARGP_KEY_END:
 135                 break;
 136         default:
 137                 rv = ARGP_ERR_UNKNOWN;
 138         }
 139
 140         return rv;
 141 }
 142
 143 static void dup2_null(int oldfd, int newfd, const char *errdest)
 144 {
 145         int rc;
 146
 147         do {
 148                 rc = dup2(oldfd, newfd);
 149         } while (rc < 0 && (errno == EINTR || errno == EBUSY));
 150
 151         if (rc < 0)
 152                 die_errno("dup2(/dev/null,%s) failed", errdest);
 153 }
 154
 155 static void daemonize(int null_fd)
 156 {
 157         int ready_pipe[2];
 158         pid_t pid;
 159         ssize_t r;
 160
 161         if (pipe(ready_pipe) < 0)
 162                 die_errno("pipe() failed");
 163         pid = fork();
 164         if (pid < 0)
 165                 die_errno("fork() failed");
 166         if (pid > 0) { /* parent */
 167                 mog_close(ready_pipe[1]);
 168                 do {
 169                         r = read(ready_pipe[0], &pid, sizeof(pid_t));
 170                 } while (r < 0 && errno == EINTR);
 171
 172                 PRESERVE_ERRNO( mog_close(ready_pipe[0]) );
 173                 if (r == sizeof(pid_t))
 174                         exit(EXIT_SUCCESS);
 175                 if (r < 0)
 176                         die_errno("ready_pipe read error");
 177                 die("daemonization error, check syslog");
 178         }
 179
 180         /* child */
 181         mog_close(ready_pipe[0]);
 182         if (setsid() < 0)
 183                 die_errno("setsid() failed");
 184
 185         pid = fork();
 186         if (pid < 0)
 187                 die_errno("fork() failed");
 188         if (pid > 0) /* intermediate parent */
 189                 exit(EXIT_SUCCESS);
 190
 191         if (chdir("/") < 0)
 192                 die_errno("chdir(/) failed");
 193
 194         dup2_null(null_fd, STDOUT_FILENO, "stdout");
 195         dup2_null(null_fd, STDERR_FILENO, "stderr");
 196         do {
 197                 r = write(ready_pipe[1], &pid, sizeof(pid_t));
 198         } while (r < 0 && errno == EINTR);
 199         if (r < 0)
 200                 syslog(LOG_CRIT, "ready_pipe write error: %m");
 201         else
 202                 assert(r == sizeof(pid_t) && "impossible short write");
 203         mog_close(ready_pipe[1]);
 204 }
 205
 206 #ifndef LOG_PERROR
 207 # define LOG_PERROR 0
 208 #endif
 209
 210 /* TODO: make logging configurable (how?) */
 211 static void log_init(bool is_daemon)
 212 {
 213         int mask = 0;
 214         int option = LOG_PID;
 215
 216         if (!is_daemon)
 217                 option |= LOG_PERROR;
 218
 219         openlog(THIS, option, LOG_DAEMON);
 220         mask |= LOG_MASK(LOG_EMERG);
 221         mask |= LOG_MASK(LOG_ALERT);
 222         mask |= LOG_MASK(LOG_CRIT);
 223         mask |= LOG_MASK(LOG_ERR);
 224         mask |= LOG_MASK(LOG_WARNING);
 225         mask |= LOG_MASK(LOG_NOTICE);
 226         mask |= LOG_MASK(LOG_INFO);
 227         /* mask |= LOG_MASK(LOG_DEBUG); */
 228         setlogmask(mask);
 229 }
 230
 231 MOG_NOINLINE static void setup(int argc, char *argv[])
 232 {
 233         int pid_fd = -1;
 234         static struct argp argp = { options, parse_opt, NULL, summary };
 235         int null_fd;
 236
 237         mog_mnt_refresh();
 238         argp_parse(&argp, argc, argv, 0, NULL, &mog_cli);
 239         mog_cfg_validate_or_die(&mog_cli);
 240         log_init(mog_cli.daemonize);
 241         mog_inherit_init();
 242         mog_cfg_svc_start_or_die(&mog_cli);
 243         mog_inherit_cleanup();
 244
 245         if (mog_cli.pidfile) pid_fd = mog_pidfile_prepare(mog_cli.pidfile);
 246
 247         null_fd = open("/dev/null", O_RDWR);
 248         if (null_fd < 0)
 249                 die_errno("open(/dev/null) failed");
 250         dup2_null(null_fd, STDIN_FILENO, "stdin");
 251
 252         /* don't daemonize if we're inheriting FDs, we're already daemonized */
 253         if (mog_cli.daemonize && !getenv("CMOGSTORED_FD"))
 254                 daemonize(null_fd);
 255
 256         mog_close(null_fd);
 257
 258         if (pid_fd >= 0 && mog_pidfile_commit(pid_fd) < 0)
 259                 syslog(LOG_ERR,
 260                        "failed to write pidfile(%s): %m. continuing...",
 261                        mog_cli.pidfile);
 262
 263         master_pid = getpid();
 264
 265         /* set svc->nmogdev on all svc */
 266         mog_mkusage_all();
 267 }
 268
 269 static void worker_wakeup_handler(int signum)
 270 {
 271         switch (signum) {
 272         case SIGUSR2: do_upgrade = 1; break;
 273         case SIGCHLD: sigchld_hit = 1; break;
 274         case SIGQUIT:
 275         case SIGTERM:
 276         case SIGINT:
 277                 do_exit = signum;
 278         }
 279         mog_notify(MOG_NOTIFY_SIGNAL);
 280 }
 281
 282 static void wakeup_noop(int signum)
 283 {
 284         /* just something to cause EINTR */
 285 }
 286
 287 static void master_wakeup_handler(int signum)
 288 {
 289         switch (signum) {
 290         case SIGCHLD: sigchld_hit = 1; break;
 291         case SIGUSR2: do_upgrade = 1; break;
 292         case SIGQUIT:
 293         case SIGTERM:
 294         case SIGINT:
 295                 do_exit = signum;
 296         }
 297         mog_selfwake_trigger(master_selfwake);
 298 }
 299
 300 static void siginit(void (*wakeup_handler)(int))
 301 {
 302         struct sigaction sa;
 303
 304         memset(&sa, 0, sizeof(struct sigaction));
 305         CHECK(int, 0, sigemptyset(&sa.sa_mask) );
 306
 307         sa.sa_handler = SIG_IGN;
 308         CHECK(int, 0, sigaction(SIGPIPE, &sa, NULL));
 309
 310         sa.sa_handler = wakeup_noop;
 311         CHECK(int, 0, sigaction(SIGURG, &sa, NULL));
 312
 313         sa.sa_handler = wakeup_handler;
 314
 315         /* TERM and INT are graceful shutdown for now, no immediate shutdown */
 316         CHECK(int, 0, sigaction(SIGTERM, &sa, NULL));
 317         CHECK(int, 0, sigaction(SIGINT, &sa, NULL));
 318
 319         CHECK(int, 0, sigaction(SIGQUIT, &sa, NULL)); /* graceful shutdown */
 320         CHECK(int, 0, sigaction(SIGUSR1, &sa, NULL)); /* no-op, nginx compat */
 321         CHECK(int, 0, sigaction(SIGUSR2, &sa, NULL)); /* upgrade */
 322
 323         /*
 324          * SIGWINCH/SIGHUP are no-ops for now to allow reuse of nginx init
 325          * scripts.  We should support them in the future.
 326          * SIGWINCH will disable new connections and drop idlers
 327          * SIGHUP will reenable new connections/idlers after SIGWINCH
 328          */
 329         CHECK(int, 0, sigaction(SIGWINCH, &sa, NULL));
 330         CHECK(int, 0, sigaction(SIGHUP, &sa, NULL));
 331
 332         sa.sa_flags = SA_NOCLDSTOP;
 333         CHECK(int, 0, sigaction(SIGCHLD, &sa, NULL));
 334 }
 335
 336 static void process_died(pid_t pid, int status);
 337
 338 static void sigchld_handler(void)
 339 {
 340         sigchld_hit = 0;
 341
 342         for (;;) {
 343                 int status;
 344                 pid_t pid = waitpid(-1, &status, WNOHANG);
 345
 346                 if (pid > 0) {
 347                         process_died(pid, status);
 348                 } else if (pid == 0) {
 349                         return;
 350                 } else {
 351                         switch (errno) {
 352                         case EINTR: sigchld_hit = 1; return; /* retry later */
 353                         case ECHILD: return;
 354                         default: die_errno("waitpid");
 355                         }
 356                 }
 357         }
 358 }
 359
 360 static void upgrade_handler(void)
 361 {
 362         do_upgrade = 0;
 363         if (upgrade_pid > 0) {
 364                 syslog(LOG_INFO, "upgrade already running on PID:%d",
 365                        upgrade_pid);
 366         } else {
 367                 if (master_pid == getpid())
 368                         upgrade_pid = mog_upgrade_spawn();
 369                 /* else: worker processes (if configured) do not upgrade */
 370         }
 371 }
 372
 373 static void main_worker_loop(const pid_t parent)
 374 {
 375         while (parent == 0 || parent == getppid()) {
 376                 mog_notify_wait(mog_main.have_mgmt);
 377                 if (sigchld_hit)
 378                         sigchld_handler();
 379                 if (do_upgrade)
 380                         upgrade_handler();
 381                 if (do_exit)
 382                         cmogstored_exit();
 383                 if (mog_main.have_mgmt)
 384                         mog_mnt_refresh();
 385                 else if (mog_main.have_mgmt && !iostat_running && !do_exit)
 386                         /*
 387                          * maybe iostat was not installed/available/usable at
 388                          * startup, but became usable later
 389                          */
 390                         iostat_running = mog_iostat_respawn(0);
 391         }
 392
 393         syslog(LOG_INFO, "parent=%d abandoned us, dying", parent);
 394         cmogstored_exit();
 395 }
 396
 397 static void run_worker(const pid_t parent)
 398 {
 399         mog_notify_init();
 400         siginit(worker_wakeup_handler);
 401
 402         /* this can set mog_main->have_mgmt */
 403         mog_svc_each(mog_svc_start_each, &mog_main);
 404
 405         if (mog_main.have_mgmt) {
 406                 iostat_running = mog_iostat_respawn(0);
 407                 if (!iostat_running)
 408                         syslog(LOG_WARNING, "iostat(1) not available/running");
 409         }
 410         main_worker_loop(parent);
 411 }
 412
 413 static void fork_worker(unsigned worker_id)
 414 {
 415         pid_t pid;
 416         pid_t parent = getpid(); /* not using getppid() since it's racy */
 417
 418         pid = fork();
 419         if (pid > 0) {
 420                 mog_process_register(pid, worker_id);
 421         } else if (pid == 0) {
 422                 mog_selfwake_put(master_selfwake);
 423                 mog_process_reset();
 424                 mog_svc_each(mog_svc_atfork_child, &parent);
 425
 426                 /* worker will call mog_intr_enable() later in notify loop */
 427                 run_worker(parent);
 428                 exit(EXIT_SUCCESS);
 429         } else {
 430                 syslog(LOG_ERR, "fork() failed: %m, sleeping for 10s");
 431                 mog_sleep(10);
 432         }
 433 }
 434
 435 /* run when a worker dies */
 436 static void worker_died(pid_t pid, int status, unsigned id)
 437 {
 438         if (do_exit)
 439                 return;
 440
 441         syslog(LOG_INFO,
 442                "worker[%u] PID:%d died with status=%d, respawning",
 443                id, (int)pid, status);
 444         fork_worker(id);
 445 }
 446
 447 static void iostat_died(pid_t pid, int status)
 448 {
 449         if (do_exit)
 450                 return;
 451         iostat_running = mog_iostat_respawn(status);
 452 }
 453
 454 /* run when any process dies */
 455 static void process_died(pid_t pid, int status)
 456 {
 457         unsigned id = mog_process_reaped(pid);
 458         char *name;
 459
 460         if (mog_process_is_worker(id)) {
 461                 worker_died(pid, status, id);
 462                 return;
 463         }
 464
 465         switch (id) {
 466         case MOG_PROC_IOSTAT:
 467                 iostat_died(pid, status);
 468                 return;
 469         case MOG_PROC_UPGRADE:
 470                 assert(pid == upgrade_pid && "upgrade_pid misplaced");
 471                 syslog(LOG_INFO, "upgrade PID:%d exited with status=%d",
 472                        pid, status);
 473                 mog_pidfile_upgrade_abort();
 474                 upgrade_pid = -1;
 475                 return;
 476         default:
 477                 /* could be an inherited iostat if we're using worker+master */
 478                 name = mog_process_name(id);
 479                 if (!name)
 480                         syslog(LOG_ERR, "OOM: %m");
 481                 syslog(LOG_INFO,
 482                        "reaped %s pid=%d with status=%d, ignoring",
 483                        name ? name : "unknown", (int)pid, status);
 484                 free(name);
 485         }
 486 }
 487
 488 static void run_master(void)
 489 {
 490         unsigned id;
 491         size_t running = mog_main.worker_processes;
 492
 493         master_selfwake = mog_selfwake_new();
 494         siginit(master_wakeup_handler);
 495
 496         for (id = 0; id < mog_main.worker_processes; id++)
 497                 fork_worker(id);
 498
 499         while (running > 0) {
 500                 mog_selfwake_wait(master_selfwake);
 501                 if (sigchld_hit)
 502                         sigchld_handler();
 503                 if (do_upgrade)
 504                         upgrade_handler();
 505                 if (do_exit)
 506                         running = mog_kill_each_worker(SIGQUIT);
 507         }
 508         mog_selfwake_put(master_selfwake);
 509 }
 510
 511 int main(int argc, char *argv[], char *envp[])
 512 {
 513         mog_upgrade_prepare(argc, argv, envp);
 514         /* hack for older gcov + gcc, see nostd/setproctitle.h */
 515         spt_init(argc, argv, envp);
 516         set_program_name(argv[0]);
 517
 518         mog_intr_disable();
 519         setup(argc, argv); /* this daemonizes */
 520
 521         mog_process_init(mog_main.worker_processes);
 522         if (mog_main.worker_processes == 0)
 523                 run_worker(0);
 524         else
 525                 run_master();
 526
 527         return 0;
 528 }
 529
 530 /* called by the "shutdown" command via mgmt */
 531 void cmogstored_quit(void)
 532 {
 533         if (master_pid != getpid()) {
 534                 if (kill(master_pid, SIGQUIT) != 0)
 535                         syslog(LOG_ERR,
 536                                "SIGQUIT failed on master process (pid=%d): %m",
 537                                 master_pid);
 538         } else {
 539                 CHECK(int, 0, kill(getpid(), SIGQUIT));
 540         }
 541 }