src/vol/salvaged.c

   1 /*
   2  * Copyright 2006-2007, Sine Nomine Associates and others.
   3  * All Rights Reserved.
   4  *
   5  * This software has been released under the terms of the IBM Public
   6  * License.  For details, see the LICENSE file in the top-level source
   7  * directory or online at http://www.openafs.org/dl/license10.html
   8  */
   9
  10 /*
  11  * demand attach fs
  12  * online salvager daemon
  13  */
  14
  15 /* Main program file. Define globals. */
  16 #define MAIN 1
  17
  18 #include <afsconfig.h>
  19 #include <afs/param.h>
  20
  21
  22 #include <stdlib.h>
  23 #include <stdio.h>
  24 #include <string.h>
  25 #include <dirent.h>
  26 #include <sys/stat.h>
  27 #include <time.h>
  28 #include <errno.h>
  29 #ifdef AFS_NT40_ENV
  30 #include <io.h>
  31 #include <WINNT/afsevent.h>
  32 #else
  33 #include <sys/param.h>
  34 #include <sys/file.h>
  35 #ifndef ITIMER_REAL
  36 #include <sys/time.h>
  37 #endif /* ITIMER_REAL */
  38 #endif
  39 #ifndef WCOREDUMP
  40 #define WCOREDUMP(x)    ((x) & 0200)
  41 #endif
  42 #include <rx/xdr.h>
  43 #include <afs/afsint.h>
  44 #include <afs/afs_assert.h>
  45 #if !defined(AFS_SGI_ENV) && !defined(AFS_NT40_ENV)
  46 #if defined(AFS_VFSINCL_ENV)
  47 #include <sys/vnode.h>
  48 #ifdef  AFS_SUN5_ENV
  49 #include <sys/fs/ufs_inode.h>
  50 #else
  51 #if defined(AFS_DARWIN_ENV) || defined(AFS_XBSD_ENV)
  52 #include <ufs/ufs/dinode.h>
  53 #include <ufs/ffs/fs.h>
  54 #else
  55 #include <ufs/inode.h>
  56 #endif
  57 #endif
  58 #else /* AFS_VFSINCL_ENV */
  59 #ifdef  AFS_OSF_ENV
  60 #include <ufs/inode.h>
  61 #else /* AFS_OSF_ENV */
  62 #if !defined(AFS_LINUX20_ENV) && !defined(AFS_XBSD_ENV) && !defined(AFS_DARWIN_ENV)
  63 #include <sys/inode.h>
  64 #endif
  65 #endif
  66 #endif /* AFS_VFSINCL_ENV */
  67 #endif /* AFS_SGI_ENV */
  68 #ifdef  AFS_AIX_ENV
  69 #include <sys/vfs.h>
  70 #include <sys/lockf.h>
  71 #else
  72 #ifdef  AFS_HPUX_ENV
  73 #include <unistd.h>
  74 #include <checklist.h>
  75 #else
  76 #if defined(AFS_SGI_ENV)
  77 #include <unistd.h>
  78 #include <fcntl.h>
  79 #include <mntent.h>
  80 #else
  81 #if     defined(AFS_SUN_ENV) || defined(AFS_SUN5_ENV)
  82 #ifdef    AFS_SUN5_ENV
  83 #include <unistd.h>
  84 #include <sys/mnttab.h>
  85 #include <sys/mntent.h>
  86 #else
  87 #include <mntent.h>
  88 #endif
  89 #else
  90 #endif /* AFS_SGI_ENV */
  91 #endif /* AFS_HPUX_ENV */
  92 #endif
  93 #endif
  94 #include <fcntl.h>
  95 #ifndef AFS_NT40_ENV
  96 #include <afs/osi_inode.h>
  97 #endif
  98 #include <afs/cmd.h>
  99 #include <afs/afsutil.h>
 100 #include <afs/fileutil.h>
 101 #include <afs/procmgmt.h>       /* signal(), kill(), wait(), etc. */
 102 #include <afs/dir.h>
 103 #ifndef AFS_NT40_ENV
 104 #include <syslog.h>
 105 #endif
 106
 107 #include "nfs.h"
 108 #include "lwp.h"
 109 #include "lock.h"
 110 #include <afs/afssyscalls.h>
 111 #include "ihandle.h"
 112 #include "vnode.h"
 113 #include "volume.h"
 114 #include "partition.h"
 115 #include "daemon_com.h"
 116 #include "fssync.h"
 117 #include "salvsync.h"
 118 #include "viceinode.h"
 119 #include "salvage.h"
 120 #include "vol-salvage.h"
 121 #include "common.h"
 122 #ifdef AFS_NT40_ENV
 123 #include <pthread.h>
 124 #endif
 125
 126
 127 #if !defined(AFS_DEMAND_ATTACH_FS)
 128 #error "online salvager only supported for demand attach fileserver"
 129 #endif /* AFS_DEMAND_ATTACH_FS */
 130
 131 #if defined(AFS_NT40_ENV)
 132 #error "online salvager not supported on NT"
 133 #endif /* AFS_NT40_ENV */
 134
 135 /*@+fcnmacros +macrofcndecl@*/
 136 #ifdef O_LARGEFILE
 137 #define afs_fopen       fopen64
 138 #else /* !O_LARGEFILE */
 139 #define afs_fopen       fopen
 140 #endif /* !O_LARGEFILE */
 141 /*@=fcnmacros =macrofcndecl@*/
 142
 143
 144
 145 static volatile int current_workers = 0;
 146 static volatile struct rx_queue pending_q;
 147 static pthread_mutex_t worker_lock;
 148 static pthread_cond_t worker_cv;
 149
 150 static void * SalvageChildReaperThread(void *);
 151 static int DoSalvageVolume(struct SalvageQueueNode * node, int slot);
 152
 153 static void SalvageServer(int argc, char **argv);
 154 static void SalvageClient(VolumeId vid, char * pname);
 155
 156 static int Reap_Child(char * prog, int * pid, int * status);
 157
 158 static void * SalvageLogCleanupThread(void *);
 159 static int SalvageLogCleanup(int pid);
 160
 161 static void * SalvageLogScanningThread(void *);
 162 static void ScanLogs(struct rx_queue *log_watch_queue);
 163
 164 struct cmdline_rock {
 165     int argc;
 166     char **argv;
 167 };
 168
 169 struct log_cleanup_node {
 170     struct rx_queue q;
 171     int pid;
 172 };
 173
 174 struct {
 175     struct rx_queue queue_head;
 176     pthread_cond_t queue_change_cv;
 177 } log_cleanup_queue;
 178
 179
 180 #define DEFAULT_PARALLELISM 4 /* allow 4 parallel salvage workers by default */
 181
 182 static int
 183 handleit(struct cmd_syndesc *as, void *arock)
 184 {
 185     struct cmd_item *ti;
 186     char pname[100], *temp;
 187     afs_int32 seenpart = 0, seenvol = 0;
 188     VolumeId vid = 0;
 189     struct cmdline_rock *rock = (struct cmdline_rock *)arock;
 190
 191 #ifdef AFS_SGI_VNODE_GLUE
 192     if (afs_init_kernel_config(-1) < 0) {
 193         printf
 194             ("Can't determine NUMA configuration, not starting salvager.\n");
 195         exit(1);
 196     }
 197 #endif
 198
 199     if (as->parms[2].items)     /* -debug */
 200         debug = 1;
 201     if (as->parms[3].items)     /* -nowrite */
 202         Testing = 1;
 203     if (as->parms[4].items)     /* -inodes */
 204         ListInodeOption = 1;
 205     if (as->parms[5].items)     /* -oktozap */
 206         OKToZap = 1;
 207     if (as->parms[6].items)     /* -rootinodes */
 208         ShowRootFiles = 1;
 209     if (as->parms[7].items)     /* -salvagedirs */
 210         RebuildDirs = 1;
 211     if (as->parms[8].items)     /* -ForceReads */
 212         forceR = 1;
 213     if ((ti = as->parms[9].items)) {    /* -Parallel # */
 214         temp = ti->data;
 215         if (strncmp(temp, "all", 3) == 0) {
 216             PartsPerDisk = 1;
 217             temp += 3;
 218         }
 219         if (strlen(temp) != 0) {
 220             Parallel = atoi(temp);
 221             if (Parallel < 1)
 222                 Parallel = 1;
 223             if (Parallel > MAXPARALLEL) {
 224                 printf("Setting parallel salvages to maximum of %d \n",
 225                        MAXPARALLEL);
 226                 Parallel = MAXPARALLEL;
 227             }
 228         }
 229     } else {
 230         Parallel = MIN(DEFAULT_PARALLELISM, MAXPARALLEL);
 231     }
 232     if ((ti = as->parms[10].items)) {   /* -tmpdir */
 233         DIR *dirp;
 234
 235         tmpdir = ti->data;
 236         dirp = opendir(tmpdir);
 237         if (!dirp) {
 238             printf
 239                 ("Can't open temporary placeholder dir %s; using current partition \n",
 240                  tmpdir);
 241             tmpdir = NULL;
 242         } else
 243             closedir(dirp);
 244     }
 245     if ((ti = as->parms[11].items))     /* -showlog */
 246         ShowLog = 1;
 247     if ((ti = as->parms[12].items)) {   /* -orphans */
 248         if (Testing)
 249             orphans = ORPH_IGNORE;
 250         else if (strcmp(ti->data, "remove") == 0
 251                  || strcmp(ti->data, "r") == 0)
 252             orphans = ORPH_REMOVE;
 253         else if (strcmp(ti->data, "attach") == 0
 254                  || strcmp(ti->data, "a") == 0)
 255             orphans = ORPH_ATTACH;
 256     }
 257 #ifndef AFS_NT40_ENV            /* ignore options on NT */
 258     if ((ti = as->parms[13].items)) {   /* -syslog */
 259         useSyslog = 1;
 260         ShowLog = 0;
 261     }
 262     if ((ti = as->parms[14].items)) {   /* -syslogfacility */
 263         useSyslogFacility = atoi(ti->data);
 264     }
 265
 266     if ((ti = as->parms[15].items)) {   /* -datelogs */
 267         TimeStampLogFile((char *)AFSDIR_SERVER_SALSRVLOG_FILEPATH);
 268     }
 269 #endif
 270
 271     if ((ti = as->parms[16].items)) {   /* -client */
 272         if ((ti = as->parms[0].items)) {        /* -partition */
 273             seenpart = 1;
 274             strlcpy(pname, ti->data, sizeof(pname));
 275         }
 276         if ((ti = as->parms[1].items)) {        /* -volumeid */
 277             char *end;
 278             unsigned long vid_l;
 279             seenvol = 1;
 280             vid_l = strtoul(ti->data, &end, 10);
 281             if (vid_l >= MAX_AFS_UINT32 || vid_l == ULONG_MAX || *end != '\0') {
 282                 printf("Invalid volume id specified; salvage aborted\n");
 283                 exit(-1);
 284             }
 285             vid = (VolumeId)vid_l;
 286         }
 287
 288         if (ShowLog) {
 289             printf("-showlog does not work with -client\n");
 290             exit(-1);
 291         }
 292
 293         if (!seenpart || !seenvol) {
 294             printf("You must specify '-partition' and '-volumeid' with the '-client' option\n");
 295             exit(-1);
 296         }
 297
 298         SalvageClient(vid, pname);
 299
 300     } else {  /* salvageserver mode */
 301         SalvageServer(rock->argc, rock->argv);
 302     }
 303     return (0);
 304 }
 305
 306
 307 #ifndef AFS_NT40_ENV
 308 #include "AFS_component_version_number.c"
 309 #endif
 310 #define MAX_ARGS 128
 311 #ifdef AFS_NT40_ENV
 312 char *save_args[MAX_ARGS];
 313 int n_save_args = 0;
 314 pthread_t main_thread;
 315 #endif
 316
 317 int
 318 main(int argc, char **argv)
 319 {
 320     struct cmd_syndesc *ts;
 321     int err = 0;
 322     struct cmdline_rock arock;
 323
 324 #ifdef  AFS_AIX32_ENV
 325     /*
 326      * The following signal action for AIX is necessary so that in case of a
 327      * crash (i.e. core is generated) we can include the user's data section
 328      * in the core dump. Unfortunately, by default, only a partial core is
 329      * generated which, in many cases, isn't too useful.
 330      */
 331     struct sigaction nsa;
 332
 333     sigemptyset(&nsa.sa_mask);
 334     nsa.sa_handler = SIG_DFL;
 335     nsa.sa_flags = SA_FULLDUMP;
 336     sigaction(SIGABRT, &nsa, NULL);
 337     sigaction(SIGSEGV, &nsa, NULL);
 338 #endif
 339
 340     /* Initialize directory paths */
 341     if (!(initAFSDirPath() & AFSDIR_SERVER_PATHS_OK)) {
 342 #ifdef AFS_NT40_ENV
 343         ReportErrorEventAlt(AFSEVT_SVR_NO_INSTALL_DIR, 0, argv[0], 0);
 344 #endif
 345         fprintf(stderr, "%s: Unable to obtain AFS server directory.\n",
 346                 argv[0]);
 347         exit(2);
 348     }
 349 #ifdef AFS_NT40_ENV
 350     /* Default to binary mode for fopen() */
 351     _set_fmode(_O_BINARY);
 352
 353     main_thread = pthread_self();
 354     if (spawnDatap && spawnDataLen) {
 355         /* This is a child per partition salvager. Don't setup log or
 356          * try to lock the salvager lock.
 357          */
 358         if (nt_SetupPartitionSalvage(spawnDatap, spawnDataLen) < 0)
 359             exit(3);
 360     } else {
 361 #endif
 362
 363 #ifndef AFS_NT40_ENV
 364         if (geteuid() != 0) {
 365             printf("Salvager must be run as root.\n");
 366             fflush(stdout);
 367             Exit(0);
 368         }
 369 #endif
 370
 371         /* bad for normal help flag processing, but can do nada */
 372
 373 #ifdef AFS_NT40_ENV
 374     }
 375 #endif
 376
 377     arock.argc = argc;
 378     arock.argv = argv;
 379
 380     ts = cmd_CreateSyntax("initcmd", handleit, &arock, "initialize the program");
 381     cmd_AddParm(ts, "-partition", CMD_SINGLE, CMD_OPTIONAL,
 382                 "Name of partition to salvage");
 383     cmd_AddParm(ts, "-volumeid", CMD_SINGLE, CMD_OPTIONAL,
 384                 "Volume Id to salvage");
 385     cmd_AddParm(ts, "-debug", CMD_FLAG, CMD_OPTIONAL,
 386                 "Run in Debugging mode");
 387     cmd_AddParm(ts, "-nowrite", CMD_FLAG, CMD_OPTIONAL,
 388                 "Run readonly/test mode");
 389     cmd_AddParm(ts, "-inodes", CMD_FLAG, CMD_OPTIONAL,
 390                 "Just list affected afs inodes - debugging flag");
 391     cmd_AddParm(ts, "-oktozap", CMD_FLAG, CMD_OPTIONAL,
 392                 "Give permission to destroy bogus inodes/volumes - debugging flag");
 393     cmd_AddParm(ts, "-rootinodes", CMD_FLAG, CMD_OPTIONAL,
 394                 "Show inodes owned by root - debugging flag");
 395     cmd_AddParm(ts, "-salvagedirs", CMD_FLAG, CMD_OPTIONAL,
 396                 "Force rebuild/salvage of all directories");
 397     cmd_AddParm(ts, "-blockreads", CMD_FLAG, CMD_OPTIONAL,
 398                 "Read smaller blocks to handle IO/bad blocks");
 399     cmd_AddParm(ts, "-parallel", CMD_SINGLE, CMD_OPTIONAL,
 400                 "# of max parallel partition salvaging");
 401     cmd_AddParm(ts, "-tmpdir", CMD_SINGLE, CMD_OPTIONAL,
 402                 "Name of dir to place tmp files ");
 403     cmd_AddParm(ts, "-showlog", CMD_FLAG, CMD_OPTIONAL,
 404                 "Show log file upon completion");
 405     cmd_AddParm(ts, "-orphans", CMD_SINGLE, CMD_OPTIONAL,
 406                 "ignore | remove | attach");
 407
 408     /* note - syslog isn't avail on NT, but if we make it conditional, have
 409      * to deal with screwy offsets for cmd params */
 410     cmd_AddParm(ts, "-syslog", CMD_FLAG, CMD_OPTIONAL,
 411                 "Write salvage log to syslogs");
 412     cmd_AddParm(ts, "-syslogfacility", CMD_SINGLE, CMD_OPTIONAL,
 413                 "Syslog facility number to use");
 414     cmd_AddParm(ts, "-datelogs", CMD_FLAG, CMD_OPTIONAL,
 415                 "Include timestamp in logfile filename");
 416
 417     cmd_AddParm(ts, "-client", CMD_FLAG, CMD_OPTIONAL,
 418                 "Use SALVSYNC to ask salvageserver to salvage a volume");
 419
 420     err = cmd_Dispatch(argc, argv);
 421     Exit(err);
 422     return 0; /* not reached */
 423 }
 424
 425 static void
 426 SalvageClient(VolumeId vid, char * pname)
 427 {
 428     int done = 0;
 429     afs_int32 code;
 430     SYNC_response res;
 431     SALVSYNC_response_hdr sres;
 432     VolumePackageOptions opts;
 433
 434     VOptDefaults(volumeUtility, &opts);
 435     if (VInitVolumePackage2(volumeUtility, &opts)) {
 436         /* VInitVolumePackage2 can fail on e.g. partition attachment errors,
 437          * but we don't really care, since all we're doing is trying to use
 438          * SALVSYNC */
 439         fprintf(stderr, "errors encountered initializing volume package, but "
 440                         "trying to continue anyway\n");
 441     }
 442     SALVSYNC_clientInit();
 443
 444     code = SALVSYNC_SalvageVolume(vid, pname, SALVSYNC_SALVAGE, SALVSYNC_OPERATOR, 0, NULL);
 445     if (code != SYNC_OK) {
 446         goto sync_error;
 447     }
 448
 449     res.payload.buf = (void *) &sres;
 450     res.payload.len = sizeof(sres);
 451
 452     while(!done) {
 453         sleep(2);
 454         code = SALVSYNC_SalvageVolume(vid, pname, SALVSYNC_QUERY, SALVSYNC_WHATEVER, 0, &res);
 455         if (code != SYNC_OK) {
 456             goto sync_error;
 457         }
 458         switch (sres.state) {
 459         case SALVSYNC_STATE_ERROR:
 460             printf("salvageserver reports salvage ended in an error; check log files for more details\n");
 461         case SALVSYNC_STATE_DONE:
 462         case SALVSYNC_STATE_UNKNOWN:
 463             done = 1;
 464         }
 465     }
 466     SALVSYNC_clientFinis();
 467     return;
 468
 469  sync_error:
 470     if (code == SYNC_DENIED) {
 471         printf("salvageserver refused to salvage volume %u on partition %s\n",
 472                vid, pname);
 473     } else if (code == SYNC_BAD_COMMAND) {
 474         printf("SALVSYNC protocol mismatch; please make sure fileserver, volserver, salvageserver and salvager are same version\n");
 475     } else if (code == SYNC_COM_ERROR) {
 476         printf("SALVSYNC communications error\n");
 477     }
 478     SALVSYNC_clientFinis();
 479     exit(-1);
 480 }
 481
 482 static int * child_slot;
 483
 484 static void
 485 SalvageServer(int argc, char **argv)
 486 {
 487     int pid, ret;
 488     struct SalvageQueueNode * node;
 489     pthread_t tid;
 490     pthread_attr_t attrs;
 491     int slot;
 492     VolumePackageOptions opts;
 493
 494     /* All entries to the log will be appended.  Useful if there are
 495      * multiple salvagers appending to the log.
 496      */
 497
 498     CheckLogFile((char *)AFSDIR_SERVER_SALSRVLOG_FILEPATH);
 499 #ifndef AFS_NT40_ENV
 500 #ifdef AFS_LINUX20_ENV
 501     fcntl(fileno(logFile), F_SETFL, O_APPEND);  /* Isn't this redundant? */
 502 #else
 503     fcntl(fileno(logFile), F_SETFL, FAPPEND);   /* Isn't this redundant? */
 504 #endif
 505 #endif
 506     setlinebuf(logFile);
 507
 508     fprintf(logFile, "%s\n", cml_version_number);
 509     LogCommandLine(argc, argv, "Online Salvage Server",
 510                    SalvageVersion, "Starting OpenAFS", Log);
 511     /* Get and hold a lock for the duration of the salvage to make sure
 512      * that no other salvage runs at the same time.  The routine
 513      * VInitVolumePackage2 (called below) makes sure that a file server or
 514      * other volume utilities don't interfere with the salvage.
 515      */
 516
 517     /* even demand attach online salvager
 518      * still needs this because we don't want
 519      * a stand-alone salvager to conflict with
 520      * the salvager daemon */
 521     ObtainSharedSalvageLock();
 522
 523     child_slot = (int *) malloc(Parallel * sizeof(int));
 524     osi_Assert(child_slot != NULL);
 525     memset(child_slot, 0, Parallel * sizeof(int));
 526
 527     /* initialize things */
 528     VOptDefaults(salvageServer, &opts);
 529     if (VInitVolumePackage2(salvageServer, &opts)) {
 530         Log("Shutting down: errors encountered initializing volume package\n");
 531         Exit(1);
 532     }
 533     DInit(10);
 534     queue_Init(&pending_q);
 535     queue_Init(&log_cleanup_queue);
 536     MUTEX_INIT(&worker_lock, "worker", MUTEX_DEFAULT, 0);
 537     CV_INIT(&worker_cv, "worker", CV_DEFAULT, 0);
 538     CV_INIT(&log_cleanup_queue.queue_change_cv, "queuechange", CV_DEFAULT, 0);
 539     osi_Assert(pthread_attr_init(&attrs) == 0);
 540
 541     /* start up the reaper and log cleaner threads */
 542     osi_Assert(pthread_attr_setdetachstate(&attrs, PTHREAD_CREATE_DETACHED) == 0);
 543     osi_Assert(pthread_create(&tid,
 544                           &attrs,
 545                           &SalvageChildReaperThread,
 546                           NULL) == 0);
 547     osi_Assert(pthread_create(&tid,
 548                           &attrs,
 549                           &SalvageLogCleanupThread,
 550                           NULL) == 0);
 551     osi_Assert(pthread_create(&tid,
 552                           &attrs,
 553                           &SalvageLogScanningThread,
 554                           NULL) == 0);
 555
 556     /* loop forever serving requests */
 557     while (1) {
 558         node = SALVSYNC_getWork();
 559         osi_Assert(node != NULL);
 560
 561         Log("dispatching child to salvage volume %u...\n",
 562             node->command.sop.parent);
 563
 564         VOL_LOCK;
 565         /* find a slot */
 566         for (slot = 0; slot < Parallel; slot++) {
 567           if (!child_slot[slot])
 568             break;
 569         }
 570         osi_Assert (slot < Parallel);
 571
 572     do_fork:
 573         pid = Fork();
 574         if (pid == 0) {
 575             VOL_UNLOCK;
 576             ret = DoSalvageVolume(node, slot);
 577             Exit(ret);
 578         } else if (pid < 0) {
 579             Log("failed to fork child worker process\n");
 580             sleep(1);
 581             goto do_fork;
 582         } else {
 583             child_slot[slot] = pid;
 584             node->pid = pid;
 585             VOL_UNLOCK;
 586
 587             MUTEX_ENTER(&worker_lock);
 588             current_workers++;
 589
 590             /* let the reaper thread know another worker was spawned */
 591             CV_BROADCAST(&worker_cv);
 592
 593             /* if we're overquota, wait for the reaper */
 594             while (current_workers >= Parallel) {
 595                 CV_WAIT(&worker_cv, &worker_lock);
 596             }
 597             MUTEX_EXIT(&worker_lock);
 598         }
 599     }
 600 }
 601
 602 static int
 603 DoSalvageVolume(struct SalvageQueueNode * node, int slot)
 604 {
 605     char childLog[AFSDIR_PATH_MAX];
 606     struct DiskPartition64 * partP;
 607
 608     /* do not allow further forking inside salvager */
 609     canfork = 0;
 610
 611     /* do not attempt to close parent's logFile handle as
 612      * another thread may have held the lock on the FILE
 613      * structure when fork was called! */
 614
 615     afs_snprintf(childLog, sizeof(childLog), "%s.%d",
 616                  AFSDIR_SERVER_SLVGLOG_FILEPATH, getpid());
 617
 618     logFile = afs_fopen(childLog, "a");
 619     if (!logFile) {             /* still nothing, use stdout */
 620         logFile = stdout;
 621         ShowLog = 0;
 622     }
 623
 624     if (node->command.sop.parent <= 0) {
 625         Log("salvageServer: invalid volume id specified; salvage aborted\n");
 626         return 1;
 627     }
 628
 629     partP = VGetPartition(node->command.sop.partName, 0);
 630     if (!partP) {
 631         Log("salvageServer: Unknown or unmounted partition %s; salvage aborted\n",
 632             node->command.sop.partName);
 633         return 1;
 634     }
 635
 636     /* obtain a shared salvage lock in the child worker, so if the
 637      * salvageserver restarts (and we continue), we will still hold a lock and
 638      * prevent standalone salvagers from interfering */
 639     ObtainSharedSalvageLock();
 640
 641     /* Salvage individual volume; don't notify fs */
 642     SalvageFileSys1(partP, node->command.sop.parent);
 643
 644     fclose(logFile);
 645     return 0;
 646 }
 647
 648
 649 static void *
 650 SalvageChildReaperThread(void * args)
 651 {
 652     int slot, pid, status;
 653     struct log_cleanup_node * cleanup;
 654
 655     MUTEX_ENTER(&worker_lock);
 656
 657     /* loop reaping our children */
 658     while (1) {
 659         /* wait() won't block unless we have children, so
 660          * block on the cond var if we're childless */
 661         while (current_workers == 0) {
 662             CV_WAIT(&worker_cv, &worker_lock);
 663         }
 664
 665         MUTEX_EXIT(&worker_lock);
 666
 667         cleanup = (struct log_cleanup_node *) malloc(sizeof(struct log_cleanup_node));
 668
 669         while (Reap_Child("salvageserver", &pid, &status) < 0) {
 670             /* try to prevent livelock if something goes wrong */
 671             sleep(1);
 672         }
 673
 674         VOL_LOCK;
 675         for (slot = 0; slot < Parallel; slot++) {
 676             if (child_slot[slot] == pid)
 677                 break;
 678         }
 679         osi_Assert(slot < Parallel);
 680         child_slot[slot] = 0;
 681         VOL_UNLOCK;
 682
 683         SALVSYNC_doneWorkByPid(pid, status);
 684
 685         MUTEX_ENTER(&worker_lock);
 686
 687         if (cleanup) {
 688             cleanup->pid = pid;
 689             queue_Append(&log_cleanup_queue, cleanup);
 690             CV_SIGNAL(&log_cleanup_queue.queue_change_cv);
 691         }
 692
 693         /* ok, we've reaped a child */
 694         current_workers--;
 695         CV_BROADCAST(&worker_cv);
 696     }
 697
 698     return NULL;
 699 }
 700
 701 static int
 702 Reap_Child(char *prog, int * pid, int * status)
 703 {
 704     int ret;
 705     ret = wait(status);
 706
 707     if (ret >= 0) {
 708         *pid = ret;
 709         if (WCOREDUMP(*status))
 710             Log("\"%s\" core dumped!\n", prog);
 711         if ((WIFSIGNALED(*status) != 0) ||
 712             ((WEXITSTATUS(*status) != 0) &&
 713              (WEXITSTATUS(*status) != SALSRV_EXIT_VOLGROUP_LINK)))
 714             Log("\"%s\" (pid=%d) terminated abnormally!\n", prog, ret);
 715     } else {
 716         Log("wait returned -1\n");
 717     }
 718     return ret;
 719 }
 720
 721 /*
 722  * thread to combine salvager child logs
 723  * back into the main salvageserver log
 724  */
 725 static void *
 726 SalvageLogCleanupThread(void * arg)
 727 {
 728     struct log_cleanup_node * cleanup;
 729
 730     MUTEX_ENTER(&worker_lock);
 731
 732     while (1) {
 733         while (queue_IsEmpty(&log_cleanup_queue)) {
 734             CV_WAIT(&log_cleanup_queue.queue_change_cv, &worker_lock);
 735         }
 736
 737         while (queue_IsNotEmpty(&log_cleanup_queue)) {
 738             cleanup = queue_First(&log_cleanup_queue, log_cleanup_node);
 739             queue_Remove(cleanup);
 740             MUTEX_EXIT(&worker_lock);
 741             SalvageLogCleanup(cleanup->pid);
 742             free(cleanup);
 743             MUTEX_ENTER(&worker_lock);
 744         }
 745     }
 746
 747     MUTEX_EXIT(&worker_lock);
 748     return NULL;
 749 }
 750
 751 #define LOG_XFER_BUF_SIZE 65536
 752 static int
 753 SalvageLogCleanup(int pid)
 754 {
 755     int pidlog, len;
 756     char fn[AFSDIR_PATH_MAX];
 757     static char buf[LOG_XFER_BUF_SIZE];
 758
 759     afs_snprintf(fn, sizeof(fn), "%s.%d",
 760                  AFSDIR_SERVER_SLVGLOG_FILEPATH, pid);
 761
 762
 763     pidlog = open(fn, O_RDONLY);
 764     unlink(fn);
 765     if (pidlog < 0)
 766         return 1;
 767
 768     len = read(pidlog, buf, LOG_XFER_BUF_SIZE);
 769     while (len) {
 770         fwrite(buf, len, 1, logFile);
 771         len = read(pidlog, buf, LOG_XFER_BUF_SIZE);
 772     }
 773
 774     close(pidlog);
 775
 776     return 0;
 777 }
 778
 779 /* wake up every five minutes to see if a non-child salvage has finished */
 780 #define SALVAGE_SCAN_POLL_INTERVAL 300
 781
 782 /**
 783  * Thread to look for SalvageLog.$pid files that are not from our child
 784  * worker salvagers, and notify SalvageLogCleanupThread to clean them
 785  * up. This can happen if we restart during salvages, or the
 786  * salvageserver crashes or something.
 787  *
 788  * @param arg  unused
 789  *
 790  * @return always NULL
 791  */
 792 static void *
 793 SalvageLogScanningThread(void * arg)
 794 {
 795     struct rx_queue log_watch_queue;
 796
 797     queue_Init(&log_watch_queue);
 798
 799     {
 800         DIR *dp;
 801         struct dirent *dirp;
 802         char prefix[AFSDIR_PATH_MAX];
 803         size_t prefix_len;
 804
 805         afs_snprintf(prefix, sizeof(prefix), "%s.", AFSDIR_SLVGLOG_FILE);
 806         prefix_len = strlen(prefix);
 807
 808         dp = opendir(AFSDIR_LOGS_DIR);
 809         osi_Assert(dp);
 810
 811         while ((dirp = readdir(dp)) != NULL) {
 812             pid_t pid;
 813             struct log_cleanup_node *cleanup;
 814             int i;
 815
 816             if (strncmp(dirp->d_name, prefix, prefix_len) != 0) {
 817                 /* not a salvage logfile; skip */
 818                 continue;
 819             }
 820
 821             errno = 0;
 822             pid = strtol(dirp->d_name + prefix_len, NULL, 10);
 823
 824             if (errno != 0) {
 825                 /* file is SalvageLog.<something> but <something> isn't
 826                  * a pid, so skip */
 827                  continue;
 828             }
 829
 830             VOL_LOCK;
 831             for (i = 0; i < Parallel; ++i) {
 832                 if (pid == child_slot[i]) {
 833                     break;
 834                 }
 835             }
 836             VOL_UNLOCK;
 837             if (i < Parallel) {
 838                 /* this pid is one of our children, so the reaper thread
 839                  * will take care of it; skip */
 840                 continue;
 841             }
 842
 843             cleanup =
 844                 (struct log_cleanup_node *) malloc(sizeof(struct log_cleanup_node));
 845             cleanup->pid = pid;
 846
 847             queue_Append(&log_watch_queue, cleanup);
 848         }
 849
 850         closedir(dp);
 851     }
 852
 853     ScanLogs(&log_watch_queue);
 854
 855     while (queue_IsNotEmpty(&log_watch_queue)) {
 856         sleep(SALVAGE_SCAN_POLL_INTERVAL);
 857         ScanLogs(&log_watch_queue);
 858     }
 859
 860     return NULL;
 861 }
 862
 863 /**
 864  * look through log_watch_queue, and if any processes are not still
 865  * running, hand them off to the SalvageLogCleanupThread
 866  *
 867  * @param log_watch_queue  a queue of PIDs that we should clean up if
 868  * that PID has died
 869  */
 870 static void
 871 ScanLogs(struct rx_queue *log_watch_queue)
 872 {
 873     struct log_cleanup_node *cleanup, *next;
 874
 875     MUTEX_ENTER(&worker_lock);
 876
 877     for (queue_Scan(log_watch_queue, cleanup, next, log_cleanup_node)) {
 878         /* if a process is still running, assume it's the salvage process
 879          * still going, and keep waiting for it */
 880         if (kill(cleanup->pid, 0) < 0 && errno == ESRCH) {
 881             queue_Remove(cleanup);
 882             queue_Append(&log_cleanup_queue, cleanup);
 883             CV_SIGNAL(&log_cleanup_queue.queue_change_cv);
 884         }
 885     }
 886
 887     MUTEX_EXIT(&worker_lock);
 888 }