dm-merge.c

   1 /******************************************************************************
   2  * dm-merge.c
   3  *
   4  * Written by hondza. As far as I'm concerned public domain. Several (GPL'd)
   5  * lines (PAGE_ALIGN stuff...) taken from linux kernel source.
   6  *
   7  * Do whatever you wish with it, but don't blame me.
   8  *
   9  *****************************************************************************/
  10
  11 #define _FILE_OFFSET_BITS 64
  12 #define _GNU_SOURCE
  13
  14 #include <stdio.h>
  15 #include <string.h>
  16 #include <stdlib.h>
  17
  18 #include <stddef.h> /* ptrdiff_t */
  19
  20 #include <stdint.h> /* uint64_t */
  21
  22 #include <errno.h>
  23
  24 #include <getopt.h>
  25
  26 #include <unistd.h>
  27 #include <sys/types.h>
  28 #include <sys/stat.h>
  29 #include <fcntl.h>
  30
  31 #include <sys/ioctl.h>
  32 #include <linux/fs.h> /* BLKGETSIZE64, BLKFLSBUF */
  33
  34 #include <endian.h>
  35
  36
  37 /* endian stuff */
  38 #if __BYTE_ORDER == __BIG_ENDIAN
  39 #include <linux/byteorder/big_endian.h>
  40 #else
  41 #include <linux/byteorder/little_endian.h>
  42 #endif
  43
  44
  45 #define SNAP_MAGIC 0x70416e53 /* 'SnAp' */
  46 #define SNAPSHOT_DISK_VERSION 1
  47
  48
  49 /* doubtful stuff; only used with O_DIRECT */
  50 #define MAX_CHUNKSIZE 524288
  51 #define PAGE_SIZE 4096
  52 #define PAGE_ALIGN(addr) ALIGN(addr, PAGE_SIZE)
  53 #define ALIGN(x,a)      __ALIGN_MASK(x,(typeof(x))(a)-1)
  54 #define __ALIGN_MASK(x,mask)    (((x)+(mask))&~(mask))
  55
  56
  57 uint32_t *magic;
  58 uint32_t *valid;
  59 uint32_t *version;
  60 uint32_t chunk_size=512, *cs;
  61
  62
  63 struct disk_exception {
  64     uint64_t old_chunk;
  65     uint64_t new_chunk;
  66 };
  67
  68
  69 unsigned char *header=NULL, *header_orig=NULL, *buf=NULL, *buf_orig=NULL, *chunk=NULL, *chunk_orig=NULL;
  70 char *input_filename=NULL, *output_filename=NULL;
  71 int inputfd=-1, outputfd=-1, dry=1, print_dds=0, do_direct_io = 0, verbose = 0, err=-1, err2=-1, flags=0, go=1, relaxed=0;
  72 unsigned int i=0, retries=0;
  73 struct stat st;
  74 uint64_t input_length=0, output_length=0;
  75 off_t chunk_now=1;
  76 struct disk_exception de;
  77 uint64_t *temp_u64;
  78 uint64_t total_count=0;
  79
  80
  81 /* taken from libtomcrypt */
  82 void burn_stack(unsigned long len)
  83 {
  84    unsigned char buf[32];
  85    memset(buf, 0, sizeof(buf));
  86    if (len > (unsigned long)sizeof(buf))
  87       burn_stack(len - sizeof(buf));
  88 }
  89
  90
  91 /* atexit calls it */
  92 void cleanup()
  93 {
  94     if(input_filename) free(input_filename);
  95     if(output_filename) free(output_filename);
  96     if(buf) memset(buf, 0, chunk_size);
  97     if(buf_orig) free(buf_orig);
  98     if(chunk) memset(chunk, 0, chunk_size);
  99     if(chunk_orig) free(chunk_orig);
 100     if(header) memset(header, 0, 512);
 101     if(header_orig) free(header_orig);
 102     /* obfuscate it a little */
 103     burn_stack(10*4096);
 104 } /* cleanup */
 105
 106
 107 /* called for each exception */
 108 inline void read_write_chunk()
 109 {
 110     retries=0;
 111
 112     if(print_dds) fprintf(stdout, "dd of=\"%s\" seek=%llu if='%s' iflag=direct skip=%llu count=1 bs=%db\n", output_filename ? output_filename : "${origin}", de.old_chunk, input_filename, de.new_chunk, chunk_size/512);
 113
 114     if(dry) return;
 115
 116     /* read */
 117     do {
 118         if(0 != retries)
 119             fprintf(stderr, "Warning: retrying pread() on exception chunk at %llu\n", de.new_chunk*chunk_size);
 120
 121         err = pread(inputfd, chunk, chunk_size, de.new_chunk*chunk_size);
 122         if(-1 == err)
 123         {
 124             if(EINTR == errno)
 125                 continue;
 126             perror("pread(inputfd)");
 127             exit(1);
 128         }
 129         else if(0 == err)
 130         {
 131             fputs("pread(inputfd): early EOF!\n", stderr);
 132             exit(1);
 133         }
 134         else if(err != chunk_size)
 135         {
 136             if(retries++ < 2)
 137                 continue;
 138             fputs("pread(inputfd): incomplete read!\n", stderr);
 139             exit(1);
 140         }
 141
 142         break;
 143     } while(1);
 144
 145
 146     /* write */
 147     retries = 0;
 148     do {
 149         if(0 != retries)
 150             fprintf(stderr, "Warning: retrying pwrite() at %llu\n", de.old_chunk*chunk_size);
 151
 152         err = pwrite(outputfd, chunk, chunk_size, de.old_chunk*chunk_size);
 153         if(-1 == err)
 154         {
 155             if(EINTR == errno)
 156                 continue;
 157             perror("pwrite(outputfd)");
 158             exit(1);
 159         }
 160         else if(err != chunk_size)
 161         {
 162             if(retries++ < 2)
 163                 continue;
 164             fputs("pwrite(outputfd): incomplete write!\n", stderr);
 165             exit(1);
 166         }
 167
 168         break;
 169     } while(1);
 170
 171 } /* read_write_chunk() */
 172
 173
 174
 175 void help()
 176 {
 177     fputs("Usage: dm-merge [options] -i <snapshot_device> [ -o <output_device> ]\n\n", stderr);
 178     fputs("Options:\n", stderr);
 179     fputs("-f\t\tREALLY do it (no dry run)\n", stderr);
 180     fputs("-x\t\tRelaxed mode; allow some things (namely ioctl) to fail\n", stderr);
 181     fputs("-d\t\tPrint dd lines as list_exception_chunks.pl would\n", stderr);
 182     fputs("-D\t\tTry to use O_DIRECT\n", stderr);
 183     fputs("-v\t\tBe verbose (more '-v's increase verbosity)\n", stderr);
 184     fputs("-i <file>\tInput (snapshot) COW (copy-on-write) filename\n", stderr);
 185     fputs("-o <file>\tOutput (the device being snapshoted) filename\n\n", stderr);
 186     fputs("This program is still experimental. USE WITH CARE! Read the README!\n\n", stderr);
 187 } /* help() */
 188
 189
 190
 191
 192 int main(int argc, char ** argv)
 193 {
 194     int c;
 195
 196     atexit(cleanup);
 197
 198     while( (c = getopt(argc, argv, "fdi:o:Dvhx")) != -1 )
 199     {
 200         switch(c)
 201         {
 202             case 'x': relaxed = 1; break;
 203             case 'f': dry = 0; break;
 204             case 'd': print_dds = 1; break;
 205             case 'D': do_direct_io = 1; break;
 206             case 'v': verbose++; break;
 207             case 'i': input_filename = strdup(optarg); break;
 208             case 'o': output_filename = strdup(optarg); break;
 209             case 'h': help(); exit(0);
 210             default: help(); exit(1);
 211         } /* switch argument */
 212     } /* while getopt() */
 213
 214
 215     if(!input_filename)
 216     {
 217         fputs("Error: input filename not specified\n\n", stderr);
 218         help();
 219         exit(1);
 220     }
 221
 222     if(!dry && !output_filename)
 223     {
 224         fputs("Error: no dry run and no output filename\n\n", stderr);
 225         help();
 226         exit(1);
 227     }
 228
 229
 230     /* better safe than sorry */
 231     sync();
 232
 233     /* check and open snapshot */
 234
 235     flags = O_RDONLY;
 236
 237     err = stat(input_filename, &st);
 238     if(-1 == err)
 239     {
 240         perror("stat(snapshot)");
 241         exit(1);
 242     }
 243
 244     /* block device; will set O_DIRECT */
 245     if(S_ISBLK(st.st_mode) && do_direct_io)
 246         flags |= O_DIRECT;
 247
 248     inputfd = open(input_filename, flags);
 249     if(-1 == inputfd)
 250     {
 251         perror("open(snapshot)");
 252         exit(1);
 253     }
 254
 255     /* determine size & flush buffers */
 256     if(S_ISBLK(st.st_mode))
 257     {
 258         err = ioctl(inputfd, BLKGETSIZE64, &input_length);
 259         if(-1 == err)
 260         {
 261             perror("ioctl(snapshot, BLKGETSIZE64)");
 262             exit(1);
 263         }
 264
 265         err = ioctl(inputfd, BLKFLSBUF, 0);
 266         if(-1 == err)
 267         {
 268             perror("ioctl(snapshot, BLKFLSBUF)");
 269             if(!relaxed) exit(1);
 270             else fputs("relaxed mode, will continue ...\n", stderr);
 271         }
 272     }
 273     else
 274         input_length = st.st_size;
 275
 276     /* now the same for the output */
 277     if(!dry)
 278     {
 279         flags = O_WRONLY;
 280
 281         err = stat(output_filename, &st);
 282         if(-1 == err)
 283         {
 284             perror("stat(output)");
 285             exit(1);
 286         }
 287
 288         /* block device; will set O_DIRECT */
 289         if(S_ISBLK(st.st_mode) && do_direct_io)
 290             flags |= O_DIRECT;
 291
 292         outputfd = open(output_filename, flags);
 293         if(-1 == outputfd)
 294         {
 295             perror("open(output)");
 296             exit(1);
 297         }
 298
 299         /* determine size & flush buffers */
 300         if(S_ISBLK(st.st_mode))
 301         {
 302             err = ioctl(outputfd, BLKGETSIZE64, &output_length);
 303             if(-1 == err)
 304             {
 305                 perror("ioctl(output, BLKGETSIZE64)");
 306                 exit(1);
 307             }
 308
 309             err = ioctl(outputfd, BLKFLSBUF, 0);
 310             if(-1 == err)
 311             {
 312                 perror("ioctl(output, BLKFLSBUF)");
 313                 if(!relaxed) exit(1);
 314                 else fputs("relaxed mode, will continue ...\n", stderr);
 315             }
 316         }
 317         else
 318             output_length = st.st_size;
 319     }
 320
 321
 322     /* FIXME perhaps add an override option? */
 323     if(input_length < 4096 || (!dry && output_length < (4 * 1024 * 1024)))
 324     {
 325         fputs("Error: suspicious file/device sizes\n", stderr);
 326         exit(1);
 327     }
 328
 329
 330     /* the allocations; special care for O_DIRECT */
 331     if(do_direct_io)
 332     {
 333         header_orig = malloc(512 + PAGE_SIZE);
 334         if(!header_orig)
 335         {
 336             perror("malloc()");
 337             exit(1);
 338         }
 339         header = (unsigned char *) PAGE_ALIGN((ptrdiff_t) header_orig);
 340         if(verbose)
 341             fprintf(stdout, "header_orig = %p (%lu), header = %p (%lu)\n", header_orig, (unsigned long) header_orig % PAGE_SIZE, header, (unsigned long) header % PAGE_SIZE);
 342     }
 343     else
 344     {
 345         header_orig = header = malloc(512);
 346         if(!header_orig)
 347         {
 348             perror("malloc()");
 349             exit(1);
 350         }
 351     }
 352
 353     /* Not sure if BLKFLSBUF waits for the flushing to finish; better safe than sorry */
 354     fputs("Artificial sleep (1 second)\n", stdout);
 355     sleep(1);
 356
 357
 358     /* FIXME: do retries here as well? */
 359     err = pread(inputfd, header, 512, 0);
 360     if(-1 == err)
 361     {
 362         perror("read(snapshot, header)");
 363         exit(1);
 364     }
 365
 366     magic = (uint32_t *) header;
 367     *magic = __le32_to_cpu(*magic);
 368     if(SNAP_MAGIC != *magic)
 369     {
 370         fputs("Invalid header MAGIC\n", stderr);
 371         fprintf(stderr, "%#x != %#x\n", *magic, SNAP_MAGIC);
 372         exit(1);
 373     }
 374     fprintf(stdout, "Found a proper MAGIC header: %#x\n", *magic);
 375
 376     valid = (uint32_t *) (header+4);
 377     *valid = __le32_to_cpu(*valid);
 378     if(0 == *valid)
 379     {
 380         fputs("valid == 0\n", stderr);
 381         exit(1);
 382     }
 383     fprintf(stdout, "valid = %u\n", *valid);
 384
 385     version = (uint32_t *) (header+8);
 386     *version = __le32_to_cpu(*version);
 387     if(*version != SNAPSHOT_DISK_VERSION)
 388     {
 389         fputs("version != 1\n", stderr);
 390         exit(1);
 391     }
 392     fprintf(stdout, "version = %u\n", *version);
 393
 394     cs = (uint32_t *) (header+12);
 395     *cs =  __le32_to_cpu(*cs);
 396     chunk_size = *cs;
 397     if(chunk_size < 1 || chunk_size > 1024 || (0 != (chunk_size & (chunk_size-1))))
 398     {
 399         fputs("chunk size has to be >=1 and <=1024 and a power of 2\n", stderr);
 400         exit(1);
 401     }
 402     fprintf(stdout, "chunk_size = %u (%u bytes)\n", chunk_size, chunk_size*512);
 403
 404     chunk_size *= 512;
 405
 406     /* the allocations; special care for O_DIRECT */
 407     if(do_direct_io)
 408     {
 409         buf_orig = buf = malloc(chunk_size + PAGE_SIZE);
 410         chunk_orig = chunk = malloc(chunk_size + PAGE_SIZE);
 411         if(!buf_orig || !chunk_orig)
 412         {
 413             perror("malloc()");
 414             exit(1);
 415         }
 416         buf = (unsigned char *) PAGE_ALIGN((ptrdiff_t) buf_orig);
 417         if(verbose)
 418             fprintf(stdout, "buf_orig = %p (%lu), buf = %p (%lu)\n", buf_orig, (unsigned long) buf_orig % PAGE_SIZE, buf, (unsigned long) buf % PAGE_SIZE);
 419         chunk = (unsigned char *) PAGE_ALIGN((ptrdiff_t) chunk_orig);
 420         if(verbose)
 421             fprintf(stdout, "chunk_orig = %p (%lu), chunk = %p (%lu)\n", chunk_orig, (unsigned long) chunk_orig % PAGE_SIZE, chunk, (unsigned long) chunk % PAGE_SIZE);
 422     }
 423     else
 424     {
 425         buf_orig = buf = malloc(chunk_size);
 426         chunk_orig = chunk = malloc(chunk_size);
 427         if(!buf_orig || !chunk_orig)
 428         {
 429             perror("malloc()");
 430             exit(1);
 431         }
 432     }
 433
 434
 435     /*
 436      * do the work
 437      */
 438     do
 439     {
 440         retries=0;
 441
 442         do {
 443             if(0 != retries)
 444                 fprintf(stderr, "Warning: retrying pread() on exception area %llu at %llu\n", chunk_now, chunk_now*chunk_size);
 445
 446             err = pread(inputfd, buf, chunk_size, chunk_now*chunk_size);
 447             if(-1 == err)
 448             {
 449                 if(EINTR == errno)
 450                     continue;
 451                 perror("pread(inputfd)");
 452                 exit(1);
 453             }
 454             else if(0 == err)
 455             {
 456                 fputs("pread(inputfd): early EOF!\n", stderr);
 457                 exit(1);
 458             }
 459             else if(err != chunk_size)
 460             {
 461                 if(retries++ < 2)
 462                     continue;
 463                 fputs("pread(inputfd): incomplete read!\n", stderr);
 464                 exit(1);
 465             }
 466
 467             break;
 468         } while(1);
 469
 470         /* process the exception area */
 471         for(i=0; i < chunk_size/16; i++)
 472         {
 473             temp_u64 = (uint64_t *)(buf+(i*16));
 474             de.old_chunk = __le64_to_cpu(*temp_u64);
 475             temp_u64 = (uint64_t *)(buf+(i*16)+8);
 476             de.new_chunk = __le64_to_cpu(*temp_u64);
 477
 478             if(verbose >= 2)
 479                 fprintf(stdout, "... chunk_now = %llu, i = %u, de.old_chunk = %llu, de.new_chunk = %llu, old %p, new %p\n", chunk_now, i, de.old_chunk, de.new_chunk, buf+(i*16), buf+(i*16)+8);
 480
 481             /* 0 as a new chunk means "we've reached the end" */
 482             if(0 == de.new_chunk)
 483             {
 484                 go = 0;
 485                 break;
 486             }
 487             else if((1 == chunk_now && 0 == i && de.new_chunk != 2) || (de.new_chunk < 2))
 488             {
 489                 fputs("(1 == chunk_now && 0 == i && de.new_chunk != 2) || (de.new_chunk < 2), perhaps not a snapshot?\n", stderr);
 490                 exit(1);
 491             }
 492
 493             total_count++;
 494
 495             /* the data transfer */
 496             read_write_chunk();
 497         } /* for i */
 498
 499         /* next hop */
 500         chunk_now += chunk_size/16 + 1;
 501         if(verbose && go)
 502             fprintf(stdout, "Seeking into exception area in chunk %llu\n", chunk_now);
 503
 504     } while(go);
 505
 506     /* flush buffers again (no error handling this time as there's nothing to do anyway) */
 507     if(S_ISBLK(st.st_mode)) err = ioctl(outputfd, BLKFLSBUF, 0);
 508
 509     /* better safe than sorry */
 510     sync();
 511
 512     fprintf(stdout, "Found %llu exceptions of chunksize %u, total size %llu bytes (%llu KiB, %.3Lf MiB, %.3Lf GiB).\n", total_count, chunk_size, total_count*chunk_size, (total_count*chunk_size)/1024, ((long double)(total_count*chunk_size))/(1024*1024), ((long double)(total_count*chunk_size))/(1024*1024*1024));
 513
 514     close(inputfd);
 515     if(-1 != outputfd) close(outputfd);
 516
 517     memset(buf, 0, chunk_size);
 518     memset(chunk, 0, chunk_size);
 519
 520     /* cleanup() will do the rest */
 521
 522     return 0;
 523 } /* main() */
 524