src/apps/installer/CopyEngine.cpp

   1 /*
   2  * Copyright 2008-2009, Stephan Aßmus <superstippi@gmx.de>
   3  * Copyright 2014, Axel Dörfler, axeld@pinc-software.de
   4  * All rights reserved. Distributed under the terms of the MIT License.
   5  */
   6
   7
   8 #include "CopyEngine.h"
   9
  10 #include <new>
  11
  12 #include <math.h>
  13 #include <stdio.h>
  14 #include <string.h>
  15 #include <sys/resource.h>
  16
  17 #include <Directory.h>
  18 #include <fs_attr.h>
  19 #include <NodeInfo.h>
  20 #include <Path.h>
  21 #include <SymLink.h>
  22
  23 #include "SemaphoreLocker.h"
  24 #include "ProgressReporter.h"
  25
  26
  27 using std::nothrow;
  28
  29
  30 // #pragma mark - EntryFilter
  31
  32
  33 CopyEngine::EntryFilter::~EntryFilter()
  34 {
  35 }
  36
  37
  38 // #pragma mark - CopyEngine
  39
  40
  41 CopyEngine::CopyEngine(ProgressReporter* reporter, EntryFilter* entryFilter)
  42         :
  43         fBufferQueue(),
  44         fWriterThread(-1),
  45         fQuitting(false),
  46
  47         fAbsoluteSourcePath(),
  48
  49         fBytesRead(0),
  50         fLastBytesRead(0),
  51         fItemsCopied(0),
  52         fLastItemsCopied(0),
  53         fTimeRead(0),
  54
  55         fBytesWritten(0),
  56         fTimeWritten(0),
  57
  58         fBytesToCopy(0),
  59         fItemsToCopy(0),
  60
  61         fCurrentTargetFolder(NULL),
  62         fCurrentItem(NULL),
  63
  64         fProgressReporter(reporter),
  65         fEntryFilter(entryFilter)
  66 {
  67         fWriterThread = spawn_thread(_WriteThreadEntry, "buffer writer",
  68                 B_NORMAL_PRIORITY, this);
  69
  70         if (fWriterThread >= B_OK)
  71                 resume_thread(fWriterThread);
  72
  73         // ask for a bunch more file descriptors so that nested copying works well
  74         struct rlimit rl;
  75         rl.rlim_cur = 512;
  76         rl.rlim_max = RLIM_SAVED_MAX;
  77         setrlimit(RLIMIT_NOFILE, &rl);
  78 }
  79
  80
  81 CopyEngine::~CopyEngine()
  82 {
  83         while (fBufferQueue.Size() > 0)
  84                 snooze(10000);
  85
  86         fQuitting = true;
  87         if (fWriterThread >= B_OK) {
  88                 int32 exitValue;
  89                 wait_for_thread(fWriterThread, &exitValue);
  90         }
  91 }
  92
  93
  94 void
  95 CopyEngine::ResetTargets(const char* source)
  96 {
  97         // TODO: One could subtract the bytes/items which were added to the
  98         // ProgressReporter before resetting them...
  99
 100         fAbsoluteSourcePath = source;
 101
 102         fBytesRead = 0;
 103         fLastBytesRead = 0;
 104         fItemsCopied = 0;
 105         fLastItemsCopied = 0;
 106         fTimeRead = 0;
 107
 108         fBytesWritten = 0;
 109         fTimeWritten = 0;
 110
 111         fBytesToCopy = 0;
 112         fItemsToCopy = 0;
 113
 114         fCurrentTargetFolder = NULL;
 115         fCurrentItem = NULL;
 116 }
 117
 118
 119 status_t
 120 CopyEngine::CollectTargets(const char* source, sem_id cancelSemaphore)
 121 {
 122         int32 level = 0;
 123         status_t ret = _CollectCopyInfo(source, level, cancelSemaphore);
 124         if (ret == B_OK && fProgressReporter != NULL)
 125                 fProgressReporter->AddItems(fItemsToCopy, fBytesToCopy);
 126         return ret;
 127 }
 128
 129
 130 status_t
 131 CopyEngine::CopyFolder(const char* source, const char* destination,
 132         sem_id cancelSemaphore)
 133 {
 134         int32 level = 0;
 135         return _CopyFolder(source, destination, level, cancelSemaphore);
 136 }
 137
 138
 139 status_t
 140 CopyEngine::CopyFile(const BEntry& _source, const BEntry& _destination,
 141         sem_id cancelSemaphore)
 142 {
 143         SemaphoreLocker lock(cancelSemaphore);
 144         if (cancelSemaphore >= 0 && !lock.IsLocked()) {
 145                 // We are supposed to quit
 146                 return B_CANCELED;
 147         }
 148
 149         BFile source(&_source, B_READ_ONLY);
 150         status_t ret = source.InitCheck();
 151         if (ret < B_OK)
 152                 return ret;
 153
 154         BFile* destination = new (nothrow) BFile(&_destination,
 155                 B_WRITE_ONLY | B_CREATE_FILE | B_ERASE_FILE);
 156         ret = destination->InitCheck();
 157         if (ret < B_OK) {
 158                 delete destination;
 159                 return ret;
 160         }
 161
 162         int32 loopIteration = 0;
 163
 164         while (true) {
 165                 if (fBufferQueue.Size() >= BUFFER_COUNT) {
 166                         // the queue is "full", just wait a bit, the
 167                         // write thread will empty it
 168                         snooze(1000);
 169                         continue;
 170                 }
 171
 172                 // allocate buffer
 173                 Buffer* buffer = new (nothrow) Buffer(destination);
 174                 if (!buffer || !buffer->buffer) {
 175                         delete destination;
 176                         delete buffer;
 177                         fprintf(stderr, "reading loop: out of memory\n");
 178                         return B_NO_MEMORY;
 179                 }
 180
 181                 // fill buffer
 182                 ssize_t read = source.Read(buffer->buffer, buffer->size);
 183                 if (read < 0) {
 184                         ret = (status_t)read;
 185                         fprintf(stderr, "Failed to read data: %s\n", strerror(ret));
 186                         delete buffer;
 187                         delete destination;
 188                         break;
 189                 }
 190
 191                 fBytesRead += read;
 192                 loopIteration += 1;
 193                 if (loopIteration % 2 == 0)
 194                         _UpdateProgress();
 195
 196                 buffer->deleteFile = read == 0;
 197                 if (read > 0)
 198                         buffer->validBytes = (size_t)read;
 199                 else
 200                         buffer->validBytes = 0;
 201
 202                 // enqueue the buffer
 203                 ret = fBufferQueue.Push(buffer);
 204                 if (ret < B_OK) {
 205                         buffer->deleteFile = false;
 206                         delete buffer;
 207                         delete destination;
 208                         return ret;
 209                 }
 210
 211                 // quit if done
 212                 if (read == 0)
 213                         break;
 214         }
 215
 216         return ret;
 217 }
 218
 219
 220 // #pragma mark -
 221
 222
 223 status_t
 224 CopyEngine::_CollectCopyInfo(const char* _source, int32& level,
 225         sem_id cancelSemaphore)
 226 {
 227         level++;
 228
 229         BDirectory source(_source);
 230         status_t ret = source.InitCheck();
 231         if (ret < B_OK)
 232                 return ret;
 233
 234         BEntry entry;
 235         while (source.GetNextEntry(&entry) == B_OK) {
 236                 SemaphoreLocker lock(cancelSemaphore);
 237                 if (cancelSemaphore >= 0 && !lock.IsLocked()) {
 238                         // We are supposed to quit
 239                         return B_CANCELED;
 240                 }
 241
 242                 struct stat statInfo;
 243                 entry.GetStat(&statInfo);
 244
 245                 BPath sourceEntryPath;
 246                 status_t ret = entry.GetPath(&sourceEntryPath);
 247                 if (ret < B_OK)
 248                         return ret;
 249
 250                 if (fEntryFilter != NULL
 251                         && !fEntryFilter->ShouldCopyEntry(entry,
 252                                 _RelativeEntryPath(sourceEntryPath.Path()), statInfo, level)) {
 253                         continue;
 254                 }
 255
 256                 if (S_ISDIR(statInfo.st_mode)) {
 257                         // handle recursive directory copy
 258                         BPath srcFolder;
 259                         ret = entry.GetPath(&srcFolder);
 260                         if (ret < B_OK)
 261                                 return ret;
 262
 263                         if (cancelSemaphore >= 0)
 264                                 lock.Unlock();
 265
 266                         ret = _CollectCopyInfo(srcFolder.Path(), level, cancelSemaphore);
 267                         if (ret < B_OK)
 268                                 return ret;
 269                 } else if (S_ISLNK(statInfo.st_mode)) {
 270                         // link, ignore size
 271                 } else {
 272                         // file data
 273                         fBytesToCopy += statInfo.st_size;
 274                 }
 275
 276                 fItemsToCopy++;
 277         }
 278
 279         level--;
 280         return B_OK;
 281 }
 282
 283
 284 status_t
 285 CopyEngine::_CopyFolder(const char* _source, const char* _destination,
 286         int32& level, sem_id cancelSemaphore)
 287 {
 288         level++;
 289         fCurrentTargetFolder = _destination;
 290
 291         BDirectory source(_source);
 292         status_t ret = source.InitCheck();
 293         if (ret < B_OK)
 294                 return ret;
 295
 296         ret = create_directory(_destination, 0777);
 297         if (ret < B_OK && ret != B_FILE_EXISTS) {
 298                 fprintf(stderr, "Could not create '%s': %s\n", _destination,
 299                         strerror(ret));
 300                 return ret;
 301         }
 302
 303         BDirectory destination(_destination);
 304         ret = destination.InitCheck();
 305         if (ret < B_OK)
 306                 return ret;
 307
 308         BEntry entry;
 309         while (source.GetNextEntry(&entry) == B_OK) {
 310                 SemaphoreLocker lock(cancelSemaphore);
 311                 if (cancelSemaphore >= 0 && !lock.IsLocked()) {
 312                         // We are supposed to quit
 313                         return B_CANCELED;
 314                 }
 315
 316                 const char* name = entry.Name();
 317                 BPath sourceEntryPath;
 318                 status_t ret = entry.GetPath(&sourceEntryPath);
 319                 if (ret != B_OK)
 320                         return ret;
 321                 const char* relativeSourceEntryPath
 322                         = _RelativeEntryPath(sourceEntryPath.Path());
 323
 324                 struct stat statInfo;
 325                 entry.GetStat(&statInfo);
 326
 327                 if (fEntryFilter != NULL
 328                         && !fEntryFilter->ShouldCopyEntry(entry, relativeSourceEntryPath,
 329                                 statInfo, level)) {
 330                         continue;
 331                 }
 332
 333                 fItemsCopied++;
 334                 fCurrentItem = name;
 335                 _UpdateProgress();
 336
 337                 BEntry copy(&destination, name);
 338                 bool copyAttributes = true;
 339
 340                 if (S_ISDIR(statInfo.st_mode)) {
 341                         // handle recursive directory copy
 342
 343                         if (copy.Exists()) {
 344                                 ret = B_OK;
 345                                 if (copy.IsDirectory()) {
 346                                         if (fEntryFilter
 347                                                 && fEntryFilter->ShouldClobberFolder(entry,
 348                                                         relativeSourceEntryPath, statInfo, level)) {
 349                                                 ret = _RemoveFolder(copy);
 350                                         } else {
 351                                                 // Do not overwrite attributes on folders that exist.
 352                                                 // This should work better when the install target
 353                                                 // already contains a Haiku installation.
 354                                                 copyAttributes = false;
 355                                         }
 356                                 } else
 357                                         ret = copy.Remove();
 358
 359                                 if (ret != B_OK) {
 360                                         fprintf(stderr, "Failed to make room for folder '%s': "
 361                                                 "%s\n", name, strerror(ret));
 362                                         return ret;
 363                                 }
 364                         }
 365
 366                         BPath dstFolder;
 367                         ret = copy.GetPath(&dstFolder);
 368                         if (ret < B_OK)
 369                                 return ret;
 370
 371                         if (cancelSemaphore >= 0)
 372                                 lock.Unlock();
 373
 374                         ret = _CopyFolder(sourceEntryPath.Path(), dstFolder.Path(), level,
 375                                 cancelSemaphore);
 376                         if (ret < B_OK)
 377                                 return ret;
 378
 379                         if (cancelSemaphore >= 0 && !lock.Lock()) {
 380                                 // We are supposed to quit
 381                                 return B_CANCELED;
 382                         }
 383                 } else {
 384                         if (copy.Exists()) {
 385                                 if (copy.IsDirectory())
 386                                         ret = _RemoveFolder(copy);
 387                                 else
 388                                         ret = copy.Remove();
 389                                 if (ret != B_OK) {
 390                                         fprintf(stderr, "Failed to make room for entry '%s': "
 391                                                 "%s\n", name, strerror(ret));
 392                                         return ret;
 393                                 }
 394                         }
 395                         if (S_ISLNK(statInfo.st_mode)) {
 396                                 // copy symbolic links
 397                                 BSymLink srcLink(&entry);
 398                                 if (ret < B_OK)
 399                                         return ret;
 400
 401                                 char linkPath[B_PATH_NAME_LENGTH];
 402                                 ssize_t read = srcLink.ReadLink(linkPath, B_PATH_NAME_LENGTH - 1);
 403                                 if (read < 0)
 404                                         return (status_t)read;
 405
 406                                 // just in case it already exists...
 407                                 copy.Remove();
 408
 409                                 BSymLink dstLink;
 410                                 ret = destination.CreateSymLink(name, linkPath, &dstLink);
 411                                 if (ret < B_OK)
 412                                         return ret;
 413                         } else {
 414                                 // copy file data
 415                                 // NOTE: Do not pass the locker, we simply keep holding the lock!
 416                                 ret = CopyFile(entry, copy);
 417                                 if (ret < B_OK)
 418                                         return ret;
 419                         }
 420                 }
 421
 422                 if (!copyAttributes)
 423                         continue;
 424
 425                 ret = copy.SetTo(&destination, name);
 426                 if (ret != B_OK)
 427                         return ret;
 428
 429                 // copy attributes
 430                 BNode sourceNode(&entry);
 431                 BNode targetNode(&copy);
 432                 char attrName[B_ATTR_NAME_LENGTH];
 433                 while (sourceNode.GetNextAttrName(attrName) == B_OK) {
 434                         attr_info info;
 435                         if (sourceNode.GetAttrInfo(attrName, &info) < B_OK)
 436                                 continue;
 437                         size_t size = 4096;
 438                         uint8 buffer[size];
 439                         off_t offset = 0;
 440                         ssize_t read = sourceNode.ReadAttr(attrName, info.type,
 441                                 offset, buffer, std::min((off_t)size, info.size));
 442                         // NOTE: It's important to still write the attribute even if
 443                         // we have read 0 bytes!
 444                         while (read >= 0) {
 445                                 targetNode.WriteAttr(attrName, info.type, offset, buffer, read);
 446                                 offset += read;
 447                                 read = sourceNode.ReadAttr(attrName, info.type,
 448                                         offset, buffer, std::min((off_t)size, info.size - offset));
 449                                 if (read == 0)
 450                                         break;
 451                         }
 452                 }
 453
 454                 // copy basic attributes
 455                 copy.SetPermissions(statInfo.st_mode);
 456                 copy.SetOwner(statInfo.st_uid);
 457                 copy.SetGroup(statInfo.st_gid);
 458                 copy.SetModificationTime(statInfo.st_mtime);
 459                 copy.SetCreationTime(statInfo.st_crtime);
 460         }
 461
 462         level--;
 463         return B_OK;
 464 }
 465
 466
 467 status_t
 468 CopyEngine::_RemoveFolder(BEntry& entry)
 469 {
 470         BDirectory directory(&entry);
 471         status_t ret = directory.InitCheck();
 472         if (ret != B_OK)
 473                 return ret;
 474
 475         BEntry subEntry;
 476         while (directory.GetNextEntry(&subEntry) == B_OK) {
 477                 if (subEntry.IsDirectory()) {
 478                         ret = _RemoveFolder(subEntry);
 479                         if (ret != B_OK)
 480                                 return ret;
 481                 } else {
 482                         ret = subEntry.Remove();
 483                         if (ret != B_OK)
 484                                 return ret;
 485                 }
 486         }
 487         return entry.Remove();
 488 }
 489
 490
 491 const char*
 492 CopyEngine::_RelativeEntryPath(const char* absoluteSourcePath) const
 493 {
 494         if (strncmp(absoluteSourcePath, fAbsoluteSourcePath,
 495                         fAbsoluteSourcePath.Length()) != 0) {
 496                 return absoluteSourcePath;
 497         }
 498
 499         const char* relativePath
 500                 = absoluteSourcePath + fAbsoluteSourcePath.Length();
 501         return relativePath[0] == '/' ? relativePath + 1 : relativePath;
 502 }
 503
 504
 505 void
 506 CopyEngine::_UpdateProgress()
 507 {
 508         if (fProgressReporter == NULL)
 509                 return;
 510
 511         uint64 items = 0;
 512         if (fLastItemsCopied < fItemsCopied) {
 513                 items = fItemsCopied - fLastItemsCopied;
 514                 fLastItemsCopied = fItemsCopied;
 515         }
 516
 517         off_t bytes = 0;
 518         if (fLastBytesRead < fBytesRead) {
 519                 bytes = fBytesRead - fLastBytesRead;
 520                 fLastBytesRead = fBytesRead;
 521         }
 522
 523         fProgressReporter->ItemsWritten(items, bytes, fCurrentItem,
 524                 fCurrentTargetFolder);
 525 }
 526
 527
 528 int32
 529 CopyEngine::_WriteThreadEntry(void* cookie)
 530 {
 531         CopyEngine* engine = (CopyEngine*)cookie;
 532         engine->_WriteThread();
 533         return B_OK;
 534 }
 535
 536
 537 void
 538 CopyEngine::_WriteThread()
 539 {
 540         bigtime_t bufferWaitTimeout = 100000;
 541
 542         while (!fQuitting) {
 543
 544                 bigtime_t now = system_time();
 545
 546                 Buffer* buffer = NULL;
 547                 status_t ret = fBufferQueue.Pop(&buffer, bufferWaitTimeout);
 548                 if (ret == B_TIMED_OUT) {
 549                         // no buffer, timeout
 550                         continue;
 551                 } else if (ret == B_NO_INIT) {
 552                         // real error
 553                         return;
 554                 } else if (ret != B_OK) {
 555                         // no buffer, queue error
 556                         snooze(10000);
 557                         continue;
 558                 }
 559
 560                 if (!buffer->deleteFile) {
 561                         ssize_t written = buffer->file->Write(buffer->buffer,
 562                                 buffer->validBytes);
 563                         if (written != (ssize_t)buffer->validBytes) {
 564                                 // TODO: this should somehow be propagated back
 565                                 // to the main thread!
 566                                 fprintf(stderr, "Failed to write data: %s\n",
 567                                         strerror((status_t)written));
 568                         }
 569                         fBytesWritten += written;
 570                 }
 571
 572                 delete buffer;
 573
 574                 // measure performance
 575                 fTimeWritten += system_time() - now;
 576         }
 577
 578         double megaBytes = (double)fBytesWritten / (1024 * 1024);
 579         double seconds = (fTimeWritten / 1000000);
 580         if (seconds > 0) {
 581                 printf("%.2f MB written (%.2f MB/s)\n", megaBytes,
 582                         megaBytes / seconds);
 583         }
 584 }