src/system/kernel/fs/vfs_request_io.cpp

   1 /*
   2  * Copyright 2008-2010, Ingo Weinhold, ingo_weinhold@gmx.de.
   3  * Distributed under the terms of the MIT License.
   4  */
   5
   6 // included by vfs.cpp
   7
   8
   9 //#define TRACE_VFS_REQUEST_IO
  10 #ifdef TRACE_VFS_REQUEST_IO
  11 #       define TRACE_RIO(x...) dprintf(x)
  12 #else
  13 #       define TRACE_RIO(x...) do {} while (false)
  14 #endif
  15
  16
  17 #include <heap.h>
  18
  19
  20 // #pragma mark - AsyncIOCallback
  21
  22
  23 AsyncIOCallback::~AsyncIOCallback()
  24 {
  25 }
  26
  27
  28 /* static */ status_t
  29 AsyncIOCallback::IORequestCallback(void* data, io_request* request,
  30         status_t status, bool partialTransfer, generic_size_t transferEndOffset)
  31 {
  32         ((AsyncIOCallback*)data)->IOFinished(status, partialTransfer,
  33                 transferEndOffset);
  34         return B_OK;
  35 }
  36
  37
  38 // #pragma mark - StackableAsyncIOCallback
  39
  40
  41 StackableAsyncIOCallback::StackableAsyncIOCallback(AsyncIOCallback* next)
  42         :
  43         fNextCallback(next)
  44 {
  45 }
  46
  47
  48 // #pragma mark -
  49
  50
  51 struct iterative_io_cookie {
  52         struct vnode*                                   vnode;
  53         file_descriptor*                                descriptor;
  54         iterative_io_get_vecs                   get_vecs;
  55         iterative_io_finished                   finished;
  56         void*                                                   cookie;
  57         off_t                                                   request_offset;
  58         io_request_finished_callback    next_finished_callback;
  59         void*                                                   next_finished_cookie;
  60 };
  61
  62
  63 class DoIO {
  64 public:
  65         DoIO(bool write)
  66                 :
  67                 fWrite(write)
  68         {
  69         }
  70
  71         virtual ~DoIO()
  72         {
  73         }
  74
  75         virtual status_t IO(off_t offset, void* buffer, size_t* length) = 0;
  76
  77 protected:
  78         bool    fWrite;
  79 };
  80
  81
  82 class CallbackIO : public DoIO {
  83 public:
  84         CallbackIO(bool write,
  85                         status_t (*doIO)(void* cookie, off_t offset, void* buffer,
  86                                 size_t* length),
  87                         void* cookie)
  88                 :
  89                 DoIO(write),
  90                 fDoIO(doIO),
  91                 fCookie(cookie)
  92         {
  93         }
  94
  95         virtual status_t IO(off_t offset, void* buffer, size_t* length)
  96         {
  97                 return fDoIO(fCookie, offset, buffer, length);
  98         }
  99
 100 private:
 101         status_t (*fDoIO)(void*, off_t, void*, size_t*);
 102         void*           fCookie;
 103 };
 104
 105
 106 class VnodeIO : public DoIO {
 107 public:
 108         VnodeIO(bool write, struct vnode* vnode, void* cookie)
 109                 :
 110                 DoIO(write),
 111                 fVnode(vnode),
 112                 fCookie(cookie)
 113         {
 114         }
 115
 116         virtual status_t IO(off_t offset, void* buffer, size_t* length)
 117         {
 118                 iovec vec;
 119                 vec.iov_base = buffer;
 120                 vec.iov_len = *length;
 121
 122                 if (fWrite) {
 123                         return FS_CALL(fVnode, write_pages, fCookie, offset, &vec, 1,
 124                                 length);
 125                 }
 126
 127                 return FS_CALL(fVnode, read_pages, fCookie, offset, &vec, 1, length);
 128         }
 129
 130 private:
 131         struct vnode*   fVnode;
 132         void*                   fCookie;
 133 };
 134
 135
 136 static status_t
 137 do_iterative_fd_io_iterate(void* _cookie, io_request* request,
 138         bool* _partialTransfer)
 139 {
 140         TRACE_RIO("[%ld] do_iterative_fd_io_iterate(request: %p)\n",
 141                 find_thread(NULL), request);
 142
 143         static const size_t kMaxSubRequests = 8;
 144
 145         iterative_io_cookie* cookie = (iterative_io_cookie*)_cookie;
 146
 147         request->DeleteSubRequests();
 148
 149         off_t requestOffset = cookie->request_offset;
 150         size_t requestLength = request->Length()
 151                 - (requestOffset - request->Offset());
 152
 153         // get the next file vecs
 154         file_io_vec vecs[kMaxSubRequests];
 155         size_t vecCount = kMaxSubRequests;
 156         status_t error = cookie->get_vecs(cookie->cookie, request, requestOffset,
 157                 requestLength, vecs, &vecCount);
 158         if (error != B_OK && error != B_BUFFER_OVERFLOW)
 159                 return error;
 160         if (vecCount == 0) {
 161                 *_partialTransfer = true;
 162                 return B_OK;
 163         }
 164         TRACE_RIO("[%ld]  got %zu file vecs\n", find_thread(NULL), vecCount);
 165
 166         // Reset the error code for the loop below
 167         error = B_OK;
 168
 169         // create subrequests for the file vecs we've got
 170         size_t subRequestCount = 0;
 171         for (size_t i = 0;
 172                 i < vecCount && subRequestCount < kMaxSubRequests && error == B_OK;
 173                 i++) {
 174                 off_t vecOffset = vecs[i].offset;
 175                 off_t vecLength = min_c(vecs[i].length, (off_t)requestLength);
 176                 TRACE_RIO("[%ld]    vec %lu offset: %lld, length: %lld\n",
 177                         find_thread(NULL), i, vecOffset, vecLength);
 178
 179                 // Special offset -1 means that this is part of sparse file that is
 180                 // zero. We fill it in right here.
 181                 if (vecOffset == -1) {
 182                         if (request->IsWrite()) {
 183                                 panic("do_iterative_fd_io_iterate(): write to sparse file "
 184                                         "vector");
 185                                 error = B_BAD_VALUE;
 186                                 break;
 187                         }
 188
 189                         error = request->ClearData(requestOffset, vecLength);
 190                         if (error != B_OK)
 191                                 break;
 192
 193                         requestOffset += vecLength;
 194                         requestLength -= vecLength;
 195                         continue;
 196                 }
 197
 198                 while (vecLength > 0 && subRequestCount < kMaxSubRequests) {
 199                         TRACE_RIO("[%ld]    creating subrequest: offset: %lld, length: "
 200                                 "%lld\n", find_thread(NULL), vecOffset, vecLength);
 201                         IORequest* subRequest;
 202                         error = request->CreateSubRequest(requestOffset, vecOffset,
 203                                 vecLength, subRequest);
 204                         if (error != B_OK)
 205                                 break;
 206
 207                         subRequestCount++;
 208
 209                         size_t lengthProcessed = subRequest->Length();
 210                         vecOffset += lengthProcessed;
 211                         vecLength -= lengthProcessed;
 212                         requestOffset += lengthProcessed;
 213                         requestLength -= lengthProcessed;
 214                 }
 215         }
 216
 217         // Only if we couldn't create any subrequests, we fail.
 218         if (error != B_OK && subRequestCount == 0)
 219                 return error;
 220
 221         // Reset the error code for the loop below
 222         error = B_OK;
 223
 224         request->Advance(requestOffset - cookie->request_offset);
 225         cookie->request_offset = requestOffset;
 226
 227         // If we don't have any sub requests at this point, that means all that
 228         // remained were zeroed sparse file vectors. So the request is done now.
 229         if (subRequestCount == 0) {
 230                 ASSERT(request->RemainingBytes() == 0);
 231                 request->SetStatusAndNotify(B_OK);
 232                 return B_OK;
 233         }
 234
 235         // Schedule the subrequests.
 236         IORequest* nextSubRequest = request->FirstSubRequest();
 237         while (nextSubRequest != NULL) {
 238                 IORequest* subRequest = nextSubRequest;
 239                 nextSubRequest = request->NextSubRequest(subRequest);
 240
 241                 if (error == B_OK) {
 242                         TRACE_RIO("[%ld]  scheduling subrequest: %p\n", find_thread(NULL),
 243                                 subRequest);
 244                         error = vfs_vnode_io(cookie->vnode, cookie->descriptor->cookie,
 245                                 subRequest);
 246                 } else {
 247                         // Once scheduling a subrequest failed, we cancel all subsequent
 248                         // subrequests.
 249                         subRequest->SetStatusAndNotify(B_CANCELED);
 250                 }
 251         }
 252
 253         // TODO: Cancel the subrequests that were scheduled successfully.
 254
 255         return B_OK;
 256 }
 257
 258
 259 static status_t
 260 do_iterative_fd_io_finish(void* _cookie, io_request* request, status_t status,
 261         bool partialTransfer, generic_size_t transferEndOffset)
 262 {
 263         iterative_io_cookie* cookie = (iterative_io_cookie*)_cookie;
 264
 265         if (cookie->finished != NULL) {
 266                 cookie->finished(cookie->cookie, request, status, partialTransfer,
 267                         transferEndOffset);
 268         }
 269
 270         put_fd(cookie->descriptor);
 271
 272         if (cookie->next_finished_callback != NULL) {
 273                 cookie->next_finished_callback(cookie->next_finished_cookie, request,
 274                         status, partialTransfer, transferEndOffset);
 275         }
 276
 277         delete cookie;
 278
 279         return B_OK;
 280 }
 281
 282
 283 static status_t
 284 do_synchronous_iterative_vnode_io(struct vnode* vnode, void* openCookie,
 285         io_request* request, iterative_io_get_vecs getVecs,
 286         iterative_io_finished finished, void* cookie)
 287 {
 288         IOBuffer* buffer = request->Buffer();
 289         VnodeIO io(request->IsWrite(), vnode, openCookie);
 290
 291         iovec vector;
 292         void* virtualVecCookie = NULL;
 293         off_t offset = request->Offset();
 294         generic_size_t length = request->Length();
 295
 296         status_t error = B_OK;
 297
 298         for (; error == B_OK && length > 0
 299                         && buffer->GetNextVirtualVec(virtualVecCookie, vector) == B_OK;) {
 300                 uint8* vecBase = (uint8*)vector.iov_base;
 301                 generic_size_t vecLength = min_c(vector.iov_len, length);
 302
 303                 while (error == B_OK && vecLength > 0) {
 304                         file_io_vec fileVecs[8];
 305                         size_t fileVecCount = 8;
 306                         error = getVecs(cookie, request, offset, vecLength, fileVecs,
 307                                 &fileVecCount);
 308                         if (error != B_OK || fileVecCount == 0)
 309                                 break;
 310
 311                         for (size_t i = 0; i < fileVecCount; i++) {
 312                                 const file_io_vec& fileVec = fileVecs[i];
 313                                 size_t toTransfer = min_c(fileVec.length, (off_t)length);
 314                                 size_t transferred = toTransfer;
 315                                 error = io.IO(fileVec.offset, vecBase, &transferred);
 316                                 if (error != B_OK)
 317                                         break;
 318
 319                                 offset += transferred;
 320                                 length -= transferred;
 321                                 vecBase += transferred;
 322                                 vecLength -= transferred;
 323
 324                                 if (transferred != toTransfer)
 325                                         break;
 326                         }
 327                 }
 328         }
 329
 330         buffer->FreeVirtualVecCookie(virtualVecCookie);
 331
 332         bool partial = length > 0;
 333         size_t bytesTransferred = request->Length() - length;
 334         request->SetTransferredBytes(partial, bytesTransferred);
 335         finished(cookie, request, error, partial, bytesTransferred);
 336         request->SetStatusAndNotify(error);
 337         return error;
 338 }
 339
 340
 341 static status_t
 342 synchronous_io(io_request* request, DoIO& io)
 343 {
 344         TRACE_RIO("[%" B_PRId32 "] synchronous_io(request: %p (offset: %" B_PRIdOFF
 345                 ", length: %" B_PRIuGENADDR "))\n", find_thread(NULL), request,
 346                 request->Offset(), request->Length());
 347
 348         IOBuffer* buffer = request->Buffer();
 349
 350         iovec vector;
 351         void* virtualVecCookie = NULL;
 352         off_t offset = request->Offset();
 353         generic_size_t length = request->Length();
 354
 355         for (; length > 0
 356                         && buffer->GetNextVirtualVec(virtualVecCookie, vector) == B_OK;) {
 357                 void* vecBase = (void*)(addr_t)vector.iov_base;
 358                 size_t vecLength = min_c(vector.iov_len, length);
 359
 360                 TRACE_RIO("[%ld]   I/O: offset: %lld, vecBase: %p, length: %lu\n",
 361                         find_thread(NULL), offset, vecBase, vecLength);
 362
 363                 size_t transferred = vecLength;
 364                 status_t error = io.IO(offset, vecBase, &transferred);
 365                 if (error != B_OK) {
 366                         TRACE_RIO("[%ld]   I/O failed: %#lx\n", find_thread(NULL), error);
 367                         buffer->FreeVirtualVecCookie(virtualVecCookie);
 368                         request->SetStatusAndNotify(error);
 369                         return error;
 370                 }
 371
 372                 offset += transferred;
 373                 length -= transferred;
 374
 375                 if (transferred != vecLength)
 376                         break;
 377         }
 378
 379         TRACE_RIO("[%ld] synchronous_io() succeeded\n", find_thread(NULL));
 380
 381         buffer->FreeVirtualVecCookie(virtualVecCookie);
 382         request->SetTransferredBytes(length > 0, request->Length() - length);
 383         request->SetStatusAndNotify(B_OK);
 384         return B_OK;
 385 }
 386
 387
 388 // #pragma mark - kernel private API
 389
 390
 391 status_t
 392 vfs_vnode_io(struct vnode* vnode, void* cookie, io_request* request)
 393 {
 394         status_t result = B_ERROR;
 395         if (!HAS_FS_CALL(vnode, io)
 396                 || (result = FS_CALL(vnode, io, cookie, request)) == B_UNSUPPORTED) {
 397                 // no io() call -- fall back to synchronous I/O
 398                 VnodeIO io(request->IsWrite(), vnode, cookie);
 399                 return synchronous_io(request, io);
 400         }
 401
 402         return result;
 403 }
 404
 405
 406 status_t
 407 vfs_synchronous_io(io_request* request,
 408         status_t (*doIO)(void* cookie, off_t offset, void* buffer, size_t* length),
 409         void* cookie)
 410 {
 411         CallbackIO io(request->IsWrite(), doIO, cookie);
 412         return synchronous_io(request, io);
 413 }
 414
 415
 416 status_t
 417 vfs_asynchronous_read_pages(struct vnode* vnode, void* cookie, off_t pos,
 418         const generic_io_vec* vecs, size_t count, generic_size_t numBytes,
 419         uint32 flags, AsyncIOCallback* callback)
 420 {
 421         IORequest* request = IORequest::Create((flags & B_VIP_IO_REQUEST) != 0);
 422         if (request == NULL) {
 423                 callback->IOFinished(B_NO_MEMORY, true, 0);
 424                 return B_NO_MEMORY;
 425         }
 426
 427         status_t status = request->Init(pos, vecs, count, numBytes, false,
 428                 flags | B_DELETE_IO_REQUEST);
 429         if (status != B_OK) {
 430                 delete request;
 431                 callback->IOFinished(status, true, 0);
 432                 return status;
 433         }
 434
 435         request->SetFinishedCallback(&AsyncIOCallback::IORequestCallback,
 436                 callback);
 437
 438         return vfs_vnode_io(vnode, cookie, request);
 439 }
 440
 441
 442 status_t
 443 vfs_asynchronous_write_pages(struct vnode* vnode, void* cookie, off_t pos,
 444         const generic_io_vec* vecs, size_t count, generic_size_t numBytes,
 445         uint32 flags, AsyncIOCallback* callback)
 446 {
 447         IORequest* request = IORequest::Create((flags & B_VIP_IO_REQUEST) != 0);
 448         if (request == NULL) {
 449                 callback->IOFinished(B_NO_MEMORY, true, 0);
 450                 return B_NO_MEMORY;
 451         }
 452
 453         status_t status = request->Init(pos, vecs, count, numBytes, true,
 454                 flags | B_DELETE_IO_REQUEST);
 455         if (status != B_OK) {
 456                 delete request;
 457                 callback->IOFinished(status, true, 0);
 458                 return status;
 459         }
 460
 461         request->SetFinishedCallback(&AsyncIOCallback::IORequestCallback,
 462                 callback);
 463
 464         return vfs_vnode_io(vnode, cookie, request);
 465 }
 466
 467
 468 // #pragma mark - public API
 469
 470
 471 status_t
 472 do_fd_io(int fd, io_request* request)
 473 {
 474         struct vnode* vnode;
 475         file_descriptor* descriptor = get_fd_and_vnode(fd, &vnode, true);
 476         if (descriptor == NULL) {
 477                 request->SetStatusAndNotify(B_FILE_ERROR);
 478                 return B_FILE_ERROR;
 479         }
 480
 481         CObjectDeleter<file_descriptor> descriptorPutter(descriptor, put_fd);
 482
 483         return vfs_vnode_io(vnode, descriptor->cookie, request);
 484 }
 485
 486
 487 status_t
 488 do_iterative_fd_io(int fd, io_request* request, iterative_io_get_vecs getVecs,
 489         iterative_io_finished finished, void* cookie)
 490 {
 491         TRACE_RIO("[%" B_PRId32 "] do_iterative_fd_io(fd: %d, request: %p "
 492                 "(offset: %" B_PRIdOFF ", length: %" B_PRIuGENADDR "))\n",
 493                 find_thread(NULL), fd, request, request->Offset(), request->Length());
 494
 495         struct vnode* vnode;
 496         file_descriptor* descriptor = get_fd_and_vnode(fd, &vnode, true);
 497         if (descriptor == NULL) {
 498                 finished(cookie, request, B_FILE_ERROR, true, 0);
 499                 request->SetStatusAndNotify(B_FILE_ERROR);
 500                 return B_FILE_ERROR;
 501         }
 502
 503         CObjectDeleter<file_descriptor> descriptorPutter(descriptor, put_fd);
 504
 505         if (!HAS_FS_CALL(vnode, io)) {
 506                 // no io() call -- fall back to synchronous I/O
 507                 return do_synchronous_iterative_vnode_io(vnode, descriptor->cookie,
 508                         request, getVecs, finished, cookie);
 509         }
 510
 511         iterative_io_cookie* iterationCookie
 512                 = (request->Flags() & B_VIP_IO_REQUEST) != 0
 513                         ? new(malloc_flags(HEAP_PRIORITY_VIP)) iterative_io_cookie
 514                         : new(std::nothrow) iterative_io_cookie;
 515         if (iterationCookie == NULL) {
 516                 // no memory -- fall back to synchronous I/O
 517                 return do_synchronous_iterative_vnode_io(vnode, descriptor->cookie,
 518                         request, getVecs, finished, cookie);
 519         }
 520
 521         iterationCookie->vnode = vnode;
 522         iterationCookie->descriptor = descriptor;
 523         iterationCookie->get_vecs = getVecs;
 524         iterationCookie->finished = finished;
 525         iterationCookie->cookie = cookie;
 526         iterationCookie->request_offset = request->Offset();
 527         iterationCookie->next_finished_callback = request->FinishedCallback(
 528                 &iterationCookie->next_finished_cookie);
 529
 530         request->SetFinishedCallback(&do_iterative_fd_io_finish, iterationCookie);
 531         request->SetIterationCallback(&do_iterative_fd_io_iterate, iterationCookie);
 532
 533         descriptorPutter.Detach();
 534                 // From now on the descriptor is put by our finish callback.
 535
 536         bool partialTransfer = false;
 537         status_t error = do_iterative_fd_io_iterate(iterationCookie, request,
 538                 &partialTransfer);
 539         if (error != B_OK || partialTransfer) {
 540                 if (partialTransfer) {
 541                         request->SetTransferredBytes(partialTransfer,
 542                                 request->TransferredBytes());
 543                 }
 544
 545                 request->SetStatusAndNotify(error);
 546                 return error;
 547         }
 548
 549         return B_OK;
 550 }