src/gallium/auxiliary/pipebuffer/pb_bufmgr_slab.c

   1 /**************************************************************************
   2  *
   3  * Copyright 2006-2008 Tungsten Graphics, Inc., Cedar Park, TX., USA
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, FREE of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  18  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  19  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  20  * USE OR OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * The above copyright notice and this permission notice (including the
  23  * next paragraph) shall be included in all copies or substantial portions
  24  * of the Software.
  25  *
  26  *
  27  **************************************************************************/
  28
  29 /**
  30  * @file
  31  * S-lab pool implementation.
  32  *
  33  * @sa http://en.wikipedia.org/wiki/Slab_allocation
  34  *
  35  * @author Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
  36  * @author Jose Fonseca <jrfonseca@tungstengraphics.com>
  37  */
  38
  39 #include "pipe/p_compiler.h"
  40 #include "util/u_debug.h"
  41 #include "os/os_thread.h"
  42 #include "pipe/p_defines.h"
  43 #include "util/u_memory.h"
  44 #include "util/u_double_list.h"
  45 #include "util/u_time.h"
  46
  47 #include "pb_buffer.h"
  48 #include "pb_bufmgr.h"
  49
  50
  51 struct pb_slab;
  52
  53
  54 /**
  55  * Buffer in a slab.
  56  *
  57  * Sub-allocation of a contiguous buffer.
  58  */
  59 struct pb_slab_buffer
  60 {
  61    struct pb_buffer base;
  62
  63    struct pb_slab *slab;
  64
  65    struct list_head head;
  66
  67    unsigned mapCount;
  68
  69    /** Offset relative to the start of the slab buffer. */
  70    pb_size start;
  71
  72    /** Use when validating, to signal that all mappings are finished */
  73    /* TODO: Actually validation does not reach this stage yet */
  74    pipe_condvar event;
  75 };
  76
  77
  78 /**
  79  * Slab -- a contiguous piece of memory.
  80  */
  81 struct pb_slab
  82 {
  83    struct list_head head;
  84    struct list_head freeBuffers;
  85    pb_size numBuffers;
  86    pb_size numFree;
  87
  88    struct pb_slab_buffer *buffers;
  89    struct pb_slab_manager *mgr;
  90
  91    /** Buffer from the provider */
  92    struct pb_buffer *bo;
  93
  94    void *virtual;
  95 };
  96
  97
  98 /**
  99  * It adds/removes slabs as needed in order to meet the allocation/destruction
 100  * of individual buffers.
 101  */
 102 struct pb_slab_manager
 103 {
 104    struct pb_manager base;
 105
 106    /** From where we get our buffers */
 107    struct pb_manager *provider;
 108
 109    /** Size of the buffers we hand on downstream */
 110    pb_size bufSize;
 111
 112    /** Size of the buffers we request upstream */
 113    pb_size slabSize;
 114
 115    /**
 116     * Alignment, usage to be used to allocate the slab buffers.
 117     *
 118     * We can only provide buffers which are consistent (in alignment, usage)
 119     * with this description.
 120     */
 121    struct pb_desc desc;
 122
 123    /**
 124     * Partial slabs
 125     *
 126     * Full slabs are not stored in any list. Empty slabs are destroyed
 127     * immediatly.
 128     */
 129    struct list_head slabs;
 130
 131    pipe_mutex mutex;
 132 };
 133
 134
 135 /**
 136  * Wrapper around several slabs, therefore capable of handling buffers of
 137  * multiple sizes.
 138  *
 139  * This buffer manager just dispatches buffer allocations to the appropriate slab
 140  * manager, according to the requested buffer size, or by passes the slab
 141  * managers altogether for even greater sizes.
 142  *
 143  * The data of this structure remains constant after
 144  * initialization and thus needs no mutex protection.
 145  */
 146 struct pb_slab_range_manager
 147 {
 148    struct pb_manager base;
 149
 150    struct pb_manager *provider;
 151
 152    pb_size minBufSize;
 153    pb_size maxBufSize;
 154
 155    /** @sa pb_slab_manager::desc */
 156    struct pb_desc desc;
 157
 158    unsigned numBuckets;
 159    pb_size *bucketSizes;
 160
 161    /** Array of pb_slab_manager, one for each bucket size */
 162    struct pb_manager **buckets;
 163 };
 164
 165
 166 static INLINE struct pb_slab_buffer *
 167 pb_slab_buffer(struct pb_buffer *buf)
 168 {
 169    assert(buf);
 170    return (struct pb_slab_buffer *)buf;
 171 }
 172
 173
 174 static INLINE struct pb_slab_manager *
 175 pb_slab_manager(struct pb_manager *mgr)
 176 {
 177    assert(mgr);
 178    return (struct pb_slab_manager *)mgr;
 179 }
 180
 181
 182 static INLINE struct pb_slab_range_manager *
 183 pb_slab_range_manager(struct pb_manager *mgr)
 184 {
 185    assert(mgr);
 186    return (struct pb_slab_range_manager *)mgr;
 187 }
 188
 189
 190 /**
 191  * Delete a buffer from the slab delayed list and put
 192  * it on the slab FREE list.
 193  */
 194 static void
 195 pb_slab_buffer_destroy(struct pb_buffer *_buf)
 196 {
 197    struct pb_slab_buffer *buf = pb_slab_buffer(_buf);
 198    struct pb_slab *slab = buf->slab;
 199    struct pb_slab_manager *mgr = slab->mgr;
 200    struct list_head *list = &buf->head;
 201
 202    pipe_mutex_lock(mgr->mutex);
 203
 204    assert(!pipe_is_referenced(&buf->base.base.reference));
 205
 206    buf->mapCount = 0;
 207
 208    LIST_DEL(list);
 209    LIST_ADDTAIL(list, &slab->freeBuffers);
 210    slab->numFree++;
 211
 212    if (slab->head.next == &slab->head)
 213       LIST_ADDTAIL(&slab->head, &mgr->slabs);
 214
 215    /* If the slab becomes totally empty, free it */
 216    if (slab->numFree == slab->numBuffers) {
 217       list = &slab->head;
 218       LIST_DELINIT(list);
 219       pb_reference(&slab->bo, NULL);
 220       FREE(slab->buffers);
 221       FREE(slab);
 222    }
 223
 224    pipe_mutex_unlock(mgr->mutex);
 225 }
 226
 227
 228 static void *
 229 pb_slab_buffer_map(struct pb_buffer *_buf,
 230                    unsigned flags)
 231 {
 232    struct pb_slab_buffer *buf = pb_slab_buffer(_buf);
 233
 234    ++buf->mapCount;
 235    return (void *) ((uint8_t *) buf->slab->virtual + buf->start);
 236 }
 237
 238
 239 static void
 240 pb_slab_buffer_unmap(struct pb_buffer *_buf)
 241 {
 242    struct pb_slab_buffer *buf = pb_slab_buffer(_buf);
 243
 244    --buf->mapCount;
 245    if (buf->mapCount == 0)
 246        pipe_condvar_broadcast(buf->event);
 247 }
 248
 249
 250 static enum pipe_error
 251 pb_slab_buffer_validate(struct pb_buffer *_buf,
 252                          struct pb_validate *vl,
 253                          unsigned flags)
 254 {
 255    struct pb_slab_buffer *buf = pb_slab_buffer(_buf);
 256    return pb_validate(buf->slab->bo, vl, flags);
 257 }
 258
 259
 260 static void
 261 pb_slab_buffer_fence(struct pb_buffer *_buf,
 262                       struct pipe_fence_handle *fence)
 263 {
 264    struct pb_slab_buffer *buf = pb_slab_buffer(_buf);
 265    pb_fence(buf->slab->bo, fence);
 266 }
 267
 268
 269 static void
 270 pb_slab_buffer_get_base_buffer(struct pb_buffer *_buf,
 271                                struct pb_buffer **base_buf,
 272                                pb_size *offset)
 273 {
 274    struct pb_slab_buffer *buf = pb_slab_buffer(_buf);
 275    pb_get_base_buffer(buf->slab->bo, base_buf, offset);
 276    *offset += buf->start;
 277 }
 278
 279
 280 static const struct pb_vtbl
 281 pb_slab_buffer_vtbl = {
 282       pb_slab_buffer_destroy,
 283       pb_slab_buffer_map,
 284       pb_slab_buffer_unmap,
 285       pb_slab_buffer_validate,
 286       pb_slab_buffer_fence,
 287       pb_slab_buffer_get_base_buffer
 288 };
 289
 290
 291 /**
 292  * Create a new slab.
 293  *
 294  * Called when we ran out of free slabs.
 295  */
 296 static enum pipe_error
 297 pb_slab_create(struct pb_slab_manager *mgr)
 298 {
 299    struct pb_slab *slab;
 300    struct pb_slab_buffer *buf;
 301    unsigned numBuffers;
 302    unsigned i;
 303    enum pipe_error ret;
 304
 305    slab = CALLOC_STRUCT(pb_slab);
 306    if (!slab)
 307       return PIPE_ERROR_OUT_OF_MEMORY;
 308
 309    slab->bo = mgr->provider->create_buffer(mgr->provider, mgr->slabSize, &mgr->desc);
 310    if(!slab->bo) {
 311       ret = PIPE_ERROR_OUT_OF_MEMORY;
 312       goto out_err0;
 313    }
 314
 315    /* Note down the slab virtual address. All mappings are accessed directly
 316     * through this address so it is required that the buffer is pinned. */
 317    slab->virtual = pb_map(slab->bo,
 318                           PIPE_BUFFER_USAGE_CPU_READ |
 319                           PIPE_BUFFER_USAGE_CPU_WRITE);
 320    if(!slab->virtual) {
 321       ret = PIPE_ERROR_OUT_OF_MEMORY;
 322       goto out_err1;
 323    }
 324    pb_unmap(slab->bo);
 325
 326    numBuffers = slab->bo->base.size / mgr->bufSize;
 327
 328    slab->buffers = CALLOC(numBuffers, sizeof(*slab->buffers));
 329    if (!slab->buffers) {
 330       ret = PIPE_ERROR_OUT_OF_MEMORY;
 331       goto out_err1;
 332    }
 333
 334    LIST_INITHEAD(&slab->head);
 335    LIST_INITHEAD(&slab->freeBuffers);
 336    slab->numBuffers = numBuffers;
 337    slab->numFree = 0;
 338    slab->mgr = mgr;
 339
 340    buf = slab->buffers;
 341    for (i=0; i < numBuffers; ++i) {
 342       pipe_reference_init(&buf->base.base.reference, 0);
 343       buf->base.base.size = mgr->bufSize;
 344       buf->base.base.alignment = 0;
 345       buf->base.base.usage = 0;
 346       buf->base.vtbl = &pb_slab_buffer_vtbl;
 347       buf->slab = slab;
 348       buf->start = i* mgr->bufSize;
 349       buf->mapCount = 0;
 350       pipe_condvar_init(buf->event);
 351       LIST_ADDTAIL(&buf->head, &slab->freeBuffers);
 352       slab->numFree++;
 353       buf++;
 354    }
 355
 356    /* Add this slab to the list of partial slabs */
 357    LIST_ADDTAIL(&slab->head, &mgr->slabs);
 358
 359    return PIPE_OK;
 360
 361 out_err1:
 362    pb_reference(&slab->bo, NULL);
 363 out_err0:
 364    FREE(slab);
 365    return ret;
 366 }
 367
 368
 369 static struct pb_buffer *
 370 pb_slab_manager_create_buffer(struct pb_manager *_mgr,
 371                               pb_size size,
 372                               const struct pb_desc *desc)
 373 {
 374    struct pb_slab_manager *mgr = pb_slab_manager(_mgr);
 375    static struct pb_slab_buffer *buf;
 376    struct pb_slab *slab;
 377    struct list_head *list;
 378
 379    /* check size */
 380    assert(size <= mgr->bufSize);
 381    if(size > mgr->bufSize)
 382       return NULL;
 383
 384    /* check if we can provide the requested alignment */
 385    assert(pb_check_alignment(desc->alignment, mgr->desc.alignment));
 386    if(!pb_check_alignment(desc->alignment, mgr->desc.alignment))
 387       return NULL;
 388    assert(pb_check_alignment(desc->alignment, mgr->bufSize));
 389    if(!pb_check_alignment(desc->alignment, mgr->bufSize))
 390       return NULL;
 391
 392    assert(pb_check_usage(desc->usage, mgr->desc.usage));
 393    if(!pb_check_usage(desc->usage, mgr->desc.usage))
 394       return NULL;
 395
 396    pipe_mutex_lock(mgr->mutex);
 397
 398    /* Create a new slab, if we run out of partial slabs */
 399    if (mgr->slabs.next == &mgr->slabs) {
 400       (void) pb_slab_create(mgr);
 401       if (mgr->slabs.next == &mgr->slabs) {
 402          pipe_mutex_unlock(mgr->mutex);
 403          return NULL;
 404       }
 405    }
 406
 407    /* Allocate the buffer from a partial (or just created) slab */
 408    list = mgr->slabs.next;
 409    slab = LIST_ENTRY(struct pb_slab, list, head);
 410
 411    /* If totally full remove from the partial slab list */
 412    if (--slab->numFree == 0)
 413       LIST_DELINIT(list);
 414
 415    list = slab->freeBuffers.next;
 416    LIST_DELINIT(list);
 417
 418    pipe_mutex_unlock(mgr->mutex);
 419    buf = LIST_ENTRY(struct pb_slab_buffer, list, head);
 420
 421    pipe_reference_init(&buf->base.base.reference, 1);
 422    buf->base.base.alignment = desc->alignment;
 423    buf->base.base.usage = desc->usage;
 424
 425    return &buf->base;
 426 }
 427
 428
 429 static void
 430 pb_slab_manager_flush(struct pb_manager *_mgr)
 431 {
 432    struct pb_slab_manager *mgr = pb_slab_manager(_mgr);
 433
 434    assert(mgr->provider->flush);
 435    if(mgr->provider->flush)
 436       mgr->provider->flush(mgr->provider);
 437 }
 438
 439
 440 static void
 441 pb_slab_manager_destroy(struct pb_manager *_mgr)
 442 {
 443    struct pb_slab_manager *mgr = pb_slab_manager(_mgr);
 444
 445    /* TODO: cleanup all allocated buffers */
 446    FREE(mgr);
 447 }
 448
 449
 450 struct pb_manager *
 451 pb_slab_manager_create(struct pb_manager *provider,
 452                        pb_size bufSize,
 453                        pb_size slabSize,
 454                        const struct pb_desc *desc)
 455 {
 456    struct pb_slab_manager *mgr;
 457
 458    mgr = CALLOC_STRUCT(pb_slab_manager);
 459    if (!mgr)
 460       return NULL;
 461
 462    mgr->base.destroy = pb_slab_manager_destroy;
 463    mgr->base.create_buffer = pb_slab_manager_create_buffer;
 464    mgr->base.flush = pb_slab_manager_flush;
 465
 466    mgr->provider = provider;
 467    mgr->bufSize = bufSize;
 468    mgr->slabSize = slabSize;
 469    mgr->desc = *desc;
 470
 471    LIST_INITHEAD(&mgr->slabs);
 472
 473    pipe_mutex_init(mgr->mutex);
 474
 475    return &mgr->base;
 476 }
 477
 478
 479 static struct pb_buffer *
 480 pb_slab_range_manager_create_buffer(struct pb_manager *_mgr,
 481                                     pb_size size,
 482                                     const struct pb_desc *desc)
 483 {
 484    struct pb_slab_range_manager *mgr = pb_slab_range_manager(_mgr);
 485    pb_size bufSize;
 486    pb_size reqSize = size;
 487    unsigned i;
 488
 489    if(desc->alignment > reqSize)
 490            reqSize = desc->alignment;
 491
 492    bufSize = mgr->minBufSize;
 493    for (i = 0; i < mgr->numBuckets; ++i) {
 494       if(bufSize >= reqSize)
 495          return mgr->buckets[i]->create_buffer(mgr->buckets[i], size, desc);
 496       bufSize *= 2;
 497    }
 498
 499    /* Fall back to allocate a buffer object directly from the provider. */
 500    return mgr->provider->create_buffer(mgr->provider, size, desc);
 501 }
 502
 503
 504 static void
 505 pb_slab_range_manager_flush(struct pb_manager *_mgr)
 506 {
 507    struct pb_slab_range_manager *mgr = pb_slab_range_manager(_mgr);
 508
 509    /* Individual slabs don't hold any temporary buffers so no need to call them */
 510
 511    assert(mgr->provider->flush);
 512    if(mgr->provider->flush)
 513       mgr->provider->flush(mgr->provider);
 514 }
 515
 516
 517 static void
 518 pb_slab_range_manager_destroy(struct pb_manager *_mgr)
 519 {
 520    struct pb_slab_range_manager *mgr = pb_slab_range_manager(_mgr);
 521    unsigned i;
 522
 523    for (i = 0; i < mgr->numBuckets; ++i)
 524       mgr->buckets[i]->destroy(mgr->buckets[i]);
 525    FREE(mgr->buckets);
 526    FREE(mgr->bucketSizes);
 527    FREE(mgr);
 528 }
 529
 530
 531 struct pb_manager *
 532 pb_slab_range_manager_create(struct pb_manager *provider,
 533                              pb_size minBufSize,
 534                              pb_size maxBufSize,
 535                              pb_size slabSize,
 536                              const struct pb_desc *desc)
 537 {
 538    struct pb_slab_range_manager *mgr;
 539    pb_size bufSize;
 540    unsigned i;
 541
 542    if(!provider)
 543       return NULL;
 544
 545    mgr = CALLOC_STRUCT(pb_slab_range_manager);
 546    if (!mgr)
 547       goto out_err0;
 548
 549    mgr->base.destroy = pb_slab_range_manager_destroy;
 550    mgr->base.create_buffer = pb_slab_range_manager_create_buffer;
 551    mgr->base.flush = pb_slab_range_manager_flush;
 552
 553    mgr->provider = provider;
 554    mgr->minBufSize = minBufSize;
 555    mgr->maxBufSize = maxBufSize;
 556
 557    mgr->numBuckets = 1;
 558    bufSize = minBufSize;
 559    while(bufSize < maxBufSize) {
 560       bufSize *= 2;
 561       ++mgr->numBuckets;
 562    }
 563
 564    mgr->buckets = CALLOC(mgr->numBuckets, sizeof(*mgr->buckets));
 565    if (!mgr->buckets)
 566       goto out_err1;
 567
 568    bufSize = minBufSize;
 569    for (i = 0; i < mgr->numBuckets; ++i) {
 570       mgr->buckets[i] = pb_slab_manager_create(provider, bufSize, slabSize, desc);
 571       if(!mgr->buckets[i])
 572          goto out_err2;
 573       bufSize *= 2;
 574    }
 575
 576    return &mgr->base;
 577
 578 out_err2:
 579    for (i = 0; i < mgr->numBuckets; ++i)
 580       if(mgr->buckets[i])
 581             mgr->buckets[i]->destroy(mgr->buckets[i]);
 582    FREE(mgr->buckets);
 583 out_err1:
 584    FREE(mgr);
 585 out_err0:
 586    return NULL;
 587 }