inline unary ops: fix sc_floor for non-sse4.1
[supercollider.git] / common / SC_AllocPool.cpp
blobb483c607b6d04a4b9b79f64426260fef2d86ca38
1 /*
2 SuperCollider real time audio synthesis system
3 Copyright (c) 2002 James McCartney. All rights reserved.
4 http://www.audiosynth.com
6 This program is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 2 of the License, or
9 (at your option) any later version.
11 This program is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with this program; if not, write to the Free Software
18 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21 #include <string.h>
22 #include <stdexcept>
23 #include "SC_AllocPool.h"
24 #include "SC_BoundsMacros.h"
25 #include <assert.h>
26 #include <string>
29 Requests are `small' if both the corresponding and the next bin are small
32 #if DEBUG
33 #define check_pool() DoCheckPool()
34 #define check_free_chunk(P) DoCheckFreeChunk(P)
35 #define check_inuse_chunk(P) DoCheckInUseChunk(P)
36 #define check_chunk(P) DoCheckChunk(P)
37 #define check_malloced_chunk(P,N) DoCheckAllocedChunk(P,N)
38 #define garbage_fill(P) DoGarbageFill(P)
39 #else
40 #define check_pool()
41 #define check_free_chunk(P)
42 #define check_inuse_chunk(P)
43 #define check_chunk(P)
44 #define check_malloced_chunk(P,N)
45 #define garbage_fill(P)
46 #endif
48 #define aligned_OK(m) ((((unsigned long)(m)) & kAlignMask) == 0)
51 void* allocmem(AllocPool *pool, int32 size);
52 void* allocmem(AllocPool *pool, int32 size)
54 return pool->Alloc(size);
57 void* reallocmem(AllocPool *pool, void* ptr, int32 size);
58 void* reallocmem(AllocPool *pool, void* ptr, int32 size)
60 return pool->Realloc(ptr, size);
63 void freemem(AllocPool *pool, void* ptr);
64 void freemem(AllocPool *pool, void* ptr)
66 pool->Free(ptr);
69 void AllocPool::InitAlloc()
71 if (mAreaInitSize == 0) return;
73 /* alloc initial area */
74 NewArea(mAreaInitSize);
75 /* get chunk */
76 AllocAreaPtr area = mAreas;
77 AllocChunkPtr chunk = &area->mChunk;
78 LinkFree(chunk);
80 check_pool();
83 void AllocPool::InitBins()
85 for (int i=0; i<kNumAllocBins; ++i) {
86 mBins[i].BeEmpty();
88 for (int i=0; i<4; ++i) {
89 mBinBlocks[i] = 0;
93 AllocPool::AllocPool(NewAreaFunc inAllocArea, FreeAreaFunc inFreeArea,
94 size_t inAreaInitSize, size_t inAreaMoreSize)
96 InitBins();
97 mAreaInitSize = inAreaInitSize;
98 mAreaMoreSize = inAreaMoreSize;
99 mAllocArea = inAllocArea;
100 mFreeArea = inFreeArea;
101 mAreas = 0;
102 check_pool();
104 InitAlloc();
107 AllocPool::~AllocPool()
109 FreeAll();
112 void AllocPool::FreeAll()
114 check_pool();
115 AllocAreaPtr area = mAreas;
116 if (area) {
117 AllocAreaPtr firstArea = area;
118 do {
119 AllocAreaPtr nextarea = area->mNext;
120 (mFreeArea)(area->mUnalignedPointerToThis);
121 area = nextarea;
122 } while (area != firstArea);
123 mAreas = NULL;
125 InitBins();
126 check_pool();
129 void AllocPool::FreeAllInternal()
131 check_pool();
132 InitBins();
134 AllocAreaPtr area = mAreas;
135 if (area) {
136 AllocAreaPtr firstArea = area;
137 do {
138 AllocAreaPtr nextarea = area->mNext;
139 size_t size = area->mSize;
140 AllocChunkPtr chunk = &area->mChunk;
141 chunk->SetSizeFree(size);
142 chunk->SetNeighborsInUse(size);
143 LinkFree(chunk);
144 area = nextarea;
145 } while (area != firstArea);
147 check_pool();
150 void AllocPool::Reinit()
152 FreeAll();
153 InitAlloc();
156 void AllocPool::Free(void *inPtr)
158 #ifdef DISABLE_MEMORY_POOLS
159 free(inPtr);
160 return;
161 #endif
163 check_pool();
164 if (inPtr == 0) return; /* free(0) has no effect */
166 AllocChunkPtr chunk = MemToChunk(inPtr);
168 check_inuse_chunk(chunk);
169 garbage_fill(chunk);
171 size_t size = chunk->Size();
173 if (!chunk->PrevInUse()) /* consolidate backward */
175 size_t prevSize = chunk->PrevSize();
176 chunk = chunk->ChunkAtOffset(0L-prevSize);
177 size += prevSize;
178 UnlinkFree(chunk);
181 AllocChunkPtr next = chunk->ChunkAtOffset(size);
182 if (!next->InUse()) /* consolidate forward */
184 size += next->Size();
185 UnlinkFree(next);
188 chunk->SetSizeFree(size);
189 if (mAreaMoreSize && chunk->IsArea()) {
190 // whole area is free
191 FreeArea(chunk);
192 } else {
193 LinkFree(chunk);
195 check_pool();
200 AllocAreaPtr AllocPool::NewArea(size_t inAreaSize)
202 void *ptr = (AllocAreaPtr)(mAllocArea)(inAreaSize + kAreaOverhead);
204 if (ptr == NULL)
205 throw std::runtime_error(std::string("Could not allocate new area"));
207 // AllocAreaPtr area = (AllocAreaPtr)((unsigned long)ptr & ~kAlignMask);
208 AllocAreaPtr area = (AllocAreaPtr)(((unsigned long)ptr + kAlignMask) & ~kAlignMask);
209 assert((area >= ptr) && ((void*)((unsigned long)area & ~kAlignMask) == area));
211 area->mUnalignedPointerToThis = ptr;
213 /* link in area */
214 if (mAreas) {
215 area->mNext = mAreas;
216 area->mPrev = mAreas->mPrev;
217 area->mNext->mPrev = area;
218 area->mPrev->mNext = area;
219 } else {
220 area->mNext = area;
221 area->mPrev = area;
223 mAreas = area;
225 /* set area size */
226 area->mSize = inAreaSize;
227 area->mChunk.BeEmpty();
228 area->mChunk.SetNeighborsInUse(inAreaSize);
229 area->mChunk.SetSizeFree(inAreaSize);
231 return area;
234 void AllocPool::FreeArea(AllocChunkPtr chunk)
236 AllocAreaPtr area = (AllocAreaPtr)((char*)chunk - sizeof(AllocAreaHdr));
238 if (area->mNext == area) {
239 mAreas = NULL;
240 } else {
241 /* unlink area */
242 mAreas = area->mPrev->mNext = area->mNext;
243 area->mNext->mPrev = area->mPrev;
246 (mFreeArea)(area->mUnalignedPointerToThis);
250 size_t AllocPool::TotalFree()
252 size_t total = 0;
253 for (int i=0; i<kNumAllocBins; ++i) {
254 AllocChunkPtr bin = mBins + i;
255 if (bin->Prev() != bin) {
256 for (AllocChunkPtr candidate = bin->Prev(); candidate != bin; candidate = candidate->Prev()) {
257 total += candidate->Size();
261 return total;
264 size_t AllocPool::LargestFreeChunk()
266 int word = 0;
267 for (int i=3; i>=0; --i) {
268 if (mBinBlocks[i]) {
269 word = i;
270 break;
273 int binBits = (int)mBinBlocks[word];
274 int bitPosition = NUMBITS(binBits) - 1;
275 int index = (word << 5) + bitPosition;
276 AllocChunkPtr bin = mBins + index;
277 //postbuf("** %p %p %p %p\n", mBinBlocks[0], mBinBlocks[1], mBinBlocks[2], mBinBlocks[3]);
278 //postbuf("%d %d %d %p %p %p\n", word, bitPosition, index, binBits, bin->Prev(), bin->Next());
280 AllocChunkPtr candidate;
281 size_t maxsize = 0;
282 for (candidate = bin->Prev(); candidate != bin; candidate = candidate->Prev()) {
283 size_t candidate_size = candidate->Size();
284 maxsize = sc_max(maxsize, candidate_size);
285 //postbuf(" %d %d\n", maxsize, candidate_size);
288 /*for (int i=0; i<kNumAllocBins; ++i) {
289 bin = mBins + i;
290 if (bin->Prev() != bin) {
291 postbuf("* %d %d\n", i, bin->Prev()->Size());
294 return maxsize;
297 void* AllocPool::Alloc(size_t inReqSize)
299 #ifdef DISABLE_MEMORY_POOLS
300 return malloc(inReqSize);
301 #endif
303 // OK it has a lot of gotos, but these remove a whole lot of common code
304 // that was obfuscating the original version of this function.
305 // So here I am choosing the OnceAndOnlyOnce principle over the caveats on gotos.
306 // The gotos only jump forward and only to the exit paths of the function
308 // The old bin block scheme has been replaced by 4 x 32 bit words so that each bin has a bit
309 // and the next bin is found using a count leading zeroes instruction. Much faster.
310 // Also now each bin's flag can be kept accurate. This simplifies the searching code quite a bit.
312 // Also fwiw, changed 'victim' in the original code to 'candidate'. 'victim' just bothered me.
315 AllocChunkPtr candidate; /* inspected/selected chunk */
316 size_t candidate_size; /* its size */
317 AllocChunkPtr remainder; /* remainder from a split */
318 int32 remainder_size; /* its size */
319 AllocAreaPtr area;
320 size_t areaSize;
322 size_t size = RequestToSize(inReqSize);
323 int index = BinIndex(size);
324 assert(index < 128);
325 AllocChunkPtr bin = mBins + index;
327 check_pool();
329 /* Check for exact match in a bin */
330 if (index < kMaxSmallBin) { /* Faster version for small requests */
331 /* No traversal or size check necessary for small bins. */
332 candidate = bin->Prev();
334 /* Also scan the next one, since it would have a remainder < kMinAllocSize */
335 if (candidate == bin) candidate = (++bin)->Prev();
336 if (candidate != bin) {
337 candidate_size = candidate->Size();
338 goto found_exact_fit;
341 index += 2; /* Set for bin scan below. We've already scanned 2 bins. */
342 } else {
343 for (candidate = bin->Prev(); candidate != bin; candidate = candidate->Prev()) {
345 candidate_size = candidate->Size();
346 remainder_size = (int)(candidate_size - size);
347 if (remainder_size >= (int32)kMinAllocSize) { /* too big */
348 --index; /* adjust to rescan below after checking last remainder */
349 break;
350 } else if (remainder_size >= 0) { /* exact fit */
351 goto found_exact_fit;
354 ++index;
357 for(; (index = NextFullBin(index)) >= 0; ++index) {
358 bin = mBins + index;
360 /* Find and use first big enough chunk ... */
361 for (candidate = bin->Prev(); candidate != bin; candidate = candidate->Prev()) {
362 candidate_size = candidate->Size();
363 remainder_size = (int)(candidate_size - size);
364 if (remainder_size >= (int32)kMinAllocSize) { /* split */
365 UnlinkFree(candidate);
366 goto found_bigger_fit;
367 } else if (remainder_size >= 0) goto found_exact_fit;
370 check_pool();
372 if (mAreaMoreSize == 0) { /* pool has a non-growable area */
373 if (mAreas != NULL /* fixed size area exhausted */
374 || size > mAreaInitSize) /* too big anyway */
375 goto found_nothing;
376 areaSize = mAreaInitSize;
377 goto split_new_area;
380 if (size > mAreaMoreSize) {
381 areaSize = size;
382 goto whole_new_area;
383 } else {
384 areaSize = mAreaMoreSize;
385 goto split_new_area;
388 // exit paths:
389 found_nothing:
390 //ipostbuf("alloc failed. size: %d\n", inReqSize);
391 throw std::runtime_error(std::string("alloc failed, increase server's memory allocation (e.g. via ServerOptions)"));
393 whole_new_area:
394 //ipostbuf("whole_new_area\n");
395 area = NewArea(areaSize);
396 if (!area) return 0;
397 candidate = &area->mChunk;
398 candidate_size = candidate->Size();
399 goto return_chunk;
401 split_new_area:
402 //ipostbuf("split_new_area\n");
403 area = NewArea(areaSize);
404 if (!area) return 0;
405 candidate = &area->mChunk;
406 candidate_size = candidate->Size();
407 remainder_size = (int)(areaSize - size);
408 // FALL THROUGH
409 found_bigger_fit:
410 //ipostbuf("found_bigger_fit\n");
411 remainder = candidate->ChunkAtOffset(size);
412 remainder->SetSizeFree(remainder_size);
413 candidate_size -= remainder_size;
414 LinkFree(remainder);
415 goto return_chunk;
417 found_exact_fit:
418 check_pool();
419 UnlinkFree(candidate);
420 // FALL THROUGH
421 return_chunk:
423 candidate->SetSizeInUse(candidate_size);
424 check_malloced_chunk(candidate, candidate_size);
425 check_pool();
426 garbage_fill(candidate);
427 return candidate->ToPtr();
431 void* AllocPool::Realloc(void* inPtr, size_t inReqSize)
433 #ifdef DISABLE_MEMORY_POOLS
434 return realloc(inPtr, inReqSize);
435 #endif
438 void *outPtr;
439 AllocChunkPtr prev;
440 check_pool();
441 bool docopy = false;
443 /* realloc of null is supposed to be same as malloc */
444 if (inPtr == 0) return Alloc(inReqSize);
446 AllocChunkPtr oldChunk = MemToChunk(inPtr);
447 AllocChunkPtr newChunk = oldChunk;
448 size_t oldsize = oldChunk->Size();
449 size_t newsize = oldsize;
450 size_t size = RequestToSize(inReqSize);
451 size_t nextsize, prevsize;
452 check_inuse_chunk(oldChunk);
454 if (oldsize < size) {
455 /* Try expanding forward */
456 AllocChunkPtr next = oldChunk->NextChunk();
457 if (!next->InUse()) {
458 nextsize = next->Size();
459 /* Forward into next chunk */
460 if (nextsize + newsize >= size) {
461 UnlinkFree(next);
462 newsize += nextsize;
463 goto split;
465 } else {
466 next = 0;
467 nextsize = 0;
471 /* Try shifting backwards. */
472 prev = oldChunk->PrevChunk();
473 if (!prev->InUse()) {
474 prevsize = prev->Size();
476 /* try forward + backward first to save a later consolidation */
477 if (next != 0) {
478 /* into next chunk */
479 if (nextsize + prevsize + newsize >= size) {
480 newsize += nextsize + prevsize;
481 UnlinkFree(next);
482 goto alloc_prev;
486 /* backward only */
487 if (prev != 0 && prevsize + newsize >= size) {
488 newsize += prevsize;
489 goto alloc_prev;
493 /* Must allocate */
495 outPtr = Alloc(inReqSize);
497 check_pool();
498 if (outPtr == 0) {
499 //ipostbuf("realloc failed. size: %d\n", inReqSize);
500 throw std::runtime_error(std::string("realloc failed, increase server's memory allocation (e.g. via ServerOptions)"));
503 /* Otherwise copy, free, and exit */
504 memcpy(outPtr, inPtr, oldsize - sizeof(AllocChunk));
505 Free(inPtr);
506 return outPtr;
507 } else goto split;
509 alloc_prev:
510 UnlinkFree(prev);
511 newChunk = prev;
512 docopy = true;
513 // FALL THROUGH
514 split: /* split off extra room in old or expanded chunk */
515 //check_pool();
516 if (newsize - size >= kMinAllocSize) { /* split off remainder */
517 size_t remainder_size = newsize - size;
518 AllocChunkPtr remainder = newChunk->ChunkAtOffset(size);
519 remainder->SetSizeInUse(remainder_size);
520 newChunk->SetSizeInUse(size);
521 Free(remainder->ToPtr()); /* let free() deal with it */
522 } else {
523 newChunk->SetSizeInUse(newsize);
525 outPtr = newChunk->ToPtr();
526 if (docopy) {
527 memmove(outPtr, inPtr, oldsize - sizeof(AllocChunk));
529 check_inuse_chunk(newChunk);
530 check_pool();
531 garbage_fill(newChunk);
532 return outPtr;
535 void AllocPool::LinkFree(AllocChunkPtr inChunk)
537 size_t size = inChunk->Size();
538 int index = BinIndex(size);
540 AllocChunkPtr bin = mBins + index;
542 if (index < kNumSmallBins || bin->IsEmpty()) {
543 inChunk->InsertAfter(bin);
544 MarkBinBlock(index);
545 } else {
546 AllocChunkPtr link = bin->Next();
547 while (link != bin && size < link->Size()) link = link->Next();
548 inChunk->InsertBefore(link);
552 void AllocPool::DoCheckArea(AllocAreaPtr area)
554 assert(area->mChunk.PrevInUse());
556 AllocChunkPtr p = &area->mChunk;
557 while (p->mSize != kChunkInUse) {
558 if (p->InUse()) {
559 DoCheckInUseChunk(p);
560 } else {
561 DoCheckFreeChunk(p);
563 p = p->NextChunk();
567 void AllocPool::DoCheckBin(AllocChunkPtr bin, long index)
569 AllocChunkPtr p = bin->Next();
571 while (p != bin) {
572 assert(BinIndex(p->Size()) == index);
573 DoCheckFreeChunk(p);
574 p = p->Next();
579 void AllocPool::DoCheckPool()
581 AllocAreaPtr area = mAreas;
582 if (area) {
583 do {
584 AllocAreaPtr nextarea = area->mNext;
585 DoCheckArea(area);
586 area = nextarea;
587 } while (area != mAreas);
590 for (int i=0; i<kNumAllocBins; ++i) {
591 AllocChunkPtr b = mBins + i;
592 DoCheckBin(b, i);
597 void AllocPool::DoCheckChunk(AllocChunkPtr p)
599 #ifndef NDEBUG
600 size_t size = p->Size();
601 //size_t maxsize = mAreaInitSize > mAreaMoreSize ? mAreaInitSize : mAreaMoreSize;
602 // assert(size < maxsize);
604 AllocChunkPtr next = p->ChunkAtOffset(size);
605 #endif
606 assert(p->mSize == next->mPrevSize);
610 void AllocPool::DoCheckFreeChunk(AllocChunkPtr p)
612 size_t size = p->Size();
613 #ifndef NDEBUG
614 AllocChunkPtr next = p->ChunkAtOffset(size);
615 #endif
616 DoCheckChunk(p);
618 /* Check whether it claims to be free ... */
619 assert(!p->InUse());
621 /* Unless an end marker, must have OK fields */
622 if (size >= kMinAllocSize)
624 assert((size & kAlignMask) == 0);
625 assert(aligned_OK(p->ToPtr()));
626 /* ... and is fully consolidated */
627 assert(p->PrevInUse());
628 assert(next->InUse());
630 /* ... and has minimally sane links */
631 assert(p->Next()->Prev() == p);
632 assert(p->Prev()->Next() == p);
634 else /* end markers are always of size 0 */
635 assert(size == 0);
638 void AllocPool::DoCheckInUseChunk(AllocChunkPtr p)
640 size_t size = p->Size();
641 AllocChunkPtr next = p->NextChunk();
643 DoCheckChunk(p);
645 /* Check whether it claims to be in use ... */
646 assert(p->InUse());
648 /* ... and is surrounded by OK chunks.
649 Since more things can be checked with free chunks than inuse ones,
650 if an inuse chunk borders them and debug is on, it's worth doing them.
652 if (!p->PrevInUse()) {
653 size_t prevsize = p->PrevSize();
654 if (prevsize > 0) {
655 AllocChunkPtr prv = p->PrevChunk();
656 assert(prv->NextChunk() == p);
657 DoCheckFreeChunk(prv);
660 if (!p->ChunkAtOffset(size)->InUse()) {
661 DoCheckFreeChunk(next);
665 void AllocPool::DoCheckAllocedChunk(AllocChunkPtr p, size_t s)
667 #ifndef NDEBUG
668 size_t size = p->Size();
669 long room = size - s;
670 #endif
672 DoCheckInUseChunk(p);
674 /* Legal size ... */
675 assert(size >= kMinAllocSize);
676 assert((size & kAlignMask) == 0);
677 assert(room >= 0);
678 assert(room < kMinAllocSize);
680 /* ... and alignment */
681 assert(aligned_OK(p->ToPtr()));
684 /* ... and was allocated at front of an available chunk */
685 assert(p->PrevInUse()); // huh?? - jmc
689 void AllocPool::DoGarbageFill(AllocChunkPtr p)
691 long size = (p->Size() - sizeof(AllocChunk));
692 DoGarbageFill(p, size);
695 void AllocPool::DoGarbageFill(AllocChunkPtr p, long size)
697 size /= sizeof(long);
698 long *ptr = (long*)p->ToPtr();
699 for (int i=0; i<size; ++i) {
700 ptr[i] = 0xA3A56955;