2 SuperCollider real time audio synthesis system
3 Copyright (c) 2002 James McCartney. All rights reserved.
4 http://www.audiosynth.com
6 This program is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation; either version 2 of the License, or
9 (at your option) any later version.
11 This program is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with this program; if not, write to the Free Software
18 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
23 #include "SC_AllocPool.h"
24 #include "SC_BoundsMacros.h"
29 Requests are `small' if both the corresponding and the next bin are small
33 #define check_pool() DoCheckPool()
34 #define check_free_chunk(P) DoCheckFreeChunk(P)
35 #define check_inuse_chunk(P) DoCheckInUseChunk(P)
36 #define check_chunk(P) DoCheckChunk(P)
37 #define check_malloced_chunk(P,N) DoCheckAllocedChunk(P,N)
38 #define garbage_fill(P) DoGarbageFill(P)
41 #define check_free_chunk(P)
42 #define check_inuse_chunk(P)
43 #define check_chunk(P)
44 #define check_malloced_chunk(P,N)
45 #define garbage_fill(P)
48 #define aligned_OK(m) ((((unsigned long)(m)) & kAlignMask) == 0)
51 void* allocmem(AllocPool *pool, int32 size);
52 void* allocmem(AllocPool *pool, int32 size)
54 return pool->Alloc(size);
57 void* reallocmem(AllocPool *pool, void* ptr, int32 size);
58 void* reallocmem(AllocPool *pool, void* ptr, int32 size)
60 return pool->Realloc(ptr, size);
63 void freemem(AllocPool *pool, void* ptr);
64 void freemem(AllocPool *pool, void* ptr)
69 void AllocPool::InitAlloc()
71 if (mAreaInitSize
== 0) return;
73 /* alloc initial area */
74 NewArea(mAreaInitSize
);
76 AllocAreaPtr area
= mAreas
;
77 AllocChunkPtr chunk
= &area
->mChunk
;
83 void AllocPool::InitBins()
85 for (int i
=0; i
<kNumAllocBins
; ++i
) {
88 for (int i
=0; i
<4; ++i
) {
93 AllocPool::AllocPool(NewAreaFunc inAllocArea
, FreeAreaFunc inFreeArea
,
94 size_t inAreaInitSize
, size_t inAreaMoreSize
)
97 mAreaInitSize
= inAreaInitSize
;
98 mAreaMoreSize
= inAreaMoreSize
;
99 mAllocArea
= inAllocArea
;
100 mFreeArea
= inFreeArea
;
107 AllocPool::~AllocPool()
112 void AllocPool::FreeAll()
115 AllocAreaPtr area
= mAreas
;
117 AllocAreaPtr firstArea
= area
;
119 AllocAreaPtr nextarea
= area
->mNext
;
120 (mFreeArea
)(area
->mUnalignedPointerToThis
);
122 } while (area
!= firstArea
);
129 void AllocPool::FreeAllInternal()
134 AllocAreaPtr area
= mAreas
;
136 AllocAreaPtr firstArea
= area
;
138 AllocAreaPtr nextarea
= area
->mNext
;
139 size_t size
= area
->mSize
;
140 AllocChunkPtr chunk
= &area
->mChunk
;
141 chunk
->SetSizeFree(size
);
142 chunk
->SetNeighborsInUse(size
);
145 } while (area
!= firstArea
);
150 void AllocPool::Reinit()
156 void AllocPool::Free(void *inPtr
)
158 #ifdef DISABLE_MEMORY_POOLS
164 if (inPtr
== 0) return; /* free(0) has no effect */
166 AllocChunkPtr chunk
= MemToChunk(inPtr
);
168 check_inuse_chunk(chunk
);
171 size_t size
= chunk
->Size();
173 if (!chunk
->PrevInUse()) /* consolidate backward */
175 size_t prevSize
= chunk
->PrevSize();
176 chunk
= chunk
->ChunkAtOffset(0L-prevSize
);
181 AllocChunkPtr next
= chunk
->ChunkAtOffset(size
);
182 if (!next
->InUse()) /* consolidate forward */
184 size
+= next
->Size();
188 chunk
->SetSizeFree(size
);
189 if (mAreaMoreSize
&& chunk
->IsArea()) {
190 // whole area is free
200 AllocAreaPtr
AllocPool::NewArea(size_t inAreaSize
)
202 void *ptr
= (AllocAreaPtr
)(mAllocArea
)(inAreaSize
+ kAreaOverhead
);
205 throw std::runtime_error(std::string("Could not allocate new area"));
207 // AllocAreaPtr area = (AllocAreaPtr)((unsigned long)ptr & ~kAlignMask);
208 AllocAreaPtr area
= (AllocAreaPtr
)(((unsigned long)ptr
+ kAlignMask
) & ~kAlignMask
);
209 assert((area
>= ptr
) && ((void*)((unsigned long)area
& ~kAlignMask
) == area
));
211 area
->mUnalignedPointerToThis
= ptr
;
215 area
->mNext
= mAreas
;
216 area
->mPrev
= mAreas
->mPrev
;
217 area
->mNext
->mPrev
= area
;
218 area
->mPrev
->mNext
= area
;
226 area
->mSize
= inAreaSize
;
227 area
->mChunk
.BeEmpty();
228 area
->mChunk
.SetNeighborsInUse(inAreaSize
);
229 area
->mChunk
.SetSizeFree(inAreaSize
);
234 void AllocPool::FreeArea(AllocChunkPtr chunk
)
236 AllocAreaPtr area
= (AllocAreaPtr
)((char*)chunk
- sizeof(AllocAreaHdr
));
238 if (area
->mNext
== area
) {
242 mAreas
= area
->mPrev
->mNext
= area
->mNext
;
243 area
->mNext
->mPrev
= area
->mPrev
;
246 (mFreeArea
)(area
->mUnalignedPointerToThis
);
250 size_t AllocPool::TotalFree()
253 for (int i
=0; i
<kNumAllocBins
; ++i
) {
254 AllocChunkPtr bin
= mBins
+ i
;
255 if (bin
->Prev() != bin
) {
256 for (AllocChunkPtr candidate
= bin
->Prev(); candidate
!= bin
; candidate
= candidate
->Prev()) {
257 total
+= candidate
->Size();
264 size_t AllocPool::LargestFreeChunk()
267 for (int i
=3; i
>=0; --i
) {
273 int binBits
= (int)mBinBlocks
[word
];
274 int bitPosition
= NUMBITS(binBits
) - 1;
275 int index
= (word
<< 5) + bitPosition
;
276 AllocChunkPtr bin
= mBins
+ index
;
277 //postbuf("** %p %p %p %p\n", mBinBlocks[0], mBinBlocks[1], mBinBlocks[2], mBinBlocks[3]);
278 //postbuf("%d %d %d %p %p %p\n", word, bitPosition, index, binBits, bin->Prev(), bin->Next());
280 AllocChunkPtr candidate
;
282 for (candidate
= bin
->Prev(); candidate
!= bin
; candidate
= candidate
->Prev()) {
283 size_t candidate_size
= candidate
->Size();
284 maxsize
= sc_max(maxsize
, candidate_size
);
285 //postbuf(" %d %d\n", maxsize, candidate_size);
288 /*for (int i=0; i<kNumAllocBins; ++i) {
290 if (bin->Prev() != bin) {
291 postbuf("* %d %d\n", i, bin->Prev()->Size());
297 void* AllocPool::Alloc(size_t inReqSize
)
299 #ifdef DISABLE_MEMORY_POOLS
300 return malloc(inReqSize
);
303 // OK it has a lot of gotos, but these remove a whole lot of common code
304 // that was obfuscating the original version of this function.
305 // So here I am choosing the OnceAndOnlyOnce principle over the caveats on gotos.
306 // The gotos only jump forward and only to the exit paths of the function
308 // The old bin block scheme has been replaced by 4 x 32 bit words so that each bin has a bit
309 // and the next bin is found using a count leading zeroes instruction. Much faster.
310 // Also now each bin's flag can be kept accurate. This simplifies the searching code quite a bit.
312 // Also fwiw, changed 'victim' in the original code to 'candidate'. 'victim' just bothered me.
315 AllocChunkPtr candidate
; /* inspected/selected chunk */
316 size_t candidate_size
; /* its size */
317 AllocChunkPtr remainder
; /* remainder from a split */
318 int32 remainder_size
; /* its size */
322 size_t size
= RequestToSize(inReqSize
);
323 int index
= BinIndex(size
);
325 AllocChunkPtr bin
= mBins
+ index
;
329 /* Check for exact match in a bin */
330 if (index
< kMaxSmallBin
) { /* Faster version for small requests */
331 /* No traversal or size check necessary for small bins. */
332 candidate
= bin
->Prev();
334 /* Also scan the next one, since it would have a remainder < kMinAllocSize */
335 if (candidate
== bin
) candidate
= (++bin
)->Prev();
336 if (candidate
!= bin
) {
337 candidate_size
= candidate
->Size();
338 goto found_exact_fit
;
341 index
+= 2; /* Set for bin scan below. We've already scanned 2 bins. */
343 for (candidate
= bin
->Prev(); candidate
!= bin
; candidate
= candidate
->Prev()) {
345 candidate_size
= candidate
->Size();
346 remainder_size
= (int)(candidate_size
- size
);
347 if (remainder_size
>= (int32
)kMinAllocSize
) { /* too big */
348 --index
; /* adjust to rescan below after checking last remainder */
350 } else if (remainder_size
>= 0) { /* exact fit */
351 goto found_exact_fit
;
357 for(; (index
= NextFullBin(index
)) >= 0; ++index
) {
360 /* Find and use first big enough chunk ... */
361 for (candidate
= bin
->Prev(); candidate
!= bin
; candidate
= candidate
->Prev()) {
362 candidate_size
= candidate
->Size();
363 remainder_size
= (int)(candidate_size
- size
);
364 if (remainder_size
>= (int32
)kMinAllocSize
) { /* split */
365 UnlinkFree(candidate
);
366 goto found_bigger_fit
;
367 } else if (remainder_size
>= 0) goto found_exact_fit
;
372 if (mAreaMoreSize
== 0) { /* pool has a non-growable area */
373 if (mAreas
!= NULL
/* fixed size area exhausted */
374 || size
> mAreaInitSize
) /* too big anyway */
376 areaSize
= mAreaInitSize
;
380 if (size
> mAreaMoreSize
) {
384 areaSize
= mAreaMoreSize
;
390 //ipostbuf("alloc failed. size: %d\n", inReqSize);
391 throw std::runtime_error(std::string("alloc failed, increase server's memory allocation (e.g. via ServerOptions)"));
394 //ipostbuf("whole_new_area\n");
395 area
= NewArea(areaSize
);
397 candidate
= &area
->mChunk
;
398 candidate_size
= candidate
->Size();
402 //ipostbuf("split_new_area\n");
403 area
= NewArea(areaSize
);
405 candidate
= &area
->mChunk
;
406 candidate_size
= candidate
->Size();
407 remainder_size
= (int)(areaSize
- size
);
410 //ipostbuf("found_bigger_fit\n");
411 remainder
= candidate
->ChunkAtOffset(size
);
412 remainder
->SetSizeFree(remainder_size
);
413 candidate_size
-= remainder_size
;
419 UnlinkFree(candidate
);
423 candidate
->SetSizeInUse(candidate_size
);
424 check_malloced_chunk(candidate
, candidate_size
);
426 garbage_fill(candidate
);
427 return candidate
->ToPtr();
431 void* AllocPool::Realloc(void* inPtr
, size_t inReqSize
)
433 #ifdef DISABLE_MEMORY_POOLS
434 return realloc(inPtr
, inReqSize
);
443 /* realloc of null is supposed to be same as malloc */
444 if (inPtr
== 0) return Alloc(inReqSize
);
446 AllocChunkPtr oldChunk
= MemToChunk(inPtr
);
447 AllocChunkPtr newChunk
= oldChunk
;
448 size_t oldsize
= oldChunk
->Size();
449 size_t newsize
= oldsize
;
450 size_t size
= RequestToSize(inReqSize
);
451 size_t nextsize
, prevsize
;
452 check_inuse_chunk(oldChunk
);
454 if (oldsize
< size
) {
455 /* Try expanding forward */
456 AllocChunkPtr next
= oldChunk
->NextChunk();
457 if (!next
->InUse()) {
458 nextsize
= next
->Size();
459 /* Forward into next chunk */
460 if (nextsize
+ newsize
>= size
) {
471 /* Try shifting backwards. */
472 prev
= oldChunk
->PrevChunk();
473 if (!prev
->InUse()) {
474 prevsize
= prev
->Size();
476 /* try forward + backward first to save a later consolidation */
478 /* into next chunk */
479 if (nextsize
+ prevsize
+ newsize
>= size
) {
480 newsize
+= nextsize
+ prevsize
;
487 if (prev
!= 0 && prevsize
+ newsize
>= size
) {
495 outPtr
= Alloc(inReqSize
);
499 //ipostbuf("realloc failed. size: %d\n", inReqSize);
500 throw std::runtime_error(std::string("realloc failed, increase server's memory allocation (e.g. via ServerOptions)"));
503 /* Otherwise copy, free, and exit */
504 memcpy(outPtr
, inPtr
, oldsize
- sizeof(AllocChunk
));
514 split
: /* split off extra room in old or expanded chunk */
516 if (newsize
- size
>= kMinAllocSize
) { /* split off remainder */
517 size_t remainder_size
= newsize
- size
;
518 AllocChunkPtr remainder
= newChunk
->ChunkAtOffset(size
);
519 remainder
->SetSizeInUse(remainder_size
);
520 newChunk
->SetSizeInUse(size
);
521 Free(remainder
->ToPtr()); /* let free() deal with it */
523 newChunk
->SetSizeInUse(newsize
);
525 outPtr
= newChunk
->ToPtr();
527 memmove(outPtr
, inPtr
, oldsize
- sizeof(AllocChunk
));
529 check_inuse_chunk(newChunk
);
531 garbage_fill(newChunk
);
535 void AllocPool::LinkFree(AllocChunkPtr inChunk
)
537 size_t size
= inChunk
->Size();
538 int index
= BinIndex(size
);
540 AllocChunkPtr bin
= mBins
+ index
;
542 if (index
< kNumSmallBins
|| bin
->IsEmpty()) {
543 inChunk
->InsertAfter(bin
);
546 AllocChunkPtr link
= bin
->Next();
547 while (link
!= bin
&& size
< link
->Size()) link
= link
->Next();
548 inChunk
->InsertBefore(link
);
552 void AllocPool::DoCheckArea(AllocAreaPtr area
)
554 assert(area
->mChunk
.PrevInUse());
556 AllocChunkPtr p
= &area
->mChunk
;
557 while (p
->mSize
!= kChunkInUse
) {
559 DoCheckInUseChunk(p
);
567 void AllocPool::DoCheckBin(AllocChunkPtr bin
, long index
)
569 AllocChunkPtr p
= bin
->Next();
572 assert(BinIndex(p
->Size()) == index
);
579 void AllocPool::DoCheckPool()
581 AllocAreaPtr area
= mAreas
;
584 AllocAreaPtr nextarea
= area
->mNext
;
587 } while (area
!= mAreas
);
590 for (int i
=0; i
<kNumAllocBins
; ++i
) {
591 AllocChunkPtr b
= mBins
+ i
;
597 void AllocPool::DoCheckChunk(AllocChunkPtr p
)
600 size_t size
= p
->Size();
601 //size_t maxsize = mAreaInitSize > mAreaMoreSize ? mAreaInitSize : mAreaMoreSize;
602 // assert(size < maxsize);
604 AllocChunkPtr next
= p
->ChunkAtOffset(size
);
606 assert(p
->mSize
== next
->mPrevSize
);
610 void AllocPool::DoCheckFreeChunk(AllocChunkPtr p
)
612 size_t size
= p
->Size();
614 AllocChunkPtr next
= p
->ChunkAtOffset(size
);
618 /* Check whether it claims to be free ... */
621 /* Unless an end marker, must have OK fields */
622 if (size
>= kMinAllocSize
)
624 assert((size
& kAlignMask
) == 0);
625 assert(aligned_OK(p
->ToPtr()));
626 /* ... and is fully consolidated */
627 assert(p
->PrevInUse());
628 assert(next
->InUse());
630 /* ... and has minimally sane links */
631 assert(p
->Next()->Prev() == p
);
632 assert(p
->Prev()->Next() == p
);
634 else /* end markers are always of size 0 */
638 void AllocPool::DoCheckInUseChunk(AllocChunkPtr p
)
640 size_t size
= p
->Size();
641 AllocChunkPtr next
= p
->NextChunk();
645 /* Check whether it claims to be in use ... */
648 /* ... and is surrounded by OK chunks.
649 Since more things can be checked with free chunks than inuse ones,
650 if an inuse chunk borders them and debug is on, it's worth doing them.
652 if (!p
->PrevInUse()) {
653 size_t prevsize
= p
->PrevSize();
655 AllocChunkPtr prv
= p
->PrevChunk();
656 assert(prv
->NextChunk() == p
);
657 DoCheckFreeChunk(prv
);
660 if (!p
->ChunkAtOffset(size
)->InUse()) {
661 DoCheckFreeChunk(next
);
665 void AllocPool::DoCheckAllocedChunk(AllocChunkPtr p
, size_t s
)
668 size_t size
= p
->Size();
669 long room
= size
- s
;
672 DoCheckInUseChunk(p
);
675 assert(size
>= kMinAllocSize
);
676 assert((size
& kAlignMask
) == 0);
678 assert(room
< kMinAllocSize
);
680 /* ... and alignment */
681 assert(aligned_OK(p
->ToPtr()));
684 /* ... and was allocated at front of an available chunk */
685 assert(p
->PrevInUse()); // huh?? - jmc
689 void AllocPool::DoGarbageFill(AllocChunkPtr p
)
691 long size
= (p
->Size() - sizeof(AllocChunk
));
692 DoGarbageFill(p
, size
);
695 void AllocPool::DoGarbageFill(AllocChunkPtr p
, long size
)
697 size
/= sizeof(long);
698 long *ptr
= (long*)p
->ToPtr();
699 for (int i
=0; i
<size
; ++i
) {