1 /* zn_malloc.c - zone-based malloc routines */
2 /* $OpenLDAP: pkg/ldap/servers/slapd/zn_malloc.c,v 1.11.2.3 2008/02/11 23:26:45 kurt Exp $*/
3 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
5 * Copyright 2003-2008 The OpenLDAP Foundation.
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted only as authorized by the OpenLDAP
12 * A copy of this license is available in the file LICENSE in the
13 * top-level directory of the distribution or, alternatively, at
14 * <http://www.OpenLDAP.org/license.html>.
16 /* Portions Copyright 2004 IBM Corporation
17 * All rights reserved.
18 * Redistribution and use in source and binary forms, with or without
19 * modification, are permitted only as authorized by the OpenLDAP
23 * This work originally developed by Jong-Hyuk Choi for inclusion in
30 #include <ac/string.h>
31 #include <sys/types.h>
36 #ifdef SLAP_ZONE_ALLOC
40 static int slap_zone_cmp(const void *v1
, const void *v2
);
41 void * slap_replenish_zopool(void *ctx
);
44 slap_zo_release(void *data
)
46 struct zone_object
*zo
= (struct zone_object
*)data
;
55 struct zone_heap
*zh
= ctx
;
56 int pad
= 2*sizeof(int)-1, pad_shift
;
57 int order_start
= -1, i
, j
;
58 struct zone_object
*zo
;
63 } while (pad_shift
>>= 1);
65 ldap_pvt_thread_mutex_lock( &zh
->zh_mutex
);
66 for (i
= 0; i
< zh
->zh_zoneorder
- order_start
+ 1; i
++) {
67 zo
= LDAP_LIST_FIRST(&zh
->zh_free
[i
]);
69 struct zone_object
*zo_tmp
= zo
;
70 zo
= LDAP_LIST_NEXT(zo
, zo_link
);
71 LDAP_LIST_REMOVE(zo_tmp
, zo_link
);
72 LDAP_LIST_INSERT_HEAD(&zh
->zh_zopool
, zo_tmp
, zo_link
);
77 for (i
= 0; i
< zh
->zh_numzones
; i
++) {
78 for (j
= 0; j
< zh
->zh_zoneorder
- order_start
+ 1; j
++) {
79 ch_free(zh
->zh_maps
[i
][j
]);
81 ch_free(zh
->zh_maps
[i
]);
82 munmap(zh
->zh_zones
[i
], zh
->zh_zonesize
);
83 ldap_pvt_thread_rdwr_destroy(&zh
->zh_znlock
[i
]);
86 ch_free(zh
->zh_zones
);
87 ch_free(zh
->zh_seqno
);
88 ch_free(zh
->zh_znlock
);
90 avl_free(zh
->zh_zonetree
, slap_zo_release
);
92 zo
= LDAP_LIST_FIRST(&zh
->zh_zopool
);
94 struct zone_object
*zo_tmp
= zo
;
95 zo
= LDAP_LIST_NEXT(zo
, zo_link
);
96 if (!zo_tmp
->zo_blockhead
) {
97 LDAP_LIST_REMOVE(zo_tmp
, zo_link
);
100 zo
= LDAP_LIST_FIRST(&zh
->zh_zopool
);
102 struct zone_object
*zo_tmp
= zo
;
103 zo
= LDAP_LIST_NEXT(zo
, zo_link
);
106 ldap_pvt_thread_mutex_unlock(&zh
->zh_mutex
);
107 ldap_pvt_thread_rdwr_destroy(&zh
->zh_lock
);
108 ldap_pvt_thread_mutex_destroy(&zh
->zh_mutex
);
120 struct zone_heap
*zh
= NULL
;
122 int pad
= 2*sizeof(int)-1, pad_shift
;
124 int order
= -1, order_start
= -1, order_end
= -1;
126 struct zone_object
*zo
;
128 Debug(LDAP_DEBUG_NONE
,
129 "--> slap_zn_mem_create: initsize=%d, maxsize=%d\n",
130 initsize
, maxsize
, 0);
131 Debug(LDAP_DEBUG_NONE
,
132 "++> slap_zn_mem_create: deltasize=%d, zonesize=%d\n",
133 deltasize
, zonesize
, 0);
135 zh
= (struct zone_heap
*)ch_calloc(1, sizeof(struct zone_heap
));
137 zh
->zh_fd
= open("/dev/zero", O_RDWR
);
140 zh
->zh_zonesize
= zonesize
;
142 zh
->zh_zonesize
= SLAP_ZONE_SIZE
;
145 zpad
= zh
->zh_zonesize
- 1;
146 zh
->zh_numzones
= ((initsize
+ zpad
) & ~zpad
) / zh
->zh_zonesize
;
148 if ( maxsize
&& maxsize
>= initsize
) {
149 zh
->zh_maxzones
= ((maxsize
+ zpad
) & ~zpad
) / zh
->zh_zonesize
;
151 zh
->zh_maxzones
= ((initsize
+ zpad
) & ~zpad
) / zh
->zh_zonesize
;
155 zh
->zh_deltazones
= ((deltasize
+ zpad
) & ~zpad
) / zh
->zh_zonesize
;
157 zh
->zh_deltazones
= ((SLAP_ZONE_DELTA
+zpad
) & ~zpad
) / zh
->zh_zonesize
;
160 size_shift
= zh
->zh_zonesize
- 1;
163 } while (size_shift
>>= 1);
168 } while (pad_shift
>>= 1);
170 order
= order_end
- order_start
+ 1;
172 zh
->zh_zones
= (void **)ch_malloc(zh
->zh_maxzones
* sizeof(void*));
173 zh
->zh_znlock
= (ldap_pvt_thread_rdwr_t
*)ch_malloc(
174 zh
->zh_maxzones
* sizeof(ldap_pvt_thread_rdwr_t
*));
175 zh
->zh_maps
= (unsigned char ***)ch_malloc(
176 zh
->zh_maxzones
* sizeof(unsigned char**));
178 zh
->zh_zoneorder
= order_end
;
179 zh
->zh_free
= (struct zh_freelist
*)
180 ch_malloc(order
* sizeof(struct zh_freelist
));
181 zh
->zh_seqno
= (unsigned long *)ch_calloc(zh
->zh_maxzones
,
182 sizeof(unsigned long));
183 for (i
= 0; i
< order
; i
++) {
184 LDAP_LIST_INIT(&zh
->zh_free
[i
]);
186 LDAP_LIST_INIT(&zh
->zh_zopool
);
188 for (i
= 0; i
< zh
->zh_numzones
; i
++) {
189 zh
->zh_zones
[i
] = mmap(0, zh
->zh_zonesize
, PROT_READ
| PROT_WRITE
,
190 MAP_PRIVATE
, zh
->zh_fd
, 0);
191 zh
->zh_maps
[i
] = (unsigned char **)
192 ch_malloc(order
* sizeof(unsigned char *));
193 for (j
= 0; j
< order
; j
++) {
194 int shiftamt
= order_start
+ 1 + j
;
195 int nummaps
= zh
->zh_zonesize
>> shiftamt
;
198 if (!nummaps
) nummaps
= 1;
199 zh
->zh_maps
[i
][j
] = (unsigned char *)ch_malloc(nummaps
);
200 memset(zh
->zh_maps
[i
][j
], 0, nummaps
);
203 if (LDAP_LIST_EMPTY(&zh
->zh_zopool
)) {
204 slap_replenish_zopool(zh
);
206 zo
= LDAP_LIST_FIRST(&zh
->zh_zopool
);
207 LDAP_LIST_REMOVE(zo
, zo_link
);
208 zo
->zo_ptr
= zh
->zh_zones
[i
];
210 LDAP_LIST_INSERT_HEAD(&zh
->zh_free
[order
-1], zo
, zo_link
);
212 if (LDAP_LIST_EMPTY(&zh
->zh_zopool
)) {
213 slap_replenish_zopool(zh
);
215 zo
= LDAP_LIST_FIRST(&zh
->zh_zopool
);
216 LDAP_LIST_REMOVE(zo
, zo_link
);
217 zo
->zo_ptr
= zh
->zh_zones
[i
];
218 zo
->zo_siz
= zh
->zh_zonesize
;
220 avl_insert(&zh
->zh_zonetree
, zo
, slap_zone_cmp
, avl_dup_error
);
221 ldap_pvt_thread_rdwr_init(&zh
->zh_znlock
[i
]);
224 LDAP_STAILQ_INIT(&zh
->zh_latency_history_queue
);
225 ldap_pvt_thread_mutex_init(&zh
->zh_mutex
);
226 ldap_pvt_thread_rdwr_init(&zh
->zh_lock
);
237 struct zone_heap
*zh
= ctx
;
238 ber_len_t size_shift
;
239 int pad
= 2*sizeof(int)-1, pad_shift
;
240 int order
= -1, order_start
= -1;
241 struct zone_object
*zo
, *zo_new
, *zo_left
, *zo_right
;
242 ber_len_t
*ptr
, *new;
247 Debug(LDAP_DEBUG_NONE
,
248 "--> slap_zn_malloc: size=%d\n", size
, 0, 0);
250 if (!zh
) return ber_memalloc_x(size
, NULL
);
252 /* round up to doubleword boundary */
253 size
+= 2*sizeof(ber_len_t
) + pad
;
256 size_shift
= size
- 1;
259 } while (size_shift
>>= 1);
264 } while (pad_shift
>>= 1);
268 ldap_pvt_thread_mutex_lock( &zh
->zh_mutex
);
269 for (i
= order
; i
<= zh
->zh_zoneorder
&&
270 LDAP_LIST_EMPTY(&zh
->zh_free
[i
-order_start
]); i
++);
273 zo_new
= LDAP_LIST_FIRST(&zh
->zh_free
[i
-order_start
]);
274 LDAP_LIST_REMOVE(zo_new
, zo_link
);
275 ptr
= zo_new
->zo_ptr
;
276 idx
= zo_new
->zo_idx
;
277 diff
= (unsigned long)((char*)ptr
-
278 (char*)zh
->zh_zones
[idx
]) >> (order
+ 1);
279 zh
->zh_maps
[idx
][order
-order_start
][diff
>>3] |= (1 << (diff
& 0x7));
280 *ptr
++ = zh
->zh_seqno
[idx
];
281 *ptr
++ = size
- 2*sizeof(ber_len_t
);
282 zo_new
->zo_ptr
= NULL
;
284 LDAP_LIST_INSERT_HEAD(&zh
->zh_zopool
, zo_new
, zo_link
);
285 ldap_pvt_thread_mutex_unlock( &zh
->zh_mutex
);
286 Debug(LDAP_DEBUG_NONE
, "slap_zn_malloc: returning 0x%x, 0x%x\n",
287 ptr
, (int)ptr
>>(zh
->zh_zoneorder
+1), 0);
289 } else if (i
<= zh
->zh_zoneorder
) {
290 for (j
= i
; j
> order
; j
--) {
291 zo_left
= LDAP_LIST_FIRST(&zh
->zh_free
[j
-order_start
]);
292 LDAP_LIST_REMOVE(zo_left
, zo_link
);
293 if (LDAP_LIST_EMPTY(&zh
->zh_zopool
)) {
294 slap_replenish_zopool(zh
);
296 zo_right
= LDAP_LIST_FIRST(&zh
->zh_zopool
);
297 LDAP_LIST_REMOVE(zo_right
, zo_link
);
298 zo_right
->zo_ptr
= zo_left
->zo_ptr
+ (1 << j
);
299 zo_right
->zo_idx
= zo_left
->zo_idx
;
300 Debug(LDAP_DEBUG_NONE
,
301 "slap_zn_malloc: split (left=0x%x, right=0x%x)\n",
302 zo_left
->zo_ptr
, zo_right
->zo_ptr
, 0);
303 if (j
== order
+ 1) {
304 ptr
= zo_left
->zo_ptr
;
305 diff
= (unsigned long)((char*)ptr
-
306 (char*)zh
->zh_zones
[zo_left
->zo_idx
]) >> (order
+1);
307 zh
->zh_maps
[zo_left
->zo_idx
][order
-order_start
][diff
>>3] |=
309 *ptr
++ = zh
->zh_seqno
[zo_left
->zo_idx
];
310 *ptr
++ = size
- 2*sizeof(ber_len_t
);
311 LDAP_LIST_INSERT_HEAD(
312 &zh
->zh_free
[j
-1-order_start
], zo_right
, zo_link
);
313 LDAP_LIST_INSERT_HEAD(&zh
->zh_zopool
, zo_left
, zo_link
);
314 ldap_pvt_thread_mutex_unlock( &zh
->zh_mutex
);
315 Debug(LDAP_DEBUG_NONE
,
316 "slap_zn_malloc: returning 0x%x, 0x%x\n",
317 ptr
, (int)ptr
>>(zh
->zh_zoneorder
+1), 0);
320 LDAP_LIST_INSERT_HEAD(
321 &zh
->zh_free
[j
-1-order_start
], zo_right
, zo_link
);
322 LDAP_LIST_INSERT_HEAD(
323 &zh
->zh_free
[j
-1-order_start
], zo_left
, zo_link
);
329 if ( zh
->zh_maxzones
< zh
->zh_numzones
+ zh
->zh_deltazones
) {
330 ldap_pvt_thread_mutex_unlock( &zh
->zh_mutex
);
331 Debug( LDAP_DEBUG_TRACE
,
332 "slap_zn_malloc of %lu bytes failed, using ch_malloc\n",
334 Debug(LDAP_DEBUG_NONE
,
335 "slap_zn_malloc: returning 0x%x, 0x%x\n",
336 ptr
, (int)ptr
>>(zh
->zh_zoneorder
+1), 0);
337 return (void*)ch_malloc(size
);
340 for (i
= zh
->zh_numzones
; i
< zh
->zh_numzones
+zh
->zh_deltazones
; i
++) {
341 zh
->zh_zones
[i
] = mmap(0, zh
->zh_zonesize
, PROT_READ
| PROT_WRITE
,
342 MAP_PRIVATE
, zh
->zh_fd
, 0);
343 zh
->zh_maps
[i
] = (unsigned char **)
344 ch_malloc((zh
->zh_zoneorder
- order_start
+ 1) *
345 sizeof(unsigned char *));
346 for (j
= 0; j
< zh
->zh_zoneorder
-order_start
+1; j
++) {
347 int shiftamt
= order_start
+ 1 + j
;
348 int nummaps
= zh
->zh_zonesize
>> shiftamt
;
351 if (!nummaps
) nummaps
= 1;
352 zh
->zh_maps
[i
][j
] = (unsigned char *)ch_malloc(nummaps
);
353 memset(zh
->zh_maps
[i
][j
], 0, nummaps
);
356 if (LDAP_LIST_EMPTY(&zh
->zh_zopool
)) {
357 slap_replenish_zopool(zh
);
359 zo
= LDAP_LIST_FIRST(&zh
->zh_zopool
);
360 LDAP_LIST_REMOVE(zo
, zo_link
);
361 zo
->zo_ptr
= zh
->zh_zones
[i
];
363 LDAP_LIST_INSERT_HEAD(&zh
->
364 zh_free
[zh
->zh_zoneorder
-order_start
],zo
,zo_link
);
366 if (LDAP_LIST_EMPTY(&zh
->zh_zopool
)) {
367 slap_replenish_zopool(zh
);
369 zo
= LDAP_LIST_FIRST(&zh
->zh_zopool
);
370 LDAP_LIST_REMOVE(zo
, zo_link
);
371 zo
->zo_ptr
= zh
->zh_zones
[i
];
372 zo
->zo_siz
= zh
->zh_zonesize
;
374 avl_insert(&zh
->zh_zonetree
, zo
, slap_zone_cmp
, avl_dup_error
);
375 ldap_pvt_thread_rdwr_init(&zh
->zh_znlock
[i
]);
377 zh
->zh_numzones
+= zh
->zh_deltazones
;
378 ldap_pvt_thread_mutex_unlock( &zh
->zh_mutex
);
384 slap_zn_calloc( ber_len_t n
, ber_len_t size
, void *ctx
)
388 new = slap_zn_malloc( n
*size
, ctx
);
390 memset( new, 0, n
*size
);
396 slap_zn_realloc(void *ptr
, ber_len_t size
, void *ctx
)
398 struct zone_heap
*zh
= ctx
;
399 int pad
= 2*sizeof(int)-1, pad_shift
;
400 int order_start
= -1, order
= -1;
401 struct zone_object zoi
, *zoo
;
402 ber_len_t
*p
= (ber_len_t
*)ptr
, *new;
406 struct zone_heap
*zone
= NULL
;
408 Debug(LDAP_DEBUG_NONE
,
409 "--> slap_zn_realloc: ptr=0x%x, size=%d\n", ptr
, size
, 0);
412 return slap_zn_malloc(size
, zh
);
418 ldap_pvt_thread_mutex_lock( &zh
->zh_mutex
);
419 zoo
= avl_find(zh
->zh_zonetree
, &zoi
, slap_zone_cmp
);
420 ldap_pvt_thread_mutex_unlock( &zh
->zh_mutex
);
423 /* Not our memory? */
425 /* duplicate of realloc behavior, oh well */
426 new = ber_memrealloc_x(ptr
, size
, NULL
);
430 Debug(LDAP_DEBUG_ANY
, "ch_realloc of %lu bytes failed\n",
433 exit( EXIT_FAILURE
);
436 assert(zoo
->zo_idx
!= -1);
438 zone
= zh
->zh_zones
[zoo
->zo_idx
];
441 slap_zn_free(ptr
, zh
);
445 newptr
= slap_zn_malloc(size
, zh
);
447 AC_MEMCPY(newptr
, ptr
, size
);
449 AC_MEMCPY(newptr
, ptr
, p
[-1]);
451 slap_zn_free(ptr
, zh
);
456 slap_zn_free(void *ptr
, void *ctx
)
458 struct zone_heap
*zh
= ctx
;
459 int size
, size_shift
, order_size
;
460 int pad
= 2*sizeof(int)-1, pad_shift
;
461 ber_len_t
*p
= (ber_len_t
*)ptr
, *tmpp
;
462 int order_start
= -1, order
= -1;
463 struct zone_object zoi
, *zoo
, *zo
;
465 int i
, k
, inserted
= 0, idx
;
466 struct zone_heap
*zone
= NULL
;
471 Debug(LDAP_DEBUG_NONE
, "--> slap_zn_free: ptr=0x%x\n", ptr
, 0, 0);
474 ldap_pvt_thread_mutex_lock( &zh
->zh_mutex
);
475 zoo
= avl_find(zh
->zh_zonetree
, &zoi
, slap_zone_cmp
);
476 ldap_pvt_thread_mutex_unlock( &zh
->zh_mutex
);
480 ber_memfree_x(ptr
, NULL
);
484 zone
= zh
->zh_zones
[idx
];
487 size_shift
= size
+ 2*sizeof(ber_len_t
) - 1;
490 } while (size_shift
>>= 1);
495 } while (pad_shift
>>= 1);
497 ldap_pvt_thread_mutex_lock( &zh
->zh_mutex
);
498 for (i
= order
, tmpp
= p
; i
<= zh
->zh_zoneorder
; i
++) {
499 order_size
= 1 << (i
+1);
500 diff
= (unsigned long)((char*)tmpp
- (char*)zone
) >> (i
+1);
501 zh
->zh_maps
[idx
][i
-order_start
][diff
>>3] &= (~(1 << (diff
& 0x7)));
502 if (diff
== ((diff
>>1)<<1)) {
503 if (!(zh
->zh_maps
[idx
][i
-order_start
][(diff
+1)>>3] &
504 (1<<((diff
+1)&0x7)))) {
505 zo
= LDAP_LIST_FIRST(&zh
->zh_free
[i
-order_start
]);
507 if ((char*)zo
->zo_ptr
== (char*)tmpp
) {
508 LDAP_LIST_REMOVE( zo
, zo_link
);
509 } else if ((char*)zo
->zo_ptr
==
510 (char*)tmpp
+ order_size
) {
511 LDAP_LIST_REMOVE(zo
, zo_link
);
514 zo
= LDAP_LIST_NEXT(zo
, zo_link
);
517 if (i
< zh
->zh_zoneorder
) {
520 Debug(LDAP_DEBUG_NONE
,
521 "slap_zn_free: merging 0x%x\n",
523 LDAP_LIST_INSERT_HEAD(&zh
->zh_free
[i
-order_start
+1],
528 if (LDAP_LIST_EMPTY(&zh
->zh_zopool
)) {
529 slap_replenish_zopool(zh
);
531 zo
= LDAP_LIST_FIRST(&zh
->zh_zopool
);
532 LDAP_LIST_REMOVE(zo
, zo_link
);
535 Debug(LDAP_DEBUG_NONE
,
536 "slap_zn_free: merging 0x%x\n",
538 LDAP_LIST_INSERT_HEAD(&zh
->zh_free
[i
-order_start
],
542 Debug(LDAP_DEBUG_ANY
, "slap_zn_free: "
543 "free object not found while bit is clear.\n",
550 if (LDAP_LIST_EMPTY(&zh
->zh_zopool
)) {
551 slap_replenish_zopool(zh
);
553 zo
= LDAP_LIST_FIRST(&zh
->zh_zopool
);
554 LDAP_LIST_REMOVE(zo
, zo_link
);
557 Debug(LDAP_DEBUG_NONE
,
558 "slap_zn_free: merging 0x%x\n",
560 LDAP_LIST_INSERT_HEAD(&zh
->zh_free
[i
-order_start
],
566 if (!(zh
->zh_maps
[idx
][i
-order_start
][(diff
-1)>>3] &
567 (1<<((diff
-1)&0x7)))) {
568 zo
= LDAP_LIST_FIRST(&zh
->zh_free
[i
-order_start
]);
570 if ((char*)zo
->zo_ptr
== (char*)tmpp
) {
571 LDAP_LIST_REMOVE(zo
, zo_link
);
572 } else if ((char*)tmpp
== zo
->zo_ptr
+ order_size
) {
573 LDAP_LIST_REMOVE(zo
, zo_link
);
577 zo
= LDAP_LIST_NEXT(zo
, zo_link
);
580 if (i
< zh
->zh_zoneorder
) {
582 Debug(LDAP_DEBUG_NONE
,
583 "slap_zn_free: merging 0x%x\n",
585 LDAP_LIST_INSERT_HEAD(&zh
->zh_free
[i
-order_start
+1],
590 if (LDAP_LIST_EMPTY(&zh
->zh_zopool
)) {
591 slap_replenish_zopool(zh
);
593 zo
= LDAP_LIST_FIRST(&zh
->zh_zopool
);
594 LDAP_LIST_REMOVE(zo
, zo_link
);
597 Debug(LDAP_DEBUG_NONE
,
598 "slap_zn_free: merging 0x%x\n",
600 LDAP_LIST_INSERT_HEAD(&zh
->zh_free
[i
-order_start
],
604 Debug(LDAP_DEBUG_ANY
, "slap_zn_free: "
605 "free object not found while bit is clear.\n",
612 if (LDAP_LIST_EMPTY(&zh
->zh_zopool
)) {
613 slap_replenish_zopool(zh
);
615 zo
= LDAP_LIST_FIRST(&zh
->zh_zopool
);
616 LDAP_LIST_REMOVE(zo
, zo_link
);
619 Debug(LDAP_DEBUG_NONE
,
620 "slap_zn_free: merging 0x%x\n",
622 LDAP_LIST_INSERT_HEAD(&zh
->zh_free
[i
-order_start
],
629 ldap_pvt_thread_mutex_unlock( &zh
->zh_mutex
);
634 slap_zone_cmp(const void *v1
, const void *v2
)
636 const struct zone_object
*zo1
= v1
;
637 const struct zone_object
*zo2
= v2
;
642 zpad
= zo2
->zo_siz
- 1;
643 ptr1
= (char*)(((unsigned long)zo1
->zo_ptr
+ zpad
) & ~zpad
);
644 ptr2
= (char*)zo2
->zo_ptr
+ ((char*)ptr1
- (char*)zo1
->zo_ptr
);
645 ptr2
= (char*)(((unsigned long)ptr2
+ zpad
) & ~zpad
);
646 return (int)((char*)ptr1
- (char*)ptr2
);
650 slap_replenish_zopool(
654 struct zone_heap
* zh
= ctx
;
655 struct zone_object
*zo_block
;
658 zo_block
= (struct zone_object
*)ch_malloc(
659 SLAP_ZONE_ZOBLOCK
* sizeof(struct zone_object
));
661 if ( zo_block
== NULL
) {
665 zo_block
[0].zo_blockhead
= 1;
666 LDAP_LIST_INSERT_HEAD(&zh
->zh_zopool
, &zo_block
[0], zo_link
);
667 for (i
= 1; i
< SLAP_ZONE_ZOBLOCK
; i
++) {
668 zo_block
[i
].zo_blockhead
= 0;
669 LDAP_LIST_INSERT_HEAD(&zh
->zh_zopool
, &zo_block
[i
], zo_link
);
681 struct zone_heap
* zh
= ctx
;
682 struct zone_object zoi
, *zoo
;
683 struct zone_heap
*zone
= NULL
;
684 int seqno
= *((ber_len_t
*)ptr
- 2);
685 int idx
= -1, rc
= 0;
686 int pad
= 2*sizeof(int)-1, pad_shift
;
687 int order_start
= -1, i
;
688 struct zone_object
*zo
;
693 } while (pad_shift
>>= 1);
698 ldap_pvt_thread_mutex_lock( &zh
->zh_mutex
);
699 zoo
= avl_find(zh
->zh_zonetree
, &zoi
, slap_zone_cmp
);
704 madvise(zh
->zh_zones
[idx
], zh
->zh_zonesize
, MADV_DONTNEED
);
705 for (i
= 0; i
< zh
->zh_zoneorder
- order_start
+ 1; i
++) {
706 int shiftamt
= order_start
+ 1 + i
;
707 int nummaps
= zh
->zh_zonesize
>> shiftamt
;
710 if (!nummaps
) nummaps
= 1;
711 memset(zh
->zh_maps
[idx
][i
], 0, nummaps
);
712 zo
= LDAP_LIST_FIRST(&zh
->zh_free
[i
]);
714 struct zone_object
*zo_tmp
= zo
;
715 zo
= LDAP_LIST_NEXT(zo
, zo_link
);
716 if (zo_tmp
&& zo_tmp
->zo_idx
== idx
) {
717 LDAP_LIST_REMOVE(zo_tmp
, zo_link
);
718 LDAP_LIST_INSERT_HEAD(&zh
->zh_zopool
, zo_tmp
, zo_link
);
722 if (LDAP_LIST_EMPTY(&zh
->zh_zopool
)) {
723 slap_replenish_zopool(zh
);
725 zo
= LDAP_LIST_FIRST(&zh
->zh_zopool
);
726 LDAP_LIST_REMOVE(zo
, zo_link
);
727 zo
->zo_ptr
= zh
->zh_zones
[idx
];
729 LDAP_LIST_INSERT_HEAD(&zh
->zh_free
[zh
->zh_zoneorder
-order_start
],
733 Debug(LDAP_DEBUG_NONE
, "zone not found for (ctx=0x%x, ptr=0x%x) !\n",
737 ldap_pvt_thread_mutex_unlock( &zh
->zh_mutex
);
738 Debug(LDAP_DEBUG_NONE
, "zone %d invalidate\n", idx
, 0, 0);
749 struct zone_heap
* zh
= ctx
;
750 struct zone_object zoi
, *zoo
;
751 struct zone_heap
*zone
= NULL
;
757 zoo
= avl_find(zh
->zh_zonetree
, &zoi
, slap_zone_cmp
);
762 assert(seqno
<= zh
->zh_seqno
[idx
]);
763 rc
= (seqno
== zh
->zh_seqno
[idx
]);
773 struct zone_heap
* zh
= ctx
;
774 ldap_pvt_thread_rdwr_rlock(&zh
->zh_lock
);
781 struct zone_heap
* zh
= ctx
;
782 ldap_pvt_thread_rdwr_runlock(&zh
->zh_lock
);
789 struct zone_heap
* zh
= ctx
;
790 ldap_pvt_thread_rdwr_wlock(&zh
->zh_lock
);
797 struct zone_heap
* zh
= ctx
;
798 ldap_pvt_thread_rdwr_wunlock(&zh
->zh_lock
);
806 struct zone_heap
* zh
= ctx
;
807 struct zone_object zoi
, *zoo
;
808 struct zone_heap
*zone
= NULL
;
814 ldap_pvt_thread_mutex_lock( &zh
->zh_mutex
);
815 zoo
= avl_find(zh
->zh_zonetree
, &zoi
, slap_zone_cmp
);
816 ldap_pvt_thread_mutex_unlock( &zh
->zh_mutex
);
821 ldap_pvt_thread_rdwr_rlock(&zh
->zh_znlock
[idx
]);
830 struct zone_heap
* zh
= ctx
;
831 struct zone_object zoi
, *zoo
;
832 struct zone_heap
*zone
= NULL
;
838 ldap_pvt_thread_mutex_lock( &zh
->zh_mutex
);
839 zoo
= avl_find(zh
->zh_zonetree
, &zoi
, slap_zone_cmp
);
840 ldap_pvt_thread_mutex_unlock( &zh
->zh_mutex
);
845 ldap_pvt_thread_rdwr_runlock(&zh
->zh_znlock
[idx
]);
854 struct zone_heap
* zh
= ctx
;
855 struct zone_object zoi
, *zoo
;
856 struct zone_heap
*zone
= NULL
;
862 ldap_pvt_thread_mutex_lock( &zh
->zh_mutex
);
863 zoo
= avl_find(zh
->zh_zonetree
, &zoi
, slap_zone_cmp
);
864 ldap_pvt_thread_mutex_unlock( &zh
->zh_mutex
);
869 ldap_pvt_thread_rdwr_wlock(&zh
->zh_znlock
[idx
]);
878 struct zone_heap
* zh
= ctx
;
879 struct zone_object zoi
, *zoo
;
880 struct zone_heap
*zone
= NULL
;
886 ldap_pvt_thread_mutex_lock( &zh
->zh_mutex
);
887 zoo
= avl_find(zh
->zh_zonetree
, &zoi
, slap_zone_cmp
);
888 ldap_pvt_thread_mutex_unlock( &zh
->zh_mutex
);
893 ldap_pvt_thread_rdwr_wunlock(&zh
->zh_znlock
[idx
]);
897 #define T_SEC_IN_USEC 1000000
900 slap_timediff(struct timeval
*tv_begin
, struct timeval
*tv_end
)
902 uint64_t t_begin
, t_end
, t_diff
;
904 t_begin
= T_SEC_IN_USEC
* tv_begin
->tv_sec
+ tv_begin
->tv_usec
;
905 t_end
= T_SEC_IN_USEC
* tv_end
->tv_sec
+ tv_end
->tv_usec
;
906 t_diff
= t_end
- t_begin
;
915 slap_set_timing(struct timeval
*tv_set
)
917 gettimeofday(tv_set
, (struct timezone
*)NULL
);
921 slap_measure_timing(struct timeval
*tv_set
, struct timeval
*tv_measure
)
923 gettimeofday(tv_measure
, (struct timezone
*)NULL
);
924 return(slap_timediff(tv_set
, tv_measure
));
927 #define EMA_WEIGHT 0.999000
928 #define SLAP_ZN_LATENCY_HISTORY_QLEN 500
930 slap_zn_latency_history(void* ctx
, int ea_latency
)
932 /* TODO: monitor /proc/stat (swap) as well */
933 struct zone_heap
* zh
= ctx
;
936 zh
->zh_ema_latency
= (double)ea_latency
* (1.0 - EMA_WEIGHT
)
937 + zh
->zh_ema_latency
* EMA_WEIGHT
;
938 if (!zh
->zh_swapping
&& zh
->zh_ema_samples
++ % 100 == 99) {
939 struct zone_latency_history
*zlh_entry
;
940 zlh_entry
= ch_calloc(1, sizeof(struct zone_latency_history
));
941 zlh_entry
->zlh_latency
= zh
->zh_ema_latency
;
942 LDAP_STAILQ_INSERT_TAIL(
943 &zh
->zh_latency_history_queue
, zlh_entry
, zlh_next
);
944 zh
->zh_latency_history_qlen
++;
945 while (zh
->zh_latency_history_qlen
> SLAP_ZN_LATENCY_HISTORY_QLEN
) {
946 struct zone_latency_history
*zlh
;
947 zlh
= LDAP_STAILQ_FIRST(&zh
->zh_latency_history_queue
);
948 LDAP_STAILQ_REMOVE_HEAD(
949 &zh
->zh_latency_history_queue
, zlh_next
);
950 zh
->zh_latency_history_qlen
--;
953 if (zh
->zh_latency_history_qlen
== SLAP_ZN_LATENCY_HISTORY_QLEN
) {
954 struct zone_latency_history
*zlh_first
, *zlh_last
;
955 zlh_first
= LDAP_STAILQ_FIRST(&zh
->zh_latency_history_queue
);
956 zlh_last
= LDAP_STAILQ_LAST(&zh
->zh_latency_history_queue
,
957 zone_latency_history
, zlh_next
);
958 t_diff
= zlh_last
->zlh_latency
- zlh_first
->zlh_latency
;
960 if (t_diff
>= 2000) {
961 zh
->zh_latency_jump
++;
963 zh
->zh_latency_jump
= 0;
965 if (zh
->zh_latency_jump
> 3) {
966 zh
->zh_latency_jump
= 0;
970 return zh
->zh_swapping
;
972 #endif /* SLAP_ZONE_ALLOC */